2023-10-09T03:32:24.650 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T03:32:24.651 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology --verbose --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444 -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/orig.config.yaml 2023-10-09T03:32:24.715 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T03:32:24.804 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7417444' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 5305 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi163.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi181.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T03:32:24.805 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T03:32:24.806 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T03:32:24.806 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T03:32:24.807 INFO:teuthology.task.internal:Checking packages... 2023-10-09T03:32:24.831 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T03:32:24.831 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:32:24.831 INFO:teuthology.packaging:ref: None 2023-10-09T03:32:24.831 INFO:teuthology.packaging:tag: None 2023-10-09T03:32:24.832 INFO:teuthology.packaging:branch: reef 2023-10-09T03:32:24.832 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:32:24.832 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T03:32:25.155 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T03:32:25.155 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T03:32:25.162 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T03:32:25.162 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T03:32:25.202 INFO:teuthology.task.internal:Saving configuration 2023-10-09T03:32:25.228 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T03:32:25.235 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T03:32:25.260 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi163.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:26:37.266921', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T03:32:25.283 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:26:37.268663', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T03:32:25.283 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T03:32:25.291 INFO:teuthology.task.internal:roles: ubuntu@smithi163.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T03:32:25.291 INFO:teuthology.task.internal:roles: ubuntu@smithi181.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T03:32:25.291 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T03:32:25.364 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f6fcf33eaf0>, signals=[15]) 2023-10-09T03:32:25.364 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T03:32:25.370 INFO:teuthology.task.internal:Opening connections... 2023-10-09T03:32:25.370 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T03:32:25.372 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:32:25.438 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi181.front.sepia.ceph.com 2023-10-09T03:32:25.439 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:32:25.507 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T03:32:25.514 DEBUG:teuthology.orchestra.run.smithi163:> uname -m 2023-10-09T03:32:25.534 INFO:teuthology.orchestra.run.smithi163.stdout:x86_64 2023-10-09T03:32:25.534 DEBUG:teuthology.orchestra.run.smithi163:> cat /etc/os-release 2023-10-09T03:32:25.594 INFO:teuthology.orchestra.run.smithi163.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T03:32:25.594 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T03:32:25.594 INFO:teuthology.orchestra.run.smithi163.stdout:ID="rhel" 2023-10-09T03:32:25.594 INFO:teuthology.orchestra.run.smithi163.stdout:ID_LIKE="fedora" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:VERSION_ID="8.6" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:PLATFORM_ID="platform:el8" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:ANSI_COLOR="0;31" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T03:32:25.595 INFO:teuthology.orchestra.run.smithi163.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T03:32:25.596 INFO:teuthology.orchestra.run.smithi163.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T03:32:25.596 INFO:teuthology.orchestra.run.smithi163.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T03:32:25.596 INFO:teuthology.lock.ops:Updating smithi163.front.sepia.ceph.com on lock server 2023-10-09T03:32:25.623 DEBUG:teuthology.orchestra.run.smithi181:> uname -m 2023-10-09T03:32:25.644 INFO:teuthology.orchestra.run.smithi181.stdout:x86_64 2023-10-09T03:32:25.644 DEBUG:teuthology.orchestra.run.smithi181:> cat /etc/os-release 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:ID="rhel" 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:ID_LIKE="fedora" 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_ID="8.6" 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:PLATFORM_ID="platform:el8" 2023-10-09T03:32:25.704 INFO:teuthology.orchestra.run.smithi181.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:ANSI_COLOR="0;31" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T03:32:25.705 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T03:32:25.705 INFO:teuthology.lock.ops:Updating smithi181.front.sepia.ceph.com on lock server 2023-10-09T03:32:25.723 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T03:32:25.731 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T03:32:25.736 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T03:32:25.737 DEBUG:teuthology.orchestra.run.smithi163:> test '!' -e /home/ubuntu/cephtest 2023-10-09T03:32:25.739 DEBUG:teuthology.orchestra.run.smithi181:> test '!' -e /home/ubuntu/cephtest 2023-10-09T03:32:25.763 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T03:32:25.768 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T03:32:25.769 DEBUG:teuthology.orchestra.run.smithi163:> test -z $(ls -A /var/lib/ceph) 2023-10-09T03:32:25.798 DEBUG:teuthology.orchestra.run.smithi181:> test -z $(ls -A /var/lib/ceph) 2023-10-09T03:32:25.830 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T03:32:25.883 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T03:32:25.900 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T03:32:25.900 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T03:32:25.900 DEBUG:teuthology.orchestra.run.smithi163:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T03:32:25.901 DEBUG:teuthology.orchestra.run.smithi181:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T03:32:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:32:25.919 DEBUG:teuthology.orchestra.run.smithi163:> uname -r 2023-10-09T03:32:25.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:32:25.920 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2023-10-09T03:32:25.979 INFO:teuthology.orchestra.run.smithi163.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:25.979 INFO:teuthology.orchestra.run.smithi181.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:25.979 INFO:teuthology.task.kernel:Running kernel on smithi163: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:25.979 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum install -y kernel 2023-10-09T03:32:25.980 INFO:teuthology.task.kernel:Running kernel on smithi181: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:25.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum install -y kernel 2023-10-09T03:32:27.137 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:32:27.137 INFO:teuthology.orchestra.run.smithi163.stdout:Unable to read consumer identity 2023-10-09T03:32:27.161 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:32:27.161 INFO:teuthology.orchestra.run.smithi181.stdout:Unable to read consumer identity 2023-10-09T03:32:28.672 INFO:teuthology.orchestra.run.smithi181.stdout:Copr repo for python3-asyncssh owned by ceph 12 kB/s | 3.5 kB 00:00 2023-10-09T03:32:28.697 INFO:teuthology.orchestra.run.smithi163.stdout:Copr repo for python3-asyncssh owned by ceph 11 kB/s | 3.5 kB 00:00 2023-10-09T03:32:29.405 INFO:teuthology.orchestra.run.smithi181.stdout:Extra Packages for Enterprise Linux 22 MB/s | 16 MB 00:00 2023-10-09T03:32:29.597 INFO:teuthology.orchestra.run.smithi163.stdout:Extra Packages for Enterprise Linux 17 MB/s | 16 MB 00:00 2023-10-09T03:32:33.089 INFO:teuthology.orchestra.run.smithi181.stdout:lab-extras 95 kB/s | 24 kB 00:00 2023-10-09T03:32:33.156 INFO:teuthology.orchestra.run.smithi163.stdout:lab-extras 316 kB/s | 24 kB 00:00 2023-10-09T03:32:33.783 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T03:32:33.842 INFO:teuthology.orchestra.run.smithi163.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T03:32:33.850 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:32:33.853 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:32:33.853 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:32:33.897 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:32:33.898 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:32:33.899 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:32:33.961 DEBUG:teuthology.orchestra.run.smithi181:> echo no | sudo yum reinstall kernel || true 2023-10-09T03:32:34.018 DEBUG:teuthology.orchestra.run.smithi163:> echo no | sudo yum reinstall kernel || true 2023-10-09T03:32:34.291 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:32:34.292 INFO:teuthology.orchestra.run.smithi181.stdout:Unable to read consumer identity 2023-10-09T03:32:34.391 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:32:34.391 INFO:teuthology.orchestra.run.smithi163.stdout:Unable to read consumer identity 2023-10-09T03:32:34.555 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 03:32:32 AM UTC. 2023-10-09T03:32:34.596 INFO:teuthology.orchestra.run.smithi181.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:32:34.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error: No packages marked for reinstall. 2023-10-09T03:32:34.657 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:00:01 ago on Mon 09 Oct 2023 03:32:33 AM UTC. 2023-10-09T03:32:34.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum reinstall -y kernel || true 2023-10-09T03:32:34.698 INFO:teuthology.orchestra.run.smithi163.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:32:34.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error: No packages marked for reinstall. 2023-10-09T03:32:34.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum reinstall -y kernel || true 2023-10-09T03:32:35.011 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:32:35.011 INFO:teuthology.orchestra.run.smithi181.stdout:Unable to read consumer identity 2023-10-09T03:32:35.130 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:32:35.131 INFO:teuthology.orchestra.run.smithi163.stdout:Unable to read consumer identity 2023-10-09T03:32:35.272 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:00:03 ago on Mon 09 Oct 2023 03:32:32 AM UTC. 2023-10-09T03:32:35.313 INFO:teuthology.orchestra.run.smithi181.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:32:35.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error: No packages marked for reinstall. 2023-10-09T03:32:35.387 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T03:32:35.392 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 03:32:33 AM UTC. 2023-10-09T03:32:35.432 INFO:teuthology.orchestra.run.smithi163.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:32:35.467 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:35.467 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:35.467 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T03:32:35.467 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T03:32:35.467 DEBUG:teuthology.orchestra.run.smithi181:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T03:32:35.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error: No packages marked for reinstall. 2023-10-09T03:32:35.493 INFO:teuthology.orchestra.run.smithi181.stdout:ttyS1 2023-10-09T03:32:35.518 DEBUG:teuthology.parallel:result is None 2023-10-09T03:32:35.519 DEBUG:teuthology.orchestra.run.smithi163:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T03:32:35.599 INFO:teuthology.orchestra.run.smithi163.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:35.599 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:32:35.599 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T03:32:35.599 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T03:32:35.600 DEBUG:teuthology.orchestra.run.smithi163:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T03:32:35.627 INFO:teuthology.orchestra.run.smithi163.stdout:ttyS1 2023-10-09T03:32:35.641 DEBUG:teuthology.parallel:result is None 2023-10-09T03:32:35.641 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T03:32:35.648 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T03:32:35.648 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T03:32:35.684 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T03:32:35.700 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T03:32:35.706 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T03:32:35.712 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T03:32:35.712 DEBUG:teuthology.orchestra.run.smithi163:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T03:32:35.741 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T03:32:35.773 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T03:32:35.780 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T03:32:35.780 DEBUG:teuthology.orchestra.run.smithi163:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T03:32:35.798 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T03:32:35.837 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:32:35.847 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:32:35.868 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:32:35.878 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:32:35.880 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T03:32:35.887 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T03:32:35.887 DEBUG:teuthology.orchestra.run.smithi163:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T03:32:35.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T03:32:35.919 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T03:32:35.927 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T03:32:35.928 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T03:32:35.959 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T03:32:35.976 DEBUG:teuthology.orchestra.run.smithi163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:32:36.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:32:36.105 DEBUG:teuthology.orchestra.run.smithi163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:32:36.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:32:36.229 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:32:36.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T03:32:36.297 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:32:36.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:32:36.392 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:32:36.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:32:36.517 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:32:36.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T03:32:36.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo service rsyslog restart 2023-10-09T03:32:36.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo service rsyslog restart 2023-10-09T03:32:36.659 INFO:teuthology.orchestra.run.smithi163.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T03:32:36.681 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T03:32:37.094 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T03:32:37.102 INFO:teuthology.task.internal:Starting timer... 2023-10-09T03:32:37.102 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T03:32:37.112 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T03:32:37.121 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T03:32:37.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo service auditd rotate 2023-10-09T03:32:37.190 INFO:teuthology.orchestra.run.smithi163.stdout:Rotating logs: 2023-10-09T03:32:37.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo service auditd rotate 2023-10-09T03:32:37.253 INFO:teuthology.orchestra.run.smithi181.stdout:Rotating logs: 2023-10-09T03:32:37.255 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T03:32:37.255 DEBUG:teuthology.orchestra.run.smithi163:> /usr/sbin/getenforce 2023-10-09T03:32:37.277 INFO:teuthology.orchestra.run.smithi163.stdout:Permissive 2023-10-09T03:32:37.278 DEBUG:teuthology.orchestra.run.smithi181:> /usr/sbin/getenforce 2023-10-09T03:32:37.297 INFO:teuthology.orchestra.run.smithi181.stdout:Permissive 2023-10-09T03:32:37.297 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi163.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi181.front.sepia.ceph.com': 'permissive'} 2023-10-09T03:32:37.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T03:32:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:32:37.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T03:32:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:32:37.374 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T03:32:37.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo /usr/sbin/setenforce permissive 2023-10-09T03:32:37.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo /usr/sbin/setenforce permissive 2023-10-09T03:32:37.440 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T03:32:37.449 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T03:32:37.454 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T03:32:37.589 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T03:32:37.604 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T03:32:37.604 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi163.front.sepia.ceph.com,smithi181.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T03:41:52.911 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi163.front.sepia.ceph.com'), Remote(name='ubuntu@smithi181.front.sepia.ceph.com')] 2023-10-09T03:41:52.953 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T03:41:52.954 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:41:53.023 DEBUG:teuthology.orchestra.run.smithi163:> true 2023-10-09T03:41:53.048 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi163.front.sepia.ceph.com' 2023-10-09T03:41:53.048 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T03:41:53.049 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:41:53.118 DEBUG:teuthology.orchestra.run.smithi181:> true 2023-10-09T03:41:53.188 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2023-10-09T03:41:53.189 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T03:41:53.200 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T03:41:53.200 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T03:41:53.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T03:41:53.203 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T03:41:53.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T03:41:53.232 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T03:41:53.246 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T03:41:53.263 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T03:41:53.278 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T03:41:53.315 INFO:teuthology.orchestra.run.smithi163.stderr:sudo: ntpd: command not found 2023-10-09T03:41:53.326 INFO:teuthology.orchestra.run.smithi163.stdout:506 Cannot talk to daemon 2023-10-09T03:41:53.341 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T03:41:53.355 INFO:teuthology.orchestra.run.smithi163.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T03:41:53.366 INFO:teuthology.orchestra.run.smithi181.stderr:sudo: ntpd: command not found 2023-10-09T03:41:53.378 INFO:teuthology.orchestra.run.smithi181.stdout:506 Cannot talk to daemon 2023-10-09T03:41:53.392 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T03:41:53.406 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T03:41:53.438 INFO:teuthology.orchestra.run.smithi163.stderr:bash: ntpq: command not found 2023-10-09T03:41:53.441 INFO:teuthology.orchestra.run.smithi163.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T03:41:53.442 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================== 2023-10-09T03:41:53.442 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.442 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.442 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.442 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.475 INFO:teuthology.orchestra.run.smithi181.stderr:bash: ntpq: command not found 2023-10-09T03:41:53.479 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T03:41:53.479 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2023-10-09T03:41:53.479 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.479 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.479 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.479 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:41:53.480 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T03:41:53.489 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T03:41:53.489 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T03:41:53.489 DEBUG:teuthology.orchestra.run.smithi163:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T03:41:53.490 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi181.front.sepia.ceph.com 2023-10-09T03:41:53.490 DEBUG:teuthology.orchestra.run.smithi181:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T03:41:53.863 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:41:53.890 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:41:55.332 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 03:40:15 AM UTC. 2023-10-09T03:41:55.450 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:50 ago on Mon 09 Oct 2023 03:40:05 AM UTC. 2023-10-09T03:41:57.052 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:41:57.053 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout: Package Architecture Version Repository Size 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout:Resetting modules: 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout: container-tools 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:41:57.054 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:41:57.166 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout:Resetting modules: 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: container-tools 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:41:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:41:58.022 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:42 ago on Mon 09 Oct 2023 03:40:15 AM UTC. 2023-10-09T03:41:58.124 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:52 ago on Mon 09 Oct 2023 03:40:05 AM UTC. 2023-10-09T03:42:03.242 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:42:03.728 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:42:05.030 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:42:05.247 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:49 ago on Mon 09 Oct 2023 03:40:15 AM UTC. 2023-10-09T03:42:05.474 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:42:07.023 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:01 ago on Mon 09 Oct 2023 03:40:05 AM UTC. 2023-10-09T03:42:07.099 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:42:07.191 INFO:teuthology.orchestra.run.smithi163.stdout:========================================================================================================================== 2023-10-09T03:42:07.191 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:42:07.191 INFO:teuthology.orchestra.run.smithi163.stdout:========================================================================================================================== 2023-10-09T03:42:07.191 INFO:teuthology.orchestra.run.smithi163.stdout:Installing group/module packages: 2023-10-09T03:42:07.191 INFO:teuthology.orchestra.run.smithi163.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout:Downgrading: 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T03:42:07.192 INFO:teuthology.orchestra.run.smithi163.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout:Installing module profiles: 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: container-tools/common 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout:Enabling module streams: 2023-10-09T03:42:07.193 INFO:teuthology.orchestra.run.smithi163.stdout: container-tools 3.0 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout:========================================================================================================================== 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout:Install 7 Packages 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout:Downgrade 12 Packages 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:42:07.194 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 51 M 2023-10-09T03:42:07.195 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:42:07.488 INFO:teuthology.orchestra.run.smithi163.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 180 kB/s | 53 kB 00:00 2023-10-09T03:42:07.510 INFO:teuthology.orchestra.run.smithi163.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 163 kB/s | 51 kB 00:00 2023-10-09T03:42:07.535 INFO:teuthology.orchestra.run.smithi163.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 267 kB/s | 91 kB 00:00 2023-10-09T03:42:07.636 INFO:teuthology.orchestra.run.smithi163.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 3.4 MB/s | 511 kB 00:00 2023-10-09T03:42:07.736 INFO:teuthology.orchestra.run.smithi163.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 307 kB/s | 69 kB 00:00 2023-10-09T03:42:07.861 INFO:teuthology.orchestra.run.smithi163.stdout:(6/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 8.9 MB/s | 2.9 MB 00:00 2023-10-09T03:42:07.894 INFO:teuthology.orchestra.run.smithi163.stdout:(7/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 278 kB/s | 72 kB 00:00 2023-10-09T03:42:07.986 INFO:teuthology.orchestra.run.smithi163.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 446 kB/s | 56 kB 00:00 2023-10-09T03:42:08.029 INFO:teuthology.orchestra.run.smithi163.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 381 kB/s | 51 kB 00:00 2023-10-09T03:42:08.154 INFO:teuthology.orchestra.run.smithi163.stdout:(10/19): podman-3.0.1-9.module+el8.6.0+14874+64 26 MB/s | 11 MB 00:00 2023-10-09T03:42:08.187 INFO:teuthology.orchestra.run.smithi163.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 2.0 MB/s | 322 kB 00:00 2023-10-09T03:42:08.271 INFO:teuthology.orchestra.run.smithi163.stdout:(12/19): yajl-2.1.0-10.el8.x86_64.rpm 349 kB/s | 41 kB 00:00 2023-10-09T03:42:08.413 INFO:teuthology.orchestra.run.smithi163.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-10-09T03:42:08.538 INFO:teuthology.orchestra.run.smithi163.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 24 MB/s | 6.4 MB 00:00 2023-10-09T03:42:08.805 INFO:teuthology.orchestra.run.smithi163.stdout:(15/19): containernetworking-plugins-0.9.1-1.mo 23 MB/s | 19 MB 00:00 2023-10-09T03:42:08.847 INFO:teuthology.orchestra.run.smithi163.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 3.5 MB/s | 1.1 MB 00:00 2023-10-09T03:42:08.865 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:42:08.939 INFO:teuthology.orchestra.run.smithi163.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.4 MB/s | 185 kB 00:00 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout:========================================================================================================================== 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout:========================================================================================================================== 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout:Installing group/module packages: 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T03:42:08.956 INFO:teuthology.orchestra.run.smithi181.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout:Downgrading: 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T03:42:08.957 INFO:teuthology.orchestra.run.smithi181.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout:Installing module profiles: 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: container-tools/common 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout:Enabling module streams: 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: container-tools 3.0 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:42:08.958 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:42:08.959 INFO:teuthology.orchestra.run.smithi181.stdout:========================================================================================================================== 2023-10-09T03:42:08.959 INFO:teuthology.orchestra.run.smithi181.stdout:Install 7 Packages 2023-10-09T03:42:08.959 INFO:teuthology.orchestra.run.smithi181.stdout:Downgrade 12 Packages 2023-10-09T03:42:08.959 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:42:08.959 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 51 M 2023-10-09T03:42:08.959 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:42:08.981 INFO:teuthology.orchestra.run.smithi163.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 379 kB/s | 51 kB 00:00 2023-10-09T03:42:09.056 INFO:teuthology.orchestra.run.smithi163.stdout:(19/19): buildah-1.19.9-3.module+el8.6.0+14874+ 11 MB/s | 6.8 MB 00:00 2023-10-09T03:42:09.057 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:42:09.057 INFO:teuthology.orchestra.run.smithi163.stdout:Total 27 MB/s | 51 MB 00:01 2023-10-09T03:42:09.263 INFO:teuthology.orchestra.run.smithi181.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 174 kB/s | 53 kB 00:00 2023-10-09T03:42:09.289 INFO:teuthology.orchestra.run.smithi181.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 156 kB/s | 51 kB 00:00 2023-10-09T03:42:09.314 INFO:teuthology.orchestra.run.smithi181.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 257 kB/s | 91 kB 00:00 2023-10-09T03:42:09.347 INFO:teuthology.orchestra.run.smithi181.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.2 MB/s | 69 kB 00:00 2023-10-09T03:42:09.381 INFO:teuthology.orchestra.run.smithi181.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.3 MB/s | 511 kB 00:00 2023-10-09T03:42:09.473 INFO:teuthology.orchestra.run.smithi181.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 575 kB/s | 72 kB 00:00 2023-10-09T03:42:09.534 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:42:09.540 INFO:teuthology.orchestra.run.smithi181.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 13 MB/s | 2.9 MB 00:00 2023-10-09T03:42:09.577 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:42:09.577 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:42:09.598 INFO:teuthology.orchestra.run.smithi181.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 446 kB/s | 56 kB 00:00 2023-10-09T03:42:09.799 INFO:teuthology.orchestra.run.smithi181.stdout:(9/19): podman-3.0.1-9.module+el8.6.0+14874+644 26 MB/s | 11 MB 00:00 2023-10-09T03:42:10.342 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:42:10.343 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:42:10.343 INFO:teuthology.orchestra.run.smithi181.stdout:(10/19): conmon-2.0.26-1.module+el8.6.0+14874+6 180 kB/s | 51 kB 00:00 2023-10-09T03:42:10.343 INFO:teuthology.orchestra.run.smithi181.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 2.4 MB/s | 322 kB 00:00 2023-10-09T03:42:10.343 INFO:teuthology.orchestra.run.smithi181.stdout:(12/19): yajl-2.1.0-10.el8.x86_64.rpm 305 kB/s | 41 kB 00:00 2023-10-09T03:42:10.343 INFO:teuthology.orchestra.run.smithi181.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 13 MB/s | 2.2 MB 00:00 2023-10-09T03:42:10.343 INFO:teuthology.orchestra.run.smithi181.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 21 MB/s | 6.4 MB 00:00 2023-10-09T03:42:10.467 INFO:teuthology.orchestra.run.smithi181.stdout:(15/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-10-09T03:42:10.946 INFO:teuthology.orchestra.run.smithi181.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 4.1 MB/s | 1.1 MB 00:00 2023-10-09T03:42:10.946 INFO:teuthology.orchestra.run.smithi181.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.7 MB/s | 185 kB 00:00 2023-10-09T03:42:10.947 INFO:teuthology.orchestra.run.smithi181.stdout:(18/19): buildah-1.19.9-3.module+el8.6.0+14874+ 12 MB/s | 6.8 MB 00:00 2023-10-09T03:42:10.947 INFO:teuthology.orchestra.run.smithi181.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 302 kB/s | 51 kB 00:00 2023-10-09T03:42:10.947 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:42:10.947 INFO:teuthology.orchestra.run.smithi181.stdout:Total 29 MB/s | 51 MB 00:01 2023-10-09T03:42:11.148 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:42:11.160 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:42:11.199 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T03:42:11.203 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:42:11.203 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:42:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:42:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:42:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:42:11.580 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:42:11.661 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:42:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:42:12.781 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:42:12.833 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T03:42:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:42:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:42:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:42:13.211 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:42:28.392 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:42:30.169 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T03:42:30.274 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:42:30.409 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T03:42:30.597 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T03:42:30.775 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T03:42:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T03:42:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T03:42:30.986 INFO:teuthology.orchestra.run.smithi163.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T03:42:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:42:31.790 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T03:42:32.018 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T03:42:32.031 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T03:42:32.240 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T03:42:32.448 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T03:42:32.622 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T03:42:32.832 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T03:42:32.832 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T03:42:32.832 INFO:teuthology.orchestra.run.smithi181.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T03:42:32.832 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:42:33.425 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T03:42:33.578 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T03:42:33.704 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T03:42:33.807 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T03:42:34.076 INFO:teuthology.orchestra.run.smithi163.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T03:42:34.366 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T03:42:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T03:42:35.251 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T03:42:35.542 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T03:42:35.819 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T03:42:35.916 INFO:teuthology.orchestra.run.smithi181.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T03:42:36.017 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T03:42:36.164 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T03:42:36.200 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T03:42:36.376 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T03:42:36.491 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T03:42:36.491 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:42:36.781 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:42:36.892 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T03:42:36.955 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T03:42:36.956 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:42:37.119 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:42:37.252 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:42:37.411 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T03:42:37.568 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T03:42:37.616 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T03:42:37.772 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T03:42:37.831 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T03:42:37.910 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T03:42:38.006 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T03:42:38.043 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T03:42:38.182 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T03:42:38.196 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T03:42:38.197 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:42:38.321 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T03:42:38.322 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:42:38.359 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:42:38.637 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:42:38.811 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T03:42:38.811 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:42:38.887 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T03:42:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:42:39.182 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:42:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:42:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T03:42:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T03:42:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T03:42:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:42:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T03:42:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T03:42:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T03:42:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T03:42:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T03:42:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T03:42:39.366 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T03:42:39.516 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T03:42:39.694 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T03:42:39.866 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T03:42:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T03:42:40.150 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T03:42:40.151 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:42:40.332 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:42:40.750 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T03:42:40.750 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:25 ago on Mon 09 Oct 2023 03:40:15 AM UTC. 2023-10-09T03:42:40.855 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T03:42:41.264 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T03:42:41.265 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T03:42:41.266 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T03:42:41.267 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:42:41.267 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T03:42:41.267 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T03:42:41.267 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T03:42:41.268 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T03:42:41.268 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T03:42:41.268 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T03:42:42.729 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T03:42:42.729 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:37 ago on Mon 09 Oct 2023 03:40:05 AM UTC. 2023-10-09T03:42:45.924 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:42:46.115 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout:Downgraded: 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.116 INFO:teuthology.orchestra.run.smithi163.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.117 INFO:teuthology.orchestra.run.smithi163.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:46.118 INFO:teuthology.orchestra.run.smithi163.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:46.118 INFO:teuthology.orchestra.run.smithi163.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T03:42:46.118 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:42:46.118 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:42:46.308 DEBUG:teuthology.parallel:result is None 2023-10-09T03:42:47.406 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:42:47.597 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:42:47.597 INFO:teuthology.orchestra.run.smithi181.stdout:Downgraded: 2023-10-09T03:42:47.597 INFO:teuthology.orchestra.run.smithi181.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.597 INFO:teuthology.orchestra.run.smithi181.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:47.597 INFO:teuthology.orchestra.run.smithi181.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:42:47.598 INFO:teuthology.orchestra.run.smithi181.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:42:47.599 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:42:47.790 DEBUG:teuthology.parallel:result is None 2023-10-09T03:42:47.909 INFO:teuthology.run_tasks:Running task install... 2023-10-09T03:42:47.918 DEBUG:teuthology.task.install:project ceph 2023-10-09T03:42:47.918 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T03:42:47.918 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T03:42:47.919 INFO:teuthology.task.install:Using flavor: default 2023-10-09T03:42:47.924 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T03:42:47.925 INFO:teuthology.task.install:extra packages: [] 2023-10-09T03:42:47.925 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T03:42:47.925 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:42:47.925 INFO:teuthology.packaging:ref: None 2023-10-09T03:42:47.925 INFO:teuthology.packaging:tag: None 2023-10-09T03:42:47.926 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:42:47.926 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:42:47.926 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:42:47.928 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T03:42:47.928 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:42:47.928 INFO:teuthology.packaging:ref: None 2023-10-09T03:42:47.928 INFO:teuthology.packaging:tag: None 2023-10-09T03:42:47.928 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:42:47.928 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:42:47.929 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:42:48.289 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T03:42:48.290 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T03:42:48.315 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T03:42:48.315 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T03:42:48.430 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T03:42:48.430 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:42:48.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T03:42:48.458 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T03:42:48.458 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:42:48.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T03:42:48.464 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T03:42:48.464 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:42:48.464 INFO:teuthology.packaging:ref: None 2023-10-09T03:42:48.464 INFO:teuthology.packaging:tag: None 2023-10-09T03:42:48.464 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:42:48.464 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:42:48.464 DEBUG:teuthology.orchestra.run.smithi163:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T03:42:48.487 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T03:42:48.487 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:42:48.487 INFO:teuthology.packaging:ref: None 2023-10-09T03:42:48.488 INFO:teuthology.packaging:tag: None 2023-10-09T03:42:48.488 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:42:48.488 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:42:48.488 DEBUG:teuthology.orchestra.run.smithi181:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T03:42:48.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T03:42:48.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T03:42:48.618 DEBUG:teuthology.orchestra.run.smithi163:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T03:42:48.634 DEBUG:teuthology.orchestra.run.smithi181:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T03:42:48.686 INFO:teuthology.orchestra.run.smithi163.stdout:check_obsoletes = 1 2023-10-09T03:42:48.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum clean all 2023-10-09T03:42:48.701 INFO:teuthology.orchestra.run.smithi181.stdout:check_obsoletes = 1 2023-10-09T03:42:48.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum clean all 2023-10-09T03:42:49.063 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:42:49.072 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:42:49.745 INFO:teuthology.orchestra.run.smithi163.stdout:44 files removed 2023-10-09T03:42:49.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-radosgw 2023-10-09T03:42:49.805 INFO:teuthology.orchestra.run.smithi181.stdout:44 files removed 2023-10-09T03:42:49.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-radosgw 2023-10-09T03:42:50.124 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:42:50.185 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:42:51.023 INFO:teuthology.orchestra.run.smithi163.stdout:ceph packages for x86_64 301 kB/s | 78 kB 00:00 2023-10-09T03:42:51.150 INFO:teuthology.orchestra.run.smithi181.stdout:ceph packages for x86_64 188 kB/s | 78 kB 00:00 2023-10-09T03:42:51.407 INFO:teuthology.orchestra.run.smithi163.stdout:ceph noarch packages 26 kB/s | 9.1 kB 00:00 2023-10-09T03:42:51.501 INFO:teuthology.orchestra.run.smithi181.stdout:ceph noarch packages 29 kB/s | 9.1 kB 00:00 2023-10-09T03:42:51.716 INFO:teuthology.orchestra.run.smithi163.stdout:ceph source packages 5.0 kB/s | 1.4 kB 00:00 2023-10-09T03:42:51.735 INFO:teuthology.orchestra.run.smithi181.stdout:ceph source packages 6.8 kB/s | 1.4 kB 00:00 2023-10-09T03:42:51.842 INFO:teuthology.orchestra.run.smithi163.stdout:Copr repo for python3-asyncssh owned by ceph 32 kB/s | 3.5 kB 00:00 2023-10-09T03:42:51.877 INFO:teuthology.orchestra.run.smithi181.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2023-10-09T03:42:53.070 INFO:teuthology.orchestra.run.smithi163.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-10-09T03:42:53.478 INFO:teuthology.orchestra.run.smithi181.stdout:Extra Packages for Enterprise Linux 10 MB/s | 16 MB 00:01 2023-10-09T03:42:56.568 INFO:teuthology.orchestra.run.smithi163.stdout:lab-extras 365 kB/s | 24 kB 00:00 2023-10-09T03:42:57.037 INFO:teuthology.orchestra.run.smithi181.stdout:lab-extras 416 kB/s | 24 kB 00:00 2023-10-09T03:42:57.495 INFO:teuthology.orchestra.run.smithi163.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 55 MB/s | 47 MB 00:00 2023-10-09T03:42:57.922 INFO:teuthology.orchestra.run.smithi181.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 58 MB/s | 47 MB 00:00 2023-10-09T03:43:05.387 INFO:teuthology.orchestra.run.smithi163.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 63 MB/s | 53 MB 00:00 2023-10-09T03:43:05.756 INFO:teuthology.orchestra.run.smithi181.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 64 MB/s | 53 MB 00:00 2023-10-09T03:43:16.510 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:43:16.512 INFO:teuthology.orchestra.run.smithi163.stdout:==================================================================================================== 2023-10-09T03:43:16.512 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout:==================================================================================================== 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout:Upgrading: 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T03:43:16.513 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T03:43:16.514 INFO:teuthology.orchestra.run.smithi163.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout:==================================================================================================== 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout:Install 21 Packages 2023-10-09T03:43:16.515 INFO:teuthology.orchestra.run.smithi163.stdout:Upgrade 2 Packages 2023-10-09T03:43:16.516 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:43:16.516 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 63 M 2023-10-09T03:43:16.516 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:43:16.981 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout:==================================================================================================== 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout:==================================================================================================== 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout:Upgrading: 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T03:43:16.984 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T03:43:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout:==================================================================================================== 2023-10-09T03:43:16.986 INFO:teuthology.orchestra.run.smithi181.stdout:Install 21 Packages 2023-10-09T03:43:16.987 INFO:teuthology.orchestra.run.smithi181.stdout:Upgrade 2 Packages 2023-10-09T03:43:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:43:16.987 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 63 M 2023-10-09T03:43:16.987 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:43:17.771 INFO:teuthology.orchestra.run.smithi163.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 4.7 MB/s | 5.2 MB 00:01 2023-10-09T03:43:17.888 INFO:teuthology.orchestra.run.smithi163.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 209 kB/s | 24 kB 00:00 2023-10-09T03:43:18.263 INFO:teuthology.orchestra.run.smithi163.stdout:(3/23): ceph-common-17.2.6-1303.gb2903cb0.el8.x 15 MB/s | 23 MB 00:01 2023-10-09T03:43:18.297 INFO:teuthology.orchestra.run.smithi163.stdout:(4/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 1.9 MB/s | 797 kB 00:00 2023-10-09T03:43:18.356 INFO:teuthology.orchestra.run.smithi163.stdout:(5/23): libradosstriper1-17.2.6-1303.gb2903cb0. 5.4 MB/s | 509 kB 00:00 2023-10-09T03:43:18.422 INFO:teuthology.orchestra.run.smithi163.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 673 kB/s | 45 kB 00:00 2023-10-09T03:43:18.497 INFO:teuthology.orchestra.run.smithi163.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 1.5 MB/s | 115 kB 00:00 2023-10-09T03:43:18.574 INFO:teuthology.orchestra.run.smithi163.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 2.5 MB/s | 190 kB 00:00 2023-10-09T03:43:18.673 INFO:teuthology.orchestra.run.smithi163.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 3.6 MB/s | 370 kB 00:00 2023-10-09T03:43:18.757 INFO:teuthology.orchestra.run.smithi163.stdout:(10/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 4.2 MB/s | 358 kB 00:00 2023-10-09T03:43:18.840 INFO:teuthology.orchestra.run.smithi163.stdout:(11/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.2 MB/s | 106 kB 00:00 2023-10-09T03:43:19.241 INFO:teuthology.orchestra.run.smithi163.stdout:(12/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_ 5.9 MB/s | 5.6 MB 00:00 2023-10-09T03:43:19.416 INFO:teuthology.orchestra.run.smithi163.stdout:(13/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 6.1 MB/s | 17 MB 00:02 2023-10-09T03:43:19.441 INFO:teuthology.orchestra.run.smithi163.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 510 kB/s | 306 kB 00:00 2023-10-09T03:43:19.784 INFO:teuthology.orchestra.run.smithi163.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 108 kB/s | 59 kB 00:00 2023-10-09T03:43:19.815 INFO:teuthology.orchestra.run.smithi181.stdout:(1/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8. 6.8 MB/s | 17 MB 00:02 2023-10-09T03:43:19.856 INFO:teuthology.orchestra.run.smithi181.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 591 kB/s | 24 kB 00:00 2023-10-09T03:43:19.883 INFO:teuthology.orchestra.run.smithi163.stdout:(16/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.5 MB/s | 259 kB 00:00 2023-10-09T03:43:19.925 INFO:teuthology.orchestra.run.smithi163.stdout:(17/23): libunwind-1.3.1-3.el8.x86_64.rpm 147 kB/s | 75 kB 00:00 2023-10-09T03:43:19.951 INFO:teuthology.orchestra.run.smithi163.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 657 kB/s | 44 kB 00:00 2023-10-09T03:43:19.957 INFO:teuthology.orchestra.run.smithi181.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 7.8 MB/s | 797 kB 00:00 2023-10-09T03:43:20.018 INFO:teuthology.orchestra.run.smithi163.stdout:(19/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 767 kB/s | 47 kB 00:00 2023-10-09T03:43:20.024 INFO:teuthology.orchestra.run.smithi181.stdout:(4/23): libradosstriper1-17.2.6-1303.gb2903cb0. 7.5 MB/s | 509 kB 00:00 2023-10-09T03:43:20.043 INFO:teuthology.orchestra.run.smithi163.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 2.9 MB/s | 354 kB 00:00 2023-10-09T03:43:20.085 INFO:teuthology.orchestra.run.smithi163.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 2.7 MB/s | 1.7 MB 00:00 2023-10-09T03:43:20.469 INFO:teuthology.orchestra.run.smithi163.stdout:(22/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 8.4 MB/s | 3.6 MB 00:00 2023-10-09T03:43:20.636 INFO:teuthology.orchestra.run.smithi163.stdout:(23/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 5.6 MB/s | 3.5 MB 00:00 2023-10-09T03:43:20.638 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:43:20.639 INFO:teuthology.orchestra.run.smithi163.stdout:Total 15 MB/s | 63 MB 00:04 2023-10-09T03:43:20.641 INFO:teuthology.orchestra.run.smithi181.stdout:(5/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 9.0 MB/s | 5.6 MB 00:00 2023-10-09T03:43:20.686 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:43:20.691 INFO:teuthology.orchestra.run.smithi181.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 901 kB/s | 45 kB 00:00 2023-10-09T03:43:20.731 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:43:20.731 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:43:20.741 INFO:teuthology.orchestra.run.smithi181.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 2.3 MB/s | 115 kB 00:00 2023-10-09T03:43:20.791 INFO:teuthology.orchestra.run.smithi181.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 3.7 MB/s | 190 kB 00:00 2023-10-09T03:43:20.858 INFO:teuthology.orchestra.run.smithi181.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 5.4 MB/s | 370 kB 00:00 2023-10-09T03:43:20.934 INFO:teuthology.orchestra.run.smithi181.stdout:(10/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x8 1.4 MB/s | 5.2 MB 00:03 2023-10-09T03:43:20.959 INFO:teuthology.orchestra.run.smithi181.stdout:(11/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 3.5 MB/s | 358 kB 00:00 2023-10-09T03:43:20.993 INFO:teuthology.orchestra.run.smithi181.stdout:(12/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.8 MB/s | 106 kB 00:00 2023-10-09T03:43:21.079 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:43:21.083 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:43:21.351 INFO:teuthology.orchestra.run.smithi181.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 164 kB/s | 59 kB 00:00 2023-10-09T03:43:21.485 INFO:teuthology.orchestra.run.smithi181.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 582 kB/s | 306 kB 00:00 2023-10-09T03:43:21.519 INFO:teuthology.orchestra.run.smithi181.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 449 kB/s | 75 kB 00:00 2023-10-09T03:43:21.695 INFO:teuthology.orchestra.run.smithi181.stdout:(16/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.4 MB/s | 259 kB 00:00 2023-10-09T03:43:21.761 INFO:teuthology.orchestra.run.smithi181.stdout:(17/23): python3-prettytable-0.7.2-14.el8.noarc 659 kB/s | 44 kB 00:00 2023-10-09T03:43:21.812 INFO:teuthology.orchestra.run.smithi181.stdout:(18/23): thrift-0.13.0-2.el8.x86_64.rpm 5.3 MB/s | 1.7 MB 00:00 2023-10-09T03:43:21.878 INFO:teuthology.orchestra.run.smithi181.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.0 MB/s | 354 kB 00:00 2023-10-09T03:43:21.904 INFO:teuthology.orchestra.run.smithi181.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 510 kB/s | 47 kB 00:00 2023-10-09T03:43:22.104 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:43:22.246 INFO:teuthology.orchestra.run.smithi181.stdout:(21/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 10 MB/s | 3.6 MB 00:00 2023-10-09T03:43:22.276 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:43:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:43:22.834 INFO:teuthology.orchestra.run.smithi163.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:43:22.934 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:43:23.107 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:43:23.112 INFO:teuthology.orchestra.run.smithi181.stdout:(22/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 2.8 MB/s | 3.5 MB 00:01 2023-10-09T03:43:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T03:43:23.509 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:43:23.904 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:43:24.084 INFO:teuthology.orchestra.run.smithi163.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:43:24.149 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:43:24.324 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-10-09T03:43:24.479 INFO:teuthology.orchestra.run.smithi181.stdout:(23/23): ceph-common-17.2.6-1303.gb2903cb0.el8. 3.3 MB/s | 23 MB 00:07 2023-10-09T03:43:24.482 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:43:24.482 INFO:teuthology.orchestra.run.smithi181.stdout:Total 8.4 MB/s | 63 MB 00:07 2023-10-09T03:43:24.492 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:43:24.527 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:43:24.573 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:43:24.573 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:43:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:43:24.917 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:43:24.920 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:43:25.484 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T03:43:25.651 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:43:25.704 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:43:25.853 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T03:43:25.957 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:43:26.012 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:43:26.150 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:43:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:43:26.472 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T03:43:26.554 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:43:26.617 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:43:26.692 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:43:26.725 INFO:teuthology.orchestra.run.smithi181.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:43:26.827 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T03:43:26.946 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:43:27.016 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T03:43:27.146 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:43:27.237 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T03:43:27.366 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T03:43:27.393 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T03:43:27.550 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:43:27.948 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:43:28.165 INFO:teuthology.orchestra.run.smithi181.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:43:28.218 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:43:28.412 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-10-09T03:43:28.582 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:43:28.785 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:43:29.448 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T03:43:29.615 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:43:29.681 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:43:29.809 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T03:43:29.984 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:43:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:43:30.403 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T03:43:30.590 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:43:30.661 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:43:30.824 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T03:43:31.039 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T03:43:31.273 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T03:43:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:43:31.432 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T03:43:31.559 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:43:32.134 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:43:32.379 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:43:32.448 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:43:32.600 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:43:35.321 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:43:35.960 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:43:36.110 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:43:36.351 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:43:36.433 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:43:36.564 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:43:51.208 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /sys 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /proc 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /dev 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /run 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /mnt 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /var/tmp 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /home 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /tmp 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout:skipping the directory /dev 2023-10-09T03:43:51.209 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:43:52.883 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:43:53.967 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T03:43:53.968 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T03:43:53.969 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T03:43:54.892 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:43:54.892 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /sys 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /proc 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /dev 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /run 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /mnt 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /var/tmp 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /home 2023-10-09T03:43:54.893 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /tmp 2023-10-09T03:43:54.894 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /dev 2023-10-09T03:43:54.894 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:43:55.117 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:43:55.117 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:00:50 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:43:56.581 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:43:57.358 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:43:57.358 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:43:57.359 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:43:57.361 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:43:57.469 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:43:57.640 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T03:43:57.641 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T03:43:57.642 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout:Upgraded: 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.654 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T03:43:57.655 INFO:teuthology.orchestra.run.smithi163.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:43:57.656 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:43:57.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-test 2023-10-09T03:43:58.180 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:43:58.891 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:43:58.891 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:00:53 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:43:58.998 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:00:53 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:00.366 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout:============================================================================================ 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout:============================================================================================ 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T03:44:00.367 INFO:teuthology.orchestra.run.smithi163.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout:============================================================================================ 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout:Install 6 Packages 2023-10-09T03:44:00.368 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:00.369 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 53 M 2023-10-09T03:44:00.369 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 233 M 2023-10-09T03:44:00.369 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:44:00.406 INFO:teuthology.orchestra.run.smithi163.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.9 MB/s | 70 kB 00:00 2023-10-09T03:44:00.481 INFO:teuthology.orchestra.run.smithi163.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.8 MB/s | 323 kB 00:00 2023-10-09T03:44:00.523 INFO:teuthology.orchestra.run.smithi163.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-10-09T03:44:00.565 INFO:teuthology.orchestra.run.smithi163.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-10-09T03:44:00.607 INFO:teuthology.orchestra.run.smithi163.stdout:(5/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 736 kB/s | 175 kB 00:00 2023-10-09T03:44:01.299 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:44:01.488 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:01.488 INFO:teuthology.orchestra.run.smithi181.stdout:Upgraded: 2023-10-09T03:44:01.488 INFO:teuthology.orchestra.run.smithi181.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.488 INFO:teuthology.orchestra.run.smithi181.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.489 INFO:teuthology.orchestra.run.smithi181.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:01.490 INFO:teuthology.orchestra.run.smithi181.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T03:44:01.491 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:01.491 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:44:01.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-test 2023-10-09T03:44:01.996 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:02.801 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:00:57 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:02.949 INFO:teuthology.orchestra.run.smithi163.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 20 MB/s | 52 MB 00:02 2023-10-09T03:44:02.951 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:02.951 INFO:teuthology.orchestra.run.smithi163.stdout:Total 20 MB/s | 53 MB 00:02 2023-10-09T03:44:02.989 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:44:03.010 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:44:03.010 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:44:03.284 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:44:03.287 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:44:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:44:03.911 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:44:03.998 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:44:04.155 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:44:04.156 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================================ 2023-10-09T03:44:04.156 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:44:04.156 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================================ 2023-10-09T03:44:04.156 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:44:04.156 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout:============================================================================================ 2023-10-09T03:44:04.157 INFO:teuthology.orchestra.run.smithi181.stdout:Install 6 Packages 2023-10-09T03:44:04.158 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:04.158 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 53 M 2023-10-09T03:44:04.158 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 233 M 2023-10-09T03:44:04.158 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:44:04.193 INFO:teuthology.orchestra.run.smithi181.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.0 MB/s | 70 kB 00:00 2023-10-09T03:44:04.199 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T03:44:04.252 INFO:teuthology.orchestra.run.smithi181.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 5.4 MB/s | 323 kB 00:00 2023-10-09T03:44:04.294 INFO:teuthology.orchestra.run.smithi181.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.5 MB/s | 187 kB 00:00 2023-10-09T03:44:04.336 INFO:teuthology.orchestra.run.smithi181.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-10-09T03:44:04.353 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T03:44:04.461 INFO:teuthology.orchestra.run.smithi181.stdout:(5/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 578 kB/s | 175 kB 00:00 2023-10-09T03:44:04.526 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T03:44:04.661 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:44:06.803 INFO:teuthology.orchestra.run.smithi181.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 20 MB/s | 52 MB 00:02 2023-10-09T03:44:06.805 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:06.805 INFO:teuthology.orchestra.run.smithi181.stdout:Total 20 MB/s | 53 MB 00:02 2023-10-09T03:44:06.841 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:44:06.862 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:44:06.863 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:44:07.138 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:44:07.141 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:44:07.587 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:44:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:44:07.809 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:44:08.041 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T03:44:08.193 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T03:44:08.358 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T03:44:08.526 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:44:09.604 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:44:09.841 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:44:10.073 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:44:10.073 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T03:44:10.073 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T03:44:10.073 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T03:44:10.073 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T03:44:10.073 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T03:44:11.175 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T03:44:11.175 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:06 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:13.446 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:44:13.525 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:44:13.672 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: jq-1.6-3.el8.x86_64 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T03:44:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:13.710 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:44:13.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph 2023-10-09T03:44:13.893 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:44:13.893 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T03:44:13.893 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T03:44:13.893 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T03:44:13.893 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T03:44:13.893 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T03:44:14.189 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:14.966 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T03:44:14.967 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:09 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:14.992 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:09 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:16.382 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:44:16.385 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================= 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================= 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T03:44:16.386 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T03:44:16.387 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T03:44:16.388 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:44:16.389 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================= 2023-10-09T03:44:16.390 INFO:teuthology.orchestra.run.smithi163.stdout:Install 32 Packages 2023-10-09T03:44:16.390 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:16.390 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 28 M 2023-10-09T03:44:16.390 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 110 M 2023-10-09T03:44:16.390 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:44:16.799 INFO:teuthology.orchestra.run.smithi163.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 34 kB/s | 6.4 kB 00:00 2023-10-09T03:44:16.916 INFO:teuthology.orchestra.run.smithi163.stdout:(2/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 5.2 MB/s | 1.6 MB 00:00 2023-10-09T03:44:17.252 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:44:17.291 INFO:teuthology.orchestra.run.smithi163.stdout:(3/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 3.4 MB/s | 2.3 MB 00:00 2023-10-09T03:44:17.375 INFO:teuthology.orchestra.run.smithi163.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.8 MB/s | 238 kB 00:00 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout: jq-1.6-3.el8.x86_64 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T03:44:17.434 INFO:teuthology.orchestra.run.smithi181.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T03:44:17.435 INFO:teuthology.orchestra.run.smithi181.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T03:44:17.435 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:17.435 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:44:17.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph 2023-10-09T03:44:17.817 INFO:teuthology.orchestra.run.smithi163.stdout:(5/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 19 MB/s | 17 MB 00:00 2023-10-09T03:44:17.834 INFO:teuthology.orchestra.run.smithi163.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 95 kB/s | 44 kB 00:00 2023-10-09T03:44:17.898 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:18.017 INFO:teuthology.orchestra.run.smithi163.stdout:(7/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 3.2 MB/s | 3.9 MB 00:01 2023-10-09T03:44:18.118 INFO:teuthology.orchestra.run.smithi163.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 608 kB/s | 173 kB 00:00 2023-10-09T03:44:18.218 INFO:teuthology.orchestra.run.smithi163.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 116 kB/s | 11 kB 00:00 2023-10-09T03:44:18.914 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:13 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 176 kB/s | 18 kB 00:00 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(11/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 304 kB/s | 185 kB 00:00 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 389 kB/s | 49 kB 00:00 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 469 kB/s | 59 kB 00:00 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.3 MB/s | 283 kB 00:00 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 124 kB/s | 16 kB 00:00 2023-10-09T03:44:18.915 INFO:teuthology.orchestra.run.smithi163.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 561 kB/s | 384 kB 00:00 2023-10-09T03:44:18.917 INFO:teuthology.orchestra.run.smithi163.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 186 kB/s | 19 kB 00:00 2023-10-09T03:44:18.917 INFO:teuthology.orchestra.run.smithi163.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 219 kB/s | 24 kB 00:00 2023-10-09T03:44:18.917 INFO:teuthology.orchestra.run.smithi163.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 269 kB/s | 29 kB 00:00 2023-10-09T03:44:18.918 INFO:teuthology.orchestra.run.smithi163.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 274 kB/s | 27 kB 00:00 2023-10-09T03:44:18.972 INFO:teuthology.orchestra.run.smithi163.stdout:(21/32): python3-webtest-2.0.33-1.el8.noarch.rp 731 kB/s | 85 kB 00:00 2023-10-09T03:44:19.087 INFO:teuthology.orchestra.run.smithi163.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.2 MB/s | 255 kB 00:00 2023-10-09T03:44:19.087 INFO:teuthology.orchestra.run.smithi163.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.2 MB/s | 251 kB 00:00 2023-10-09T03:44:19.087 INFO:teuthology.orchestra.run.smithi163.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 228 kB/s | 23 kB 00:00 2023-10-09T03:44:19.097 INFO:teuthology.orchestra.run.smithi163.stdout:(25/32): python3-markupsafe-0.23-19.el8.x86_64. 516 kB/s | 39 kB 00:00 2023-10-09T03:44:19.166 INFO:teuthology.orchestra.run.smithi163.stdout:(26/32): python3-pytz-2017.2-9.el8.noarch.rpm 404 kB/s | 54 kB 00:00 2023-10-09T03:44:19.166 INFO:teuthology.orchestra.run.smithi163.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.8 MB/s | 157 kB 00:00 2023-10-09T03:44:19.182 INFO:teuthology.orchestra.run.smithi163.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-10-09T03:44:19.207 INFO:teuthology.orchestra.run.smithi163.stdout:(29/32): python3-pycparser-2.14-14.el8.noarch.r 2.1 MB/s | 109 kB 00:00 2023-10-09T03:44:19.240 INFO:teuthology.orchestra.run.smithi163.stdout:(30/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.3 MB/s | 457 kB 00:00 2023-10-09T03:44:19.265 INFO:teuthology.orchestra.run.smithi163.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.0 MB/s | 238 kB 00:00 2023-10-09T03:44:19.291 INFO:teuthology.orchestra.run.smithi163.stdout:(32/32): python3-cryptography-3.2.1-5.el8.x86_6 5.0 MB/s | 559 kB 00:00 2023-10-09T03:44:19.295 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:19.295 INFO:teuthology.orchestra.run.smithi163.stdout:Total 9.7 MB/s | 28 MB 00:02 2023-10-09T03:44:19.384 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:44:19.421 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:44:19.421 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:44:19.625 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:44:19.628 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:44:20.133 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:44:20.136 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================= 2023-10-09T03:44:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:44:20.136 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================= 2023-10-09T03:44:20.136 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T03:44:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T03:44:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T03:44:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T03:44:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:20.140 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:44:20.140 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================= 2023-10-09T03:44:20.140 INFO:teuthology.orchestra.run.smithi181.stdout:Install 32 Packages 2023-10-09T03:44:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:20.140 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 28 M 2023-10-09T03:44:20.141 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 110 M 2023-10-09T03:44:20.141 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:44:20.537 INFO:teuthology.orchestra.run.smithi181.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 37 kB/s | 6.4 kB 00:00 2023-10-09T03:44:20.654 INFO:teuthology.orchestra.run.smithi181.stdout:(2/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 5.4 MB/s | 1.6 MB 00:00 2023-10-09T03:44:20.701 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:44:20.905 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T03:44:21.096 INFO:teuthology.orchestra.run.smithi181.stdout:(3/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 3.2 MB/s | 2.3 MB 00:00 2023-10-09T03:44:21.136 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T03:44:21.188 INFO:teuthology.orchestra.run.smithi181.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.5 MB/s | 238 kB 00:00 2023-10-09T03:44:21.580 INFO:teuthology.orchestra.run.smithi181.stdout:(5/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 18 MB/s | 17 MB 00:00 2023-10-09T03:44:21.602 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-10-09T03:44:21.605 INFO:teuthology.orchestra.run.smithi181.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 104 kB/s | 44 kB 00:00 2023-10-09T03:44:21.705 INFO:teuthology.orchestra.run.smithi181.stdout:(7/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 3.4 MB/s | 3.9 MB 00:01 2023-10-09T03:44:21.905 INFO:teuthology.orchestra.run.smithi181.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 575 kB/s | 173 kB 00:00 2023-10-09T03:44:21.956 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-10-09T03:44:22.014 INFO:teuthology.orchestra.run.smithi181.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 103 kB/s | 11 kB 00:00 2023-10-09T03:44:22.123 INFO:teuthology.orchestra.run.smithi181.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 162 kB/s | 18 kB 00:00 2023-10-09T03:44:22.165 INFO:teuthology.orchestra.run.smithi181.stdout:(11/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 317 kB/s | 185 kB 00:00 2023-10-09T03:44:22.202 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-10-09T03:44:22.240 INFO:teuthology.orchestra.run.smithi181.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 416 kB/s | 49 kB 00:00 2023-10-09T03:44:22.284 INFO:teuthology.orchestra.run.smithi181.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 503 kB/s | 59 kB 00:00 2023-10-09T03:44:22.365 INFO:teuthology.orchestra.run.smithi181.stdout:(14/32): python3-cherrypy-18.4.0-1.el8.noarch.r 587 kB/s | 384 kB 00:00 2023-10-09T03:44:22.366 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-10-09T03:44:22.391 INFO:teuthology.orchestra.run.smithi181.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 143 kB/s | 16 kB 00:00 2023-10-09T03:44:22.433 INFO:teuthology.orchestra.run.smithi181.stdout:(16/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.4 MB/s | 283 kB 00:00 2023-10-09T03:44:22.475 INFO:teuthology.orchestra.run.smithi181.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 172 kB/s | 19 kB 00:00 2023-10-09T03:44:22.508 INFO:teuthology.orchestra.run.smithi181.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 204 kB/s | 24 kB 00:00 2023-10-09T03:44:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-10-09T03:44:22.542 INFO:teuthology.orchestra.run.smithi181.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 269 kB/s | 29 kB 00:00 2023-10-09T03:44:22.584 INFO:teuthology.orchestra.run.smithi181.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 253 kB/s | 27 kB 00:00 2023-10-09T03:44:22.651 INFO:teuthology.orchestra.run.smithi181.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.8 MB/s | 255 kB 00:00 2023-10-09T03:44:22.668 INFO:teuthology.orchestra.run.smithi181.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.0 MB/s | 251 kB 00:00 2023-10-09T03:44:22.709 INFO:teuthology.orchestra.run.smithi181.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 680 kB/s | 85 kB 00:00 2023-10-09T03:44:22.735 INFO:teuthology.orchestra.run.smithi181.stdout:(24/32): python3-pytz-2017.2-9.el8.noarch.rpm 811 kB/s | 54 kB 00:00 2023-10-09T03:44:22.768 INFO:teuthology.orchestra.run.smithi181.stdout:(25/32): python3-zc-lockfile-2.0-2.el8.noarch.r 195 kB/s | 23 kB 00:00 2023-10-09T03:44:22.794 INFO:teuthology.orchestra.run.smithi181.stdout:(26/32): python3-markupsafe-0.23-19.el8.x86_64. 463 kB/s | 39 kB 00:00 2023-10-09T03:44:22.827 INFO:teuthology.orchestra.run.smithi181.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.8 MB/s | 157 kB 00:00 2023-10-09T03:44:22.833 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-10-09T03:44:22.844 INFO:teuthology.orchestra.run.smithi181.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.3 MB/s | 103 kB 00:00 2023-10-09T03:44:22.870 INFO:teuthology.orchestra.run.smithi181.stdout:(29/32): python3-pycparser-2.14-14.el8.noarch.r 2.5 MB/s | 109 kB 00:00 2023-10-09T03:44:22.903 INFO:teuthology.orchestra.run.smithi181.stdout:(30/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.3 MB/s | 457 kB 00:00 2023-10-09T03:44:22.920 INFO:teuthology.orchestra.run.smithi181.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.6 MB/s | 238 kB 00:00 2023-10-09T03:44:22.945 INFO:teuthology.orchestra.run.smithi181.stdout:(32/32): python3-cryptography-3.2.1-5.el8.x86_6 5.4 MB/s | 559 kB 00:00 2023-10-09T03:44:22.949 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:22.950 INFO:teuthology.orchestra.run.smithi181.stdout:Total 10 MB/s | 28 MB 00:02 2023-10-09T03:44:23.028 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-10-09T03:44:23.041 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:44:23.079 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:44:23.079 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:44:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-10-09T03:44:23.283 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:44:23.287 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:44:23.475 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-10-09T03:44:23.640 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-10-09T03:44:23.840 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T03:44:24.014 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T03:44:24.187 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T03:44:24.333 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T03:44:24.372 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:44:24.510 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T03:44:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T03:44:24.654 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T03:44:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T03:44:24.886 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T03:44:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T03:44:25.091 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-10-09T03:44:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T03:44:25.436 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-10-09T03:44:25.464 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T03:44:25.701 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-10-09T03:44:25.702 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T03:44:25.889 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-10-09T03:44:25.932 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T03:44:26.057 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-10-09T03:44:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T03:44:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-10-09T03:44:26.504 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T03:44:26.560 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-10-09T03:44:26.783 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-10-09T03:44:26.816 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T03:44:27.006 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-10-09T03:44:27.180 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-10-09T03:44:27.388 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T03:44:27.411 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:44:27.596 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T03:44:27.785 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T03:44:27.948 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T03:44:28.117 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T03:44:28.253 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T03:44:28.509 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T03:44:28.709 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T03:44:28.887 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T03:44:28.892 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:44:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:44:29.158 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T03:44:29.429 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T03:44:29.494 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:44:29.624 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T03:44:29.649 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:44:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:44:30.007 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:44:30.025 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:44:30.163 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:44:30.481 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T03:44:30.553 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T03:44:30.554 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T03:44:30.555 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-10-09T03:44:30.556 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-10-09T03:44:30.557 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-10-09T03:44:30.557 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-10-09T03:44:30.557 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-10-09T03:44:30.557 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-10-09T03:44:30.778 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T03:44:31.131 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T03:44:31.310 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:44:32.123 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-10-09T03:44:32.123 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:27 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:32.784 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:44:32.976 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:44:33.377 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:44:33.514 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:44:33.730 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:44:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:44:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:44:34.045 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:44:34.406 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T03:44:34.407 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T03:44:34.408 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T03:44:34.409 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T03:44:34.409 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T03:44:34.409 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T03:44:34.409 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-10-09T03:44:34.409 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-10-09T03:44:34.409 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-10-09T03:44:34.410 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-10-09T03:44:34.410 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-10-09T03:44:34.410 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-10-09T03:44:34.410 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-10-09T03:44:34.641 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:44:34.828 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T03:44:34.829 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T03:44:34.830 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T03:44:34.831 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:34.832 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:44:34.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-base 2023-10-09T03:44:35.343 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:35.948 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-10-09T03:44:35.948 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:30 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:36.230 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:37.497 INFO:teuthology.orchestra.run.smithi163.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:44:37.606 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:44:37.607 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:44:37.608 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:44:38.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install cephadm 2023-10-09T03:44:38.394 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:38.527 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:44:38.715 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:38.802 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:44:38.802 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:38.802 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:38.803 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:38.803 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:44:38.803 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:38.803 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:38.803 INFO:teuthology.orchestra.run.smithi181.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T03:44:38.804 INFO:teuthology.orchestra.run.smithi181.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T03:44:38.804 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T03:44:38.804 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T03:44:38.804 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T03:44:38.804 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T03:44:38.805 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T03:44:38.805 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T03:44:38.805 INFO:teuthology.orchestra.run.smithi181.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T03:44:38.805 INFO:teuthology.orchestra.run.smithi181.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T03:44:38.805 INFO:teuthology.orchestra.run.smithi181.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T03:44:38.805 INFO:teuthology.orchestra.run.smithi181.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T03:44:38.806 INFO:teuthology.orchestra.run.smithi181.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:38.807 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:44:38.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-base 2023-10-09T03:44:39.217 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:39.422 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:34 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:40.053 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:35 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:40.698 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:40.699 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 80 k 2023-10-09T03:44:40.700 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 364 k 2023-10-09T03:44:40.700 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:44:40.948 INFO:teuthology.orchestra.run.smithi163.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 321 kB/s | 80 kB 00:00 2023-10-09T03:44:40.948 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:40.948 INFO:teuthology.orchestra.run.smithi163.stdout:Total 319 kB/s | 80 kB 00:00 2023-10-09T03:44:40.949 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:44:40.956 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:44:40.957 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:44:40.958 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:44:40.962 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:44:41.003 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:44:41.306 INFO:teuthology.orchestra.run.smithi181.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:44:41.415 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:44:41.416 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:44:41.416 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:44:41.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install cephadm 2023-10-09T03:44:42.217 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:42.665 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:42.814 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:42.985 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:43.059 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:38 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:44.064 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:44.064 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:39 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:44.414 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:44:44.492 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:44:44.492 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:44:44.492 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:44:44.492 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:44:44.493 INFO:teuthology.orchestra.run.smithi181.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T03:44:44.493 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:44.493 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:44:44.493 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:44:44.493 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:44:44.493 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:44.494 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 80 k 2023-10-09T03:44:44.494 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 364 k 2023-10-09T03:44:44.494 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:44:44.668 INFO:teuthology.orchestra.run.smithi181.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 318 kB/s | 80 kB 00:00 2023-10-09T03:44:44.669 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:44.669 INFO:teuthology.orchestra.run.smithi181.stdout:Total 316 kB/s | 80 kB 00:00 2023-10-09T03:44:44.669 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:44:44.677 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:44:44.677 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:44:44.679 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:44:44.682 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:44:44.722 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:44:46.379 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:46.559 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:46.729 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:46.779 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:44:46.960 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:46.960 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:44:46.960 INFO:teuthology.orchestra.run.smithi163.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:44:46.960 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:46.960 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:44:47.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T03:44:47.459 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:48.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:44:48.236 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:48.314 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:49.673 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repo Size 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:49.674 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:44:49.675 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:44:49.675 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:44:49.675 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:49.675 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 161 k 2023-10-09T03:44:49.675 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 490 k 2023-10-09T03:44:49.675 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:44:49.853 INFO:teuthology.orchestra.run.smithi163.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 904 kB/s | 161 kB 00:00 2023-10-09T03:44:49.854 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:49.854 INFO:teuthology.orchestra.run.smithi163.stdout:Total 898 kB/s | 161 kB 00:00 2023-10-09T03:44:49.854 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:44:49.864 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:44:49.864 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:44:49.915 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:44:49.919 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:44:50.027 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:44:50.203 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:44:50.570 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:44:50.895 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:44:51.075 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:51.075 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:44:51.075 INFO:teuthology.orchestra.run.smithi181.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:44:51.075 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:51.075 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:44:51.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T03:44:51.559 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:51.604 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:44:51.605 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:46 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:52.394 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:47 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:53.827 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:44:53.828 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:44:53.829 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:44:53.829 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:53.829 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 161 k 2023-10-09T03:44:53.829 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 490 k 2023-10-09T03:44:53.829 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:44:54.014 INFO:teuthology.orchestra.run.smithi181.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 872 kB/s | 161 kB 00:00 2023-10-09T03:44:54.015 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:44:54.015 INFO:teuthology.orchestra.run.smithi181.stdout:Total 866 kB/s | 161 kB 00:00 2023-10-09T03:44:54.015 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:44:54.026 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:44:54.026 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:44:54.077 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:44:54.081 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:44:54.104 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:44:54.188 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:44:54.283 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:54.283 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:44:54.284 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:54.284 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:44:54.284 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:44:54.352 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:44:54.418 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-mgr 2023-10-09T03:44:54.713 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:44:54.765 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:55.649 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:50 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:55.780 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:44:55.780 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:50 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:56.976 INFO:teuthology.orchestra.run.smithi163.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:44:57.084 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:44:57.085 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:44:57.085 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:44:57.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T03:44:57.555 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:58.727 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:53 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:44:58.758 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:44:58.940 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:58.940 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:44:58.940 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:44:58.940 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:44:58.941 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:44:59.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr 2023-10-09T03:44:59.412 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:00.085 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:45:00.086 INFO:teuthology.orchestra.run.smithi163.stdout:=================================================================================================== 2023-10-09T03:45:00.086 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:45:00.086 INFO:teuthology.orchestra.run.smithi163.stdout:=================================================================================================== 2023-10-09T03:45:00.086 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:45:00.086 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:45:00.087 INFO:teuthology.orchestra.run.smithi163.stdout:=================================================================================================== 2023-10-09T03:45:00.088 INFO:teuthology.orchestra.run.smithi163.stdout:Install 6 Packages 2023-10-09T03:45:00.088 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:00.088 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 2.0 M 2023-10-09T03:45:00.088 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 8.0 M 2023-10-09T03:45:00.088 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:45:00.263 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:55 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:00.923 INFO:teuthology.orchestra.run.smithi163.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1303.gb29 46 kB/s | 23 kB 00:00 2023-10-09T03:45:01.540 INFO:teuthology.orchestra.run.smithi163.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1303.gb290 24 kB/s | 14 kB 00:00 2023-10-09T03:45:01.540 INFO:teuthology.orchestra.run.smithi163.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 2.7 MB/s | 1.7 MB 00:00 2023-10-09T03:45:01.540 INFO:teuthology.orchestra.run.smithi163.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 573 kB/s | 43 kB 00:00 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 137 kB/s | 34 kB 00:00 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 979 kB/s | 196 kB 00:00 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:Total 1.8 MB/s | 2.0 MB 00:01 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:45:01.541 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:45:01.542 INFO:teuthology.orchestra.run.smithi181.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:45:01.623 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:45:01.650 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:45:01.652 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:45:01.652 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:45:01.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T03:45:01.777 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T03:45:01.957 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T03:45:02.109 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:02.112 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T03:45:02.249 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T03:45:02.594 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T03:45:02.800 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:45:03.006 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:45:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T03:45:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T03:45:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T03:45:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T03:45:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T03:45:04.092 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T03:45:04.092 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:01:59 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:04.362 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:45:04.363 INFO:teuthology.orchestra.run.smithi181.stdout:=================================================================================================== 2023-10-09T03:45:04.363 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:45:04.363 INFO:teuthology.orchestra.run.smithi181.stdout:=================================================================================================== 2023-10-09T03:45:04.363 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:45:04.363 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout:=================================================================================================== 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout:Install 6 Packages 2023-10-09T03:45:04.364 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:04.365 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 2.0 M 2023-10-09T03:45:04.365 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 8.0 M 2023-10-09T03:45:04.365 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:45:04.571 INFO:teuthology.orchestra.run.smithi181.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1303.gb29 161 kB/s | 23 kB 00:00 2023-10-09T03:45:04.662 INFO:teuthology.orchestra.run.smithi181.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1303.gb290 62 kB/s | 14 kB 00:00 2023-10-09T03:45:04.813 INFO:teuthology.orchestra.run.smithi181.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 139 kB/s | 34 kB 00:00 2023-10-09T03:45:04.863 INFO:teuthology.orchestra.run.smithi181.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 856 kB/s | 43 kB 00:00 2023-10-09T03:45:04.905 INFO:teuthology.orchestra.run.smithi181.stdout:(5/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 3.6 MB/s | 1.7 MB 00:00 2023-10-09T03:45:05.013 INFO:teuthology.orchestra.run.smithi181.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 560 kB/s | 196 kB 00:00 2023-10-09T03:45:05.016 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:05.016 INFO:teuthology.orchestra.run.smithi181.stdout:Total 3.1 MB/s | 2.0 MB 00:00 2023-10-09T03:45:05.027 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:45:05.036 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:45:05.036 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:45:05.092 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:45:05.096 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:45:05.386 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:45:05.593 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T03:45:05.798 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T03:45:05.961 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T03:45:06.090 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T03:45:06.422 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T03:45:06.623 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:45:06.630 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:45:06.814 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:06.815 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:45:06.854 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:45:06.854 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T03:45:06.854 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T03:45:06.854 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T03:45:06.854 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T03:45:06.854 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T03:45:06.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T03:45:07.308 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:07.868 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T03:45:07.868 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:02 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:08.127 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:03 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:09.552 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================================= 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================================= 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T03:45:09.554 INFO:teuthology.orchestra.run.smithi163.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================================= 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout:Install 8 Packages 2023-10-09T03:45:09.555 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:09.556 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 35 M 2023-10-09T03:45:09.556 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 210 M 2023-10-09T03:45:09.556 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:45:09.708 INFO:teuthology.orchestra.run.smithi163.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 25 MB/s | 3.7 MB 00:00 2023-10-09T03:45:09.750 INFO:teuthology.orchestra.run.smithi163.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-10-09T03:45:09.908 INFO:teuthology.orchestra.run.smithi163.stdout:(3/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 31 MB/s | 4.9 MB 00:00 2023-10-09T03:45:10.050 INFO:teuthology.orchestra.run.smithi163.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 28 MB/s | 14 MB 00:00 2023-10-09T03:45:10.101 INFO:teuthology.orchestra.run.smithi163.stdout:(5/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 13 MB/s | 644 kB 00:00 2023-10-09T03:45:10.143 INFO:teuthology.orchestra.run.smithi163.stdout:(6/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 4.0 MB/s | 171 kB 00:00 2023-10-09T03:45:10.218 INFO:teuthology.orchestra.run.smithi163.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.7 MB 00:00 2023-10-09T03:45:10.276 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:45:10.426 INFO:teuthology.orchestra.run.smithi163.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-130 8.5 MB/s | 7.4 MB 00:00 2023-10-09T03:45:10.427 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:10.427 INFO:teuthology.orchestra.run.smithi163.stdout:Total 41 MB/s | 35 MB 00:00 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T03:45:10.457 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:10.458 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:45:10.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T03:45:10.666 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:45:10.685 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:45:10.685 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:45:10.940 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:10.978 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:45:10.982 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:45:11.545 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:45:11.769 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:06 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:11.899 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:45:12.028 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:45:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:45:12.760 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:45:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:45:13.126 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================================= 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================================= 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:45:13.127 INFO:teuthology.orchestra.run.smithi181.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================================= 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout:Install 8 Packages 2023-10-09T03:45:13.128 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:13.129 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 35 M 2023-10-09T03:45:13.129 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 210 M 2023-10-09T03:45:13.129 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:45:13.294 INFO:teuthology.orchestra.run.smithi181.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 23 MB/s | 3.7 MB 00:00 2023-10-09T03:45:13.344 INFO:teuthology.orchestra.run.smithi181.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-10-09T03:45:13.517 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:45:13.536 INFO:teuthology.orchestra.run.smithi181.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 34 MB/s | 14 MB 00:00 2023-10-09T03:45:13.620 INFO:teuthology.orchestra.run.smithi181.stdout:(4/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.9 MB 00:00 2023-10-09T03:45:13.670 INFO:teuthology.orchestra.run.smithi181.stdout:(5/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 13 MB/s | 644 kB 00:00 2023-10-09T03:45:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:45:13.745 INFO:teuthology.orchestra.run.smithi181.stdout:(6/8): openblas-0.3.15-3.el8.x86_64.rpm 23 MB/s | 4.7 MB 00:00 2023-10-09T03:45:13.770 INFO:teuthology.orchestra.run.smithi181.stdout:(7/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.7 MB/s | 171 kB 00:00 2023-10-09T03:45:14.213 INFO:teuthology.orchestra.run.smithi181.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-130 6.8 MB/s | 7.4 MB 00:01 2023-10-09T03:45:14.214 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:14.214 INFO:teuthology.orchestra.run.smithi181.stdout:Total 33 MB/s | 35 MB 00:01 2023-10-09T03:45:14.410 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:45:14.450 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:45:14.469 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:45:14.469 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:45:14.745 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T03:45:14.759 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:45:14.763 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:45:14.965 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:45:15.302 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:45:15.440 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:45:15.580 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:45:15.715 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:45:16.327 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:45:16.490 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:45:16.738 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:45:17.086 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:45:17.281 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:45:17.970 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:45:18.202 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T03:45:18.259 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T03:45:18.356 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:45:18.465 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:45:18.624 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-10-09T03:45:18.625 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-10-09T03:45:19.702 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-10-09T03:45:19.702 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:14 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:45:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T03:45:21.589 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:45:21.848 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:45:21.848 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T03:45:21.848 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-10-09T03:45:21.849 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-10-09T03:45:21.849 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T03:45:21.849 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-10-09T03:45:21.849 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-10-09T03:45:21.849 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-10-09T03:45:22.322 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T03:45:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T03:45:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T03:45:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:22.510 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:45:22.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-mgr-rook 2023-10-09T03:45:22.908 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-10-09T03:45:22.908 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:17 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:23.013 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:23.873 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:18 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:25.260 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:45:25.262 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================== 2023-10-09T03:45:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:45:25.262 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================== 2023-10-09T03:45:25.262 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T03:45:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout:======================================================================================================== 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout:Install 13 Packages 2023-10-09T03:45:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:25.265 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 1.9 M 2023-10-09T03:45:25.265 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 25 M 2023-10-09T03:45:25.265 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:45:25.750 INFO:teuthology.orchestra.run.smithi163.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 129 kB/s | 16 kB 00:00 2023-10-09T03:45:25.767 INFO:teuthology.orchestra.run.smithi163.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 230 kB/s | 33 kB 00:00 2023-10-09T03:45:25.808 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:45:25.859 INFO:teuthology.orchestra.run.smithi163.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 749 kB/s | 81 kB 00:00 2023-10-09T03:45:25.875 INFO:teuthology.orchestra.run.smithi163.stdout:(4/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 187 kB/s | 47 kB 00:00 2023-10-09T03:45:25.926 INFO:teuthology.orchestra.run.smithi163.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 960 kB/s | 64 kB 00:00 2023-10-09T03:45:25.995 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:25.995 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:25.996 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:45:26.060 INFO:teuthology.orchestra.run.smithi163.stdout:(6/13): python3-kubernetes-11.0.0-6.el8.noarch. 4.0 MB/s | 1.2 MB 00:00 2023-10-09T03:45:26.076 INFO:teuthology.orchestra.run.smithi163.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 305 kB/s | 61 kB 00:00 2023-10-09T03:45:26.102 INFO:teuthology.orchestra.run.smithi163.stdout:(8/13): python3-pyasn1-modules-0.3.7-6.el8.noar 628 kB/s | 110 kB 00:00 2023-10-09T03:45:26.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-rook 2023-10-09T03:45:26.252 INFO:teuthology.orchestra.run.smithi163.stdout:(9/13): python3-jsonpointer-1.10-11.el8.noarch. 144 kB/s | 20 kB 00:00 2023-10-09T03:45:26.277 INFO:teuthology.orchestra.run.smithi163.stdout:(10/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 632 kB/s | 126 kB 00:00 2023-10-09T03:45:26.302 INFO:teuthology.orchestra.run.smithi163.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 109 kB/s | 27 kB 00:00 2023-10-09T03:45:26.344 INFO:teuthology.orchestra.run.smithi163.stdout:(12/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 2.3 MB/s | 155 kB 00:00 2023-10-09T03:45:26.361 INFO:teuthology.orchestra.run.smithi163.stdout:(13/13): python3-requests-oauthlib-1.0.0-1.el8. 393 kB/s | 43 kB 00:00 2023-10-09T03:45:26.362 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:26.362 INFO:teuthology.orchestra.run.smithi163.stdout:Total 1.8 MB/s | 1.9 MB 00:01 2023-10-09T03:45:26.428 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:45:26.444 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:45:26.444 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:45:26.490 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:26.516 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:45:26.520 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:45:26.994 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:45:27.154 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T03:45:27.358 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T03:45:27.395 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:22 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:27.582 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T03:45:27.851 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-10-09T03:45:28.028 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-10-09T03:45:28.179 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-10-09T03:45:28.341 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-10-09T03:45:28.529 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T03:45:28.720 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T03:45:28.761 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================== 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================== 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:45:28.763 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T03:45:28.764 INFO:teuthology.orchestra.run.smithi181.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T03:45:28.765 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:28.765 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:45:28.765 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================================== 2023-10-09T03:45:28.765 INFO:teuthology.orchestra.run.smithi181.stdout:Install 13 Packages 2023-10-09T03:45:28.765 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:28.765 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 1.9 M 2023-10-09T03:45:28.766 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 25 M 2023-10-09T03:45:28.766 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:45:28.911 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T03:45:28.934 INFO:teuthology.orchestra.run.smithi181.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 126 kB/s | 16 kB 00:00 2023-10-09T03:45:28.959 INFO:teuthology.orchestra.run.smithi181.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 214 kB/s | 33 kB 00:00 2023-10-09T03:45:28.984 INFO:teuthology.orchestra.run.smithi181.stdout:(3/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 264 kB/s | 47 kB 00:00 2023-10-09T03:45:29.018 INFO:teuthology.orchestra.run.smithi181.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 970 kB/s | 81 kB 00:00 2023-10-09T03:45:29.077 INFO:teuthology.orchestra.run.smithi181.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 1.1 MB/s | 61 kB 00:00 2023-10-09T03:45:29.127 INFO:teuthology.orchestra.run.smithi181.stdout:(6/13): python3-rsa-4.9-2.el8.noarch.rpm 452 kB/s | 64 kB 00:00 2023-10-09T03:45:29.161 INFO:teuthology.orchestra.run.smithi181.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 5.8 MB/s | 1.2 MB 00:00 2023-10-09T03:45:29.269 INFO:teuthology.orchestra.run.smithi181.stdout:(8/13): python3-pyasn1-modules-0.3.7-6.el8.noar 573 kB/s | 110 kB 00:00 2023-10-09T03:45:29.303 INFO:teuthology.orchestra.run.smithi181.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 151 kB/s | 27 kB 00:00 2023-10-09T03:45:29.336 INFO:teuthology.orchestra.run.smithi181.stdout:(10/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 744 kB/s | 126 kB 00:00 2023-10-09T03:45:29.403 INFO:teuthology.orchestra.run.smithi181.stdout:(11/13): python3-jsonpointer-1.10-11.el8.noarch 149 kB/s | 20 kB 00:00 2023-10-09T03:45:29.428 INFO:teuthology.orchestra.run.smithi181.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 340 kB/s | 43 kB 00:00 2023-10-09T03:45:29.504 INFO:teuthology.orchestra.run.smithi181.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 927 kB/s | 155 kB 00:00 2023-10-09T03:45:29.506 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:29.506 INFO:teuthology.orchestra.run.smithi181.stdout:Total 2.6 MB/s | 1.9 MB 00:00 2023-10-09T03:45:29.565 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:45:29.580 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:45:29.580 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:45:29.651 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:45:29.655 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:45:29.972 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T03:45:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:45:30.213 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T03:45:30.304 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T03:45:30.456 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:45:30.515 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T03:45:30.574 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:45:30.574 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T03:45:30.574 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T03:45:30.574 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T03:45:30.575 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-10-09T03:45:30.740 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T03:45:30.983 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-10-09T03:45:31.305 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-10-09T03:45:31.328 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-10-09T03:45:31.515 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-10-09T03:45:31.696 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-10-09T03:45:31.696 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:26 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:31.711 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T03:45:31.876 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T03:45:32.049 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T03:45:33.100 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T03:45:33.338 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T03:45:33.479 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:45:33.717 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:45:33.717 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T03:45:33.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-10-09T03:45:34.063 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T03:45:34.247 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:34.248 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:45:34.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T03:45:34.742 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:34.834 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-10-09T03:45:34.834 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:29 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:35.589 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:30 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:36.959 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:45:36.960 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================================== 2023-10-09T03:45:36.960 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================================== 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:45:36.961 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================================== 2023-10-09T03:45:36.962 INFO:teuthology.orchestra.run.smithi163.stdout:Install 5 Packages 2023-10-09T03:45:36.962 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:36.962 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 6.0 M 2023-10-09T03:45:36.962 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 26 M 2023-10-09T03:45:36.962 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:45:37.409 INFO:teuthology.orchestra.run.smithi163.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 402 kB/s | 121 kB 00:00 2023-10-09T03:45:37.471 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:45:37.476 INFO:teuthology.orchestra.run.smithi163.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 7.9 MB/s | 538 kB 00:00 2023-10-09T03:45:37.494 INFO:teuthology.orchestra.run.smithi163.stdout:(3/5): python3-natsort-7.1.1-2.el8.noarch.rpm 156 kB/s | 60 kB 00:00 2023-10-09T03:45:37.610 INFO:teuthology.orchestra.run.smithi163.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 36 MB/s | 4.8 MB 00:00 2023-10-09T03:45:37.657 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:37.860 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:45:37.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:37.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T03:45:37.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T03:45:37.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T03:45:37.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T03:45:37.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T03:45:37.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T03:45:37.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T03:45:37.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T03:45:37.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T03:45:37.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T03:45:37.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T03:45:37.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T03:45:37.862 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:37.862 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:45:37.864 INFO:teuthology.orchestra.run.smithi163.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 758 kB/s | 476 kB 00:00 2023-10-09T03:45:37.864 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:37.864 INFO:teuthology.orchestra.run.smithi163.stdout:Total 7.7 MB/s | 6.0 MB 00:00 2023-10-09T03:45:37.864 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:45:37.864 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:45:37.865 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:45:37.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T03:45:37.883 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:45:37.886 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:45:38.215 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:38.883 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:45:39.222 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T03:45:39.463 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T03:45:39.769 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:34 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:39.784 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:45:40.063 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T03:45:40.244 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:45:40.482 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:45:40.482 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T03:45:40.482 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T03:45:40.482 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:45:40.482 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T03:45:41.153 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:45:41.154 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================================== 2023-10-09T03:45:41.154 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================================== 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================================== 2023-10-09T03:45:41.155 INFO:teuthology.orchestra.run.smithi181.stdout:Install 5 Packages 2023-10-09T03:45:41.156 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:41.156 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 6.0 M 2023-10-09T03:45:41.156 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 26 M 2023-10-09T03:45:41.156 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:45:41.502 INFO:teuthology.orchestra.run.smithi181.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 424 kB/s | 121 kB 00:00 2023-10-09T03:45:41.518 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T03:45:41.518 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:36 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:41.527 INFO:teuthology.orchestra.run.smithi181.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 194 kB/s | 60 kB 00:00 2023-10-09T03:45:41.619 INFO:teuthology.orchestra.run.smithi181.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 4.5 MB/s | 538 kB 00:00 2023-10-09T03:45:41.661 INFO:teuthology.orchestra.run.smithi181.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.0 MB/s | 476 kB 00:00 2023-10-09T03:45:41.745 INFO:teuthology.orchestra.run.smithi181.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 22 MB/s | 4.8 MB 00:00 2023-10-09T03:45:41.747 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:41.747 INFO:teuthology.orchestra.run.smithi181.stdout:Total 10 MB/s | 6.0 MB 00:00 2023-10-09T03:45:41.800 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:45:41.812 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:45:41.812 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:45:41.891 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:45:41.894 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:45:42.833 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:45:43.164 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T03:45:43.420 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T03:45:43.774 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:45:44.013 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T03:45:44.194 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:45:44.432 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:45:44.432 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T03:45:44.432 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T03:45:44.432 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:45:44.432 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T03:45:45.397 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:45:45.449 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T03:45:45.450 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:40 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:45.582 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:45.582 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:45:45.582 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:45.582 INFO:teuthology.orchestra.run.smithi163.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T03:45:45.582 INFO:teuthology.orchestra.run.smithi163.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T03:45:45.583 INFO:teuthology.orchestra.run.smithi163.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T03:45:45.583 INFO:teuthology.orchestra.run.smithi163.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T03:45:45.583 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:45.583 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:45:45.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install ceph-fuse 2023-10-09T03:45:46.088 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:47.424 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:42 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:48.785 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repository Size 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-fuse 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout:Installing dependencies: 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:45:48.786 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:45:48.787 INFO:teuthology.orchestra.run.smithi163.stdout:Install 2 Packages 2023-10-09T03:45:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:48.787 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 959 k 2023-10-09T03:45:48.787 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 3.0 M 2023-10-09T03:45:48.787 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:45:49.290 INFO:teuthology.orchestra.run.smithi163.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 165 kB/s | 83 kB 00:00 2023-10-09T03:45:49.547 INFO:teuthology.orchestra.run.smithi163.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 1.1 MB/s | 876 kB 00:00 2023-10-09T03:45:49.548 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:49.549 INFO:teuthology.orchestra.run.smithi163.stdout:Total 1.2 MB/s | 959 kB 00:00 2023-10-09T03:45:49.562 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:45:49.586 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:45:49.586 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:45:49.666 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:45:49.670 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:45:50.235 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:45:50.494 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T03:45:50.666 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:45:50.695 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:45:50.879 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:50.879 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:45:50.879 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:45:50.879 INFO:teuthology.orchestra.run.smithi181.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T03:45:50.879 INFO:teuthology.orchestra.run.smithi181.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T03:45:50.880 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T03:45:50.880 INFO:teuthology.orchestra.run.smithi181.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T03:45:50.880 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:50.880 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:45:50.986 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:45:50.986 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T03:45:52.061 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T03:45:52.061 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:47 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:52.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-fuse 2023-10-09T03:45:52.933 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:54.400 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:49 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:55.836 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:45:55.836 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:45:55.836 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2023-10-09T03:45:55.836 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-fuse 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout:Install 2 Packages 2023-10-09T03:45:55.837 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:45:55.838 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 959 k 2023-10-09T03:45:55.838 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 3.0 M 2023-10-09T03:45:55.838 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:45:55.991 INFO:teuthology.orchestra.run.smithi181.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 540 kB/s | 83 kB 00:00 2023-10-09T03:45:56.108 INFO:teuthology.orchestra.run.smithi181.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 3.2 MB/s | 876 kB 00:00 2023-10-09T03:45:56.109 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:45:56.109 INFO:teuthology.orchestra.run.smithi181.stdout:Total 3.4 MB/s | 959 kB 00:00 2023-10-09T03:45:56.120 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:45:56.141 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:45:56.141 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:45:56.215 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:45:56.218 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:45:56.378 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:45:56.624 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T03:45:56.774 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:45:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:45:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T03:45:57.125 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:45:57.305 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:57.306 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:45:57.306 INFO:teuthology.orchestra.run.smithi163.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T03:45:57.306 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:45:57.306 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:45:57.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install librados-devel 2023-10-09T03:45:57.800 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:45:58.168 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T03:45:58.168 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:53 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:45:58.674 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:53 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:00.055 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repo Size 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:46:00.056 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:00.057 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:46:00.057 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:00.057 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 130 k 2023-10-09T03:46:00.057 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 464 k 2023-10-09T03:46:00.057 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:46:00.257 INFO:teuthology.orchestra.run.smithi163.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 649 kB/s | 130 kB 00:00 2023-10-09T03:46:00.258 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:00.258 INFO:teuthology.orchestra.run.smithi163.stdout:Total 645 kB/s | 130 kB 00:00 2023-10-09T03:46:00.258 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:46:00.264 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:46:00.264 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:46:00.318 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:46:00.321 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:46:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:46:00.614 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:46:00.793 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:00.793 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:46:00.794 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T03:46:00.794 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:00.794 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:00.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install librados-devel 2023-10-09T03:46:01.282 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:01.287 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:02.144 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:02:57 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:02.344 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:02.345 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:02:57 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:03.574 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:03.575 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:03.575 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2023-10-09T03:46:03.575 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:03.575 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:46:03.575 INFO:teuthology.orchestra.run.smithi181.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T03:46:03.575 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:03.576 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:46:03.576 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:03.576 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:46:03.576 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:03.576 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 130 k 2023-10-09T03:46:03.576 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 464 k 2023-10-09T03:46:03.577 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:46:03.874 INFO:teuthology.orchestra.run.smithi181.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 437 kB/s | 130 kB 00:00 2023-10-09T03:46:03.874 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:03.874 INFO:teuthology.orchestra.run.smithi181.stdout:Total 435 kB/s | 130 kB 00:00 2023-10-09T03:46:03.875 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:46:03.880 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:46:03.880 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:46:03.935 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:46:03.939 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:46:04.049 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:46:04.205 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:04.438 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:05.451 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:05.451 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:00 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:07.160 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:46:07.343 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:07.343 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:46:07.343 INFO:teuthology.orchestra.run.smithi163.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:07.343 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:07.343 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:07.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install libcephfs2 2023-10-09T03:46:07.864 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:09.277 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:04 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:10.171 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:46:10.352 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:10.353 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:46:10.353 INFO:teuthology.orchestra.run.smithi181.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:10.353 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:10.353 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:10.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install libcephfs2 2023-10-09T03:46:10.568 INFO:teuthology.orchestra.run.smithi163.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:10.677 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:10.678 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:10.679 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:10.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install libcephfs-devel 2023-10-09T03:46:10.833 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:11.170 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:12.397 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:07 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:12.748 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:07 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:13.664 INFO:teuthology.orchestra.run.smithi181.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:13.775 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:13.777 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:13.777 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:14.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install libcephfs-devel 2023-10-09T03:46:14.138 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:14.139 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:14.139 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repo Size 2023-10-09T03:46:14.139 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:14.139 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:46:14.139 INFO:teuthology.orchestra.run.smithi163.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T03:46:14.139 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:14.140 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:46:14.140 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:14.140 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:46:14.140 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:14.140 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 25 k 2023-10-09T03:46:14.141 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 109 k 2023-10-09T03:46:14.141 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:46:14.288 INFO:teuthology.orchestra.run.smithi163.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 168 kB/s | 25 kB 00:00 2023-10-09T03:46:14.289 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:14.289 INFO:teuthology.orchestra.run.smithi163.stdout:Total 167 kB/s | 25 kB 00:00 2023-10-09T03:46:14.289 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:46:14.293 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:46:14.293 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:46:14.295 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:46:14.299 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:46:14.349 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:46:14.456 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:14.519 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:14.717 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:15.727 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:15.727 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:10 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:15.991 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:10 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:17.423 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:17.424 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:46:17.425 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:17.426 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 25 k 2023-10-09T03:46:17.426 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 109 k 2023-10-09T03:46:17.426 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:46:17.569 INFO:teuthology.orchestra.run.smithi181.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 172 kB/s | 25 kB 00:00 2023-10-09T03:46:17.570 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:17.570 INFO:teuthology.orchestra.run.smithi181.stdout:Total 171 kB/s | 25 kB 00:00 2023-10-09T03:46:17.570 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:46:17.574 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:46:17.574 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:46:17.576 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:46:17.579 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:46:17.627 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:46:17.795 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:18.007 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:19.009 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:19.010 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:14 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:21.981 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:46:22.163 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:22.163 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:46:22.163 INFO:teuthology.orchestra.run.smithi163.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:22.163 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:22.163 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:22.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install librados2 2023-10-09T03:46:22.658 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:23.359 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:46:23.537 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:23.537 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:46:23.537 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:23.537 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:23.537 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:23.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install librados2 2023-10-09T03:46:23.886 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:18 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:24.017 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:25.012 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:20 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:25.136 INFO:teuthology.orchestra.run.smithi163.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:25.248 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:25.249 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:25.249 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:25.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install librbd1 2023-10-09T03:46:25.733 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:26.270 INFO:teuthology.orchestra.run.smithi181.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:26.381 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:26.382 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:26.382 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:26.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install librbd1 2023-10-09T03:46:26.862 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:27.056 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:22 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:28.299 INFO:teuthology.orchestra.run.smithi163.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:28.407 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:28.409 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:28.409 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:28.411 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:23 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:28.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install python3-rados 2023-10-09T03:46:28.893 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:29.679 INFO:teuthology.orchestra.run.smithi181.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:29.789 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:29.790 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:29.790 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:29.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-rados 2023-10-09T03:46:30.254 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:30.305 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:25 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:31.319 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:26 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:31.552 INFO:teuthology.orchestra.run.smithi163.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:31.660 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:31.662 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:31.662 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:31.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install python3-rgw 2023-10-09T03:46:32.140 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:32.569 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:32.678 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:32.679 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:32.679 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:32.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-rgw 2023-10-09T03:46:33.004 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:28 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:33.140 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:33.987 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:28 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:34.262 INFO:teuthology.orchestra.run.smithi163.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:34.369 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:34.370 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:34.371 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:34.499 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install python3-cephfs 2023-10-09T03:46:34.852 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:35.258 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:35.366 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:35.368 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:35.368 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:35.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-cephfs 2023-10-09T03:46:35.944 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:36.155 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:31 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:37.392 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:32 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:37.410 INFO:teuthology.orchestra.run.smithi163.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:37.519 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:37.520 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:37.520 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:37.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install python3-rbd 2023-10-09T03:46:38.008 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:38.649 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:38.758 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:38.759 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:38.759 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:38.875 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-rbd 2023-10-09T03:46:39.228 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:39.498 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:34 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:40.661 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:35 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:40.779 INFO:teuthology.orchestra.run.smithi163.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:40.887 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:40.888 INFO:teuthology.orchestra.run.smithi163.stdout:Nothing to do. 2023-10-09T03:46:40.889 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:41.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install rbd-fuse 2023-10-09T03:46:41.371 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:41.935 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:46:42.046 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:42.046 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2023-10-09T03:46:42.046 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:42.195 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install rbd-fuse 2023-10-09T03:46:42.250 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:37 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:42.550 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:43.412 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:38 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:43.617 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout: Package Architecture Version Repository Size 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:43.618 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:46:43.619 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:43.619 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:46:43.619 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:43.619 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 90 k 2023-10-09T03:46:43.619 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 264 k 2023-10-09T03:46:43.619 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:46:43.907 INFO:teuthology.orchestra.run.smithi163.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 314 kB/s | 90 kB 00:00 2023-10-09T03:46:43.907 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:43.908 INFO:teuthology.orchestra.run.smithi163.stdout:Total 312 kB/s | 90 kB 00:00 2023-10-09T03:46:43.908 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:46:43.918 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:46:43.918 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:46:43.973 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:46:43.976 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:46:44.102 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:46:44.248 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:44.472 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:44.784 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:46:44.785 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:44.786 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:46:44.786 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:44.786 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 90 k 2023-10-09T03:46:44.786 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 264 k 2023-10-09T03:46:44.786 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:46:44.955 INFO:teuthology.orchestra.run.smithi181.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 536 kB/s | 90 kB 00:00 2023-10-09T03:46:44.955 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:44.956 INFO:teuthology.orchestra.run.smithi181.stdout:Total 532 kB/s | 90 kB 00:00 2023-10-09T03:46:44.956 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:46:44.965 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:46:44.965 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:46:45.021 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:46:45.025 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:46:45.165 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:46:45.307 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:45.526 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:45.543 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:45.543 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:40 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:46.551 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:46.551 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:41 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:47.942 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:46:48.128 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:48.128 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:46:48.128 INFO:teuthology.orchestra.run.smithi163.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:48.128 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:48.128 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:48.267 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install rbd-mirror 2023-10-09T03:46:48.626 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:49.017 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:46:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:49.198 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:46:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:49.198 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:49.198 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:49.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install rbd-mirror 2023-10-09T03:46:49.495 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:44 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:49.683 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:50.555 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:45 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:50.910 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:46:50.910 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:50.910 INFO:teuthology.orchestra.run.smithi163.stdout: Package Arch Version Repo Size 2023-10-09T03:46:50.910 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:50.910 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:46:50.911 INFO:teuthology.orchestra.run.smithi163.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:46:50.911 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:50.911 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:46:50.911 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:46:50.911 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:46:50.911 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:50.912 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 3.5 M 2023-10-09T03:46:50.912 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 16 M 2023-10-09T03:46:50.912 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:46:51.526 INFO:teuthology.orchestra.run.smithi163.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 5.6 MB/s | 3.5 MB 00:00 2023-10-09T03:46:51.527 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:51.527 INFO:teuthology.orchestra.run.smithi163.stdout:Total 5.6 MB/s | 3.5 MB 00:00 2023-10-09T03:46:51.528 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:46:51.539 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:46:51.539 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:46:51.605 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:46:51.609 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:46:51.960 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:46:51.960 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:51.960 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2023-10-09T03:46:51.960 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:51.960 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:46:51.960 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:46:51.961 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:51.961 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:46:51.961 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:46:51.961 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:46:51.961 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:51.961 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 3.5 M 2023-10-09T03:46:51.962 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 16 M 2023-10-09T03:46:51.962 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:46:52.048 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:46:52.216 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:52.580 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:52.665 INFO:teuthology.orchestra.run.smithi181.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 4.9 MB/s | 3.5 MB 00:00 2023-10-09T03:46:52.666 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:46:52.666 INFO:teuthology.orchestra.run.smithi181.stdout:Total 4.9 MB/s | 3.5 MB 00:00 2023-10-09T03:46:52.667 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:46:52.679 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:46:52.679 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:46:52.746 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:46:52.749 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:46:53.186 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:53.721 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:53.875 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:53.875 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:48 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:54.757 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:46:54.757 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:49 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:46:58.073 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:46:58.254 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:58.254 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:46:58.254 INFO:teuthology.orchestra.run.smithi163.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:58.254 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:46:58.254 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:46:58.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo yum -y install rbd-nbd 2023-10-09T03:46:58.752 INFO:teuthology.orchestra.run.smithi163.stdout:Updating Subscription Management repositories. 2023-10-09T03:46:58.998 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:46:59.187 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:59.187 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:46:59.187 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:46:59.187 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:46:59.187 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:46:59.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install rbd-nbd 2023-10-09T03:46:59.677 INFO:teuthology.orchestra.run.smithi181.stdout:Updating Subscription Management repositories. 2023-10-09T03:47:00.204 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:55 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:47:01.379 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:56 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:47:01.567 INFO:teuthology.orchestra.run.smithi163.stdout:Dependencies resolved. 2023-10-09T03:47:01.568 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:47:01.568 INFO:teuthology.orchestra.run.smithi163.stdout: Package Architecture Version Repository Size 2023-10-09T03:47:01.568 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:47:01.568 INFO:teuthology.orchestra.run.smithi163.stdout:Installing: 2023-10-09T03:47:01.568 INFO:teuthology.orchestra.run.smithi163.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction Summary 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout:================================================================================ 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout:Install 1 Package 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout:Total download size: 173 k 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout:Installed size: 552 k 2023-10-09T03:47:01.569 INFO:teuthology.orchestra.run.smithi163.stdout:Downloading Packages: 2023-10-09T03:47:01.862 INFO:teuthology.orchestra.run.smithi163.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 590 kB/s | 173 kB 00:00 2023-10-09T03:47:01.863 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:47:01.863 INFO:teuthology.orchestra.run.smithi163.stdout:Total 588 kB/s | 173 kB 00:00 2023-10-09T03:47:01.863 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction check 2023-10-09T03:47:01.874 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction check succeeded. 2023-10-09T03:47:01.874 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction test 2023-10-09T03:47:01.928 INFO:teuthology.orchestra.run.smithi163.stdout:Transaction test succeeded. 2023-10-09T03:47:01.932 INFO:teuthology.orchestra.run.smithi163.stdout:Running transaction 2023-10-09T03:47:02.048 INFO:teuthology.orchestra.run.smithi163.stdout: Preparing : 1/1 2023-10-09T03:47:02.211 INFO:teuthology.orchestra.run.smithi163.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:47:02.435 INFO:teuthology.orchestra.run.smithi163.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:47:02.737 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:47:02.738 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2023-10-09T03:47:02.739 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2023-10-09T03:47:02.739 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2023-10-09T03:47:02.739 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:47:02.739 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 173 k 2023-10-09T03:47:02.739 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 552 k 2023-10-09T03:47:02.740 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2023-10-09T03:47:03.158 INFO:teuthology.orchestra.run.smithi181.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 414 kB/s | 173 kB 00:00 2023-10-09T03:47:03.158 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:47:03.158 INFO:teuthology.orchestra.run.smithi181.stdout:Total 413 kB/s | 173 kB 00:00 2023-10-09T03:47:03.158 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2023-10-09T03:47:03.169 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2023-10-09T03:47:03.169 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2023-10-09T03:47:03.225 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2023-10-09T03:47:03.228 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2023-10-09T03:47:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2023-10-09T03:47:03.504 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:47:03.510 INFO:teuthology.orchestra.run.smithi163.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:47:03.511 INFO:teuthology.orchestra.run.smithi163.stdout:Last metadata expiration check: 0:03:58 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:47:03.739 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:47:04.815 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:47:04.815 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:59 ago on Mon 09 Oct 2023 03:43:05 AM UTC. 2023-10-09T03:47:07.538 INFO:teuthology.orchestra.run.smithi163.stdout:Installed products updated. 2023-10-09T03:47:07.718 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:47:07.718 INFO:teuthology.orchestra.run.smithi163.stdout:Installed: 2023-10-09T03:47:07.718 INFO:teuthology.orchestra.run.smithi163.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:47:07.718 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:47:07.718 INFO:teuthology.orchestra.run.smithi163.stdout:Complete! 2023-10-09T03:47:07.891 DEBUG:teuthology.parallel:result is None 2023-10-09T03:47:07.999 INFO:teuthology.orchestra.run.smithi181.stdout:Installed products updated. 2023-10-09T03:47:08.178 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:47:08.178 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2023-10-09T03:47:08.178 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:47:08.179 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:47:08.179 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2023-10-09T03:47:08.343 DEBUG:teuthology.parallel:result is None 2023-10-09T03:47:08.343 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:47:08.343 INFO:teuthology.packaging:ref: None 2023-10-09T03:47:08.343 INFO:teuthology.packaging:tag: None 2023-10-09T03:47:08.344 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:47:08.344 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:47:08.344 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:47:08.748 DEBUG:teuthology.orchestra.run.smithi163:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T03:47:08.775 INFO:teuthology.orchestra.run.smithi163.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T03:47:08.775 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T03:47:08.776 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T03:47:08.776 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:47:08.776 INFO:teuthology.packaging:ref: None 2023-10-09T03:47:08.776 INFO:teuthology.packaging:tag: None 2023-10-09T03:47:08.776 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:47:08.776 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:47:08.776 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:47:09.160 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T03:47:09.185 INFO:teuthology.orchestra.run.smithi181.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T03:47:09.185 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T03:47:09.185 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T03:47:09.185 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T03:47:09.186 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:47:09.186 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T03:47:09.219 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:47:09.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T03:47:09.255 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T03:47:09.256 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:47:09.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T03:47:09.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T03:47:09.358 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:47:09.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T03:47:09.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T03:47:09.455 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T03:47:09.455 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:47:09.455 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T03:47:09.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T03:47:09.552 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:47:09.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T03:47:09.578 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T03:47:09.646 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T03:47:09.647 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:47:09.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T03:47:09.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T03:47:09.742 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:47:09.742 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T03:47:09.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T03:47:09.837 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T03:47:09.910 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T03:47:09.910 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T03:47:09.910 INFO:tasks.cephadm:Cluster fsid is 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:47:09.911 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T03:47:09.911 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.163', 'mon.c': '[v2:172.21.15.163:3301,v1:172.21.15.163:6790]', 'mon.b': '172.21.15.181'} 2023-10-09T03:47:09.911 INFO:tasks.cephadm:First mon is mon.a on smithi163 2023-10-09T03:47:09.911 INFO:tasks.cephadm:First mgr is y 2023-10-09T03:47:09.911 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T03:47:09.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo hostname $(hostname -s) 2023-10-09T03:47:09.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo hostname $(hostname -s) 2023-10-09T03:47:09.966 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-10-09T03:47:09.967 DEBUG:teuthology.orchestra.run.smithi163:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T03:47:10.115 INFO:teuthology.orchestra.run.smithi163.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 03:47 /home/ubuntu/cephtest/cephadm 2023-10-09T03:47:10.115 DEBUG:teuthology.orchestra.run.smithi181:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T03:47:10.260 INFO:teuthology.orchestra.run.smithi181.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 03:47 /home/ubuntu/cephtest/cephadm 2023-10-09T03:47:10.260 DEBUG:teuthology.orchestra.run.smithi163:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T03:47:10.278 DEBUG:teuthology.orchestra.run.smithi181:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T03:47:10.306 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-10-09T03:47:10.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T03:47:10.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T03:47:11.439 INFO:teuthology.orchestra.run.smithi163.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T03:47:11.507 INFO:teuthology.orchestra.run.smithi181.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T03:47:54.201 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T03:47:54.201 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T03:47:54.201 INFO:teuthology.orchestra.run.smithi163.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T03:47:54.201 INFO:teuthology.orchestra.run.smithi163.stdout: "repo_digests": [ 2023-10-09T03:47:54.201 INFO:teuthology.orchestra.run.smithi163.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T03:47:54.202 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T03:47:54.202 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T03:47:57.754 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T03:47:57.754 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T03:47:57.755 INFO:teuthology.orchestra.run.smithi181.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T03:47:57.755 INFO:teuthology.orchestra.run.smithi181.stdout: "repo_digests": [ 2023-10-09T03:47:57.755 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T03:47:57.755 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T03:47:57.755 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T03:47:57.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo mkdir -p /etc/ceph 2023-10-09T03:47:57.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo mkdir -p /etc/ceph 2023-10-09T03:47:57.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 777 /etc/ceph 2023-10-09T03:47:57.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 777 /etc/ceph 2023-10-09T03:47:57.917 INFO:tasks.cephadm:Writing seed config... 2023-10-09T03:47:57.918 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T03:47:57.918 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T03:47:57.918 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T03:47:57.919 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T03:47:57.920 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:47:57.920 DEBUG:teuthology.orchestra.run.smithi163:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T03:47:57.948 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 8628d772-6656-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T03:47:57.949 DEBUG:teuthology.orchestra.run.smithi163:mon.a> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T03:47:57.991 DEBUG:teuthology.orchestra.run.smithi163:mgr.y> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T03:47:58.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:47:58.024 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T03:47:58.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.163 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T03:47:58.052 INFO:journalctl@ceph.mgr.y.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:47:58.231 INFO:teuthology.orchestra.run.smithi163.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:47:58.231 INFO:teuthology.orchestra.run.smithi163.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '8628d772-6656-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.163', '--skip-admin-label'] 2023-10-09T03:47:58.299 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stdout 3.0.1 2023-10-09T03:47:58.300 INFO:teuthology.orchestra.run.smithi163.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T03:47:58.300 INFO:teuthology.orchestra.run.smithi163.stdout:Verifying podman|docker is present... 2023-10-09T03:47:58.360 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stdout 3.0.1 2023-10-09T03:47:58.361 INFO:teuthology.orchestra.run.smithi163.stdout:Verifying lvm2 is present... 2023-10-09T03:47:58.361 INFO:teuthology.orchestra.run.smithi163.stdout:Verifying time synchronization is in place... 2023-10-09T03:47:58.367 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T03:47:58.367 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T03:47:58.373 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T03:47:58.373 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout inactive 2023-10-09T03:47:58.379 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout enabled 2023-10-09T03:47:58.385 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout active 2023-10-09T03:47:58.385 INFO:teuthology.orchestra.run.smithi163.stdout:Unit chronyd.service is enabled and running 2023-10-09T03:47:58.385 INFO:teuthology.orchestra.run.smithi163.stdout:Repeating the final host check... 2023-10-09T03:47:58.444 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stdout 3.0.1 2023-10-09T03:47:58.444 INFO:teuthology.orchestra.run.smithi163.stdout:podman (/bin/podman) version 3.0.1 is present 2023-10-09T03:47:58.444 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl is present 2023-10-09T03:47:58.444 INFO:teuthology.orchestra.run.smithi163.stdout:lvcreate is present 2023-10-09T03:47:58.449 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T03:47:58.450 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T03:47:58.455 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T03:47:58.455 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout inactive 2023-10-09T03:47:58.461 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout enabled 2023-10-09T03:47:58.467 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout active 2023-10-09T03:47:58.467 INFO:teuthology.orchestra.run.smithi163.stdout:Unit chronyd.service is enabled and running 2023-10-09T03:47:58.467 INFO:teuthology.orchestra.run.smithi163.stdout:Host looks OK 2023-10-09T03:47:58.467 INFO:teuthology.orchestra.run.smithi163.stdout:Cluster fsid: 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:47:58.467 INFO:teuthology.orchestra.run.smithi163.stdout:Acquiring lock 139777498725344 on /run/cephadm/8628d772-6656-11ee-8db5-212e2dc638e7.lock 2023-10-09T03:47:58.467 INFO:teuthology.orchestra.run.smithi163.stdout:Lock 139777498725344 acquired on /run/cephadm/8628d772-6656-11ee-8db5-212e2dc638e7.lock 2023-10-09T03:47:58.468 INFO:teuthology.orchestra.run.smithi163.stdout:Verifying IP 172.21.15.163 port 3300 ... 2023-10-09T03:47:58.468 INFO:teuthology.orchestra.run.smithi163.stdout:Verifying IP 172.21.15.163 port 6789 ... 2023-10-09T03:47:58.468 INFO:teuthology.orchestra.run.smithi163.stdout:Base mon IP(s) is [172.21.15.163:3300, 172.21.15.163:6789], mon addrv is [v2:172.21.15.163:3300,v1:172.21.15.163:6789] 2023-10-09T03:47:58.471 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-10-09T03:47:58.471 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.163 metric 100 2023-10-09T03:47:58.474 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T03:47:58.474 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-10-09T03:47:58.474 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-10-09T03:47:58.476 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T03:47:58.477 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T03:47:58.477 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T03:47:58.477 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T03:47:58.477 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout inet6 fe80::9f05:49a1:a680:abae/64 scope link noprefixroute 2023-10-09T03:47:58.477 INFO:teuthology.orchestra.run.smithi163.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T03:47:58.478 INFO:teuthology.orchestra.run.smithi163.stdout:Mon IP `172.21.15.163` is in CIDR network `172.21.0.0/20` 2023-10-09T03:47:58.478 INFO:teuthology.orchestra.run.smithi163.stdout:Mon IP `172.21.15.163` is in CIDR network `172.21.0.0/20` 2023-10-09T03:47:58.478 INFO:teuthology.orchestra.run.smithi163.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T03:47:58.478 INFO:teuthology.orchestra.run.smithi163.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T03:47:58.479 INFO:teuthology.orchestra.run.smithi163.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T03:47:58.870 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stdout 5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T03:47:58.870 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T03:47:58.870 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Getting image source signatures 2023-10-09T03:47:58.870 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Copying blob sha256:1b55aded3a89dccf5f05a82d583393d22b3476851fe870b413c77ca909f5f760 2023-10-09T03:47:58.871 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T03:47:58.871 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Copying config sha256:5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T03:47:58.871 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-09T03:47:58.871 INFO:teuthology.orchestra.run.smithi163.stdout:/bin/podman: stderr Storing signatures 2023-10-09T03:47:59.921 INFO:teuthology.orchestra.run.smithi163.stdout:ceph: stdout ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T03:47:59.921 INFO:teuthology.orchestra.run.smithi163.stdout:Ceph version: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T03:47:59.921 INFO:teuthology.orchestra.run.smithi163.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T03:48:00.813 INFO:teuthology.orchestra.run.smithi163.stdout:stat: stdout 167 167 2023-10-09T03:48:00.813 INFO:teuthology.orchestra.run.smithi163.stdout:Creating initial keys... 2023-10-09T03:48:01.651 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph-authtool: stdout AQDxdyNl8s0aBxAA/xdHf34juakZqlp/SkduKg== 2023-10-09T03:48:02.540 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph-authtool: stdout AQDxdyNlr/+VORAAq1qP9qpnfe9EaQpYmq4CnA== 2023-10-09T03:48:03.491 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph-authtool: stdout AQDydyNlO5ASNRAASloSwUOZM4pg7Lds08Td9w== 2023-10-09T03:48:03.492 INFO:teuthology.orchestra.run.smithi163.stdout:Creating initial monmap... 2023-10-09T03:48:04.380 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:monmaptool for a [v2:172.21.15.163:3300,v1:172.21.15.163:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:setting min_mon_release = octopus 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/monmaptool: set fsid to 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T03:48:04.381 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:48:04.382 INFO:teuthology.orchestra.run.smithi163.stdout:Creating mon... 2023-10-09T03:48:05.312 INFO:teuthology.orchestra.run.smithi163.stdout:create mon.a on 2023-10-09T03:48:05.410 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T03:48:05.497 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T03:48:05.586 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-8628d772-6656-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-8628d772-6656-11ee-8db5-212e2dc638e7.target. 2023-10-09T03:48:05.586 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-8628d772-6656-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-8628d772-6656-11ee-8db5-212e2dc638e7.target. 2023-10-09T03:48:05.702 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T03:48:05.703 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Failed to reset failed state of unit ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T03:48:05.787 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-8628d772-6656-11ee-8db5-212e2dc638e7.target.wants/ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-8628d772-6656-11ee-8db5-212e2dc638e7@.service. 2023-10-09T03:48:06.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:05 smithi163 systemd[1]: Starting Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:48:06.358 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T03:48:06.358 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout disabled 2023-10-09T03:48:06.362 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T03:48:06.362 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout inactive 2023-10-09T03:48:06.362 INFO:teuthology.orchestra.run.smithi163.stdout:firewalld.service is not enabled 2023-10-09T03:48:06.363 INFO:teuthology.orchestra.run.smithi163.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T03:48:06.363 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for mon to start... 2023-10-09T03:48:06.363 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for mon... 2023-10-09T03:48:06.594 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:06 smithi163 bash[100012]: 692e5add7559ca9d5aac6d8e5e38053855d93db32b027cb544e7ca93fcb07705 2023-10-09T03:48:06.594 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:06 smithi163 systemd[1]: Started Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:48:06.594 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:06 smithi163 ceph-mon[100078]: mkfs 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:06.594 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:06 smithi163 ceph-mon[100078]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T03:48:07.574 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T03:48:07.575 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout id: 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout services: 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.509416s) 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout data: 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T03:48:07.576 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T03:48:07.577 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T03:48:07.577 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:07.577 INFO:teuthology.orchestra.run.smithi163.stdout:mon is available 2023-10-09T03:48:07.577 INFO:teuthology.orchestra.run.smithi163.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T03:48:07.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:07 smithi163 ceph-mon[100078]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T03:48:07.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:07 smithi163 ceph-mon[100078]: monmap e1: 1 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0]} removed_ranks: {} 2023-10-09T03:48:07.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:07 smithi163 ceph-mon[100078]: fsmap 2023-10-09T03:48:07.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:07 smithi163 ceph-mon[100078]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T03:48:07.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:07 smithi163 ceph-mon[100078]: mgrmap e1: no daemons active 2023-10-09T03:48:07.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:07 smithi163 ceph-mon[100078]: from='client.? 172.21.15.163:0/529821473' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T03:48:08.675 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:08.675 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T03:48:08.675 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout fsid = 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:08.675 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.163:3300,v1:172.21.15.163:6789] 2023-10-09T03:48:08.675 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T03:48:08.675 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T03:48:08.676 INFO:teuthology.orchestra.run.smithi163.stdout:Generating new minimal ceph.conf... 2023-10-09T03:48:08.679 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:08 smithi163 ceph-mon[100078]: from='client.? 172.21.15.163:0/2862741957' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T03:48:08.679 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:08 smithi163 ceph-mon[100078]: from='client.? 172.21.15.163:0/2862741957' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T03:48:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:09 smithi163 ceph-mon[100078]: from='client.? 172.21.15.163:0/525677346' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:48:09.748 INFO:teuthology.orchestra.run.smithi163.stdout:Restarting the monitor... 2023-10-09T03:48:10.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:09 smithi163 systemd[1]: Stopping Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:48:10.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:09 smithi163 conmon[100053]: 2023-10-09T03:48:09.988+0000 7f1127739700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T03:48:10.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:09 smithi163 conmon[100053]: 2023-10-09T03:48:09.988+0000 7f1127739700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T03:48:10.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:10 smithi163 bash[100633]: 692e5add7559ca9d5aac6d8e5e38053855d93db32b027cb544e7ca93fcb07705 2023-10-09T03:48:10.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:10 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T03:48:10.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:10 smithi163 systemd[1]: Stopped Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:48:10.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:10 smithi163 systemd[1]: Starting Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:48:11.312 INFO:teuthology.orchestra.run.smithi163.stdout:Setting mon public_network to 172.21.0.0/20 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: pidfile_write: ignore empty --pid-file 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: load: jerasure load: lrc 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: RocksDB version: 6.15.5 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Compile date Oct 8 2023 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: DB SUMMARY 2023-10-09T03:48:11.564 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: DB Session ID: LA3EA2LLZH0O0UY06JB8 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: CURRENT file: CURRENT 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83609 ; 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.error_if_exists: 0 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.create_if_missing: 0 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.paranoid_checks: 1 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.env: 0x55e604276a80 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.fs: Posix File System 2023-10-09T03:48:11.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.info_log: 0x55e605e065a0 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.statistics: (nil) 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.use_fsync: 0 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_log_file_size: 0 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.allow_fallocate: 1 2023-10-09T03:48:11.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.use_direct_reads: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.db_log_dir: 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.write_buffer_manager: 0x55e606b78900 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T03:48:11.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.unordered_write: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.row_cache: None 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.wal_filter: None 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.preserve_deletes: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.two_write_queues: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.atomic_flush: 0 2023-10-09T03:48:11.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.log_readahead_size: 0 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T03:48:11.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T03:48:11.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_background_jobs: 2 2023-10-09T03:48:11.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_background_compactions: -1 2023-10-09T03:48:11.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_subcompactions: 1 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_open_files: -1 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T03:48:11.572 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_background_flushes: -1 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Compression algorithms supported: 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kZSTD supported: 0 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kXpressCompression supported: 0 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kZlibCompression supported: 1 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kSnappyCompression supported: 1 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kLZ4Compression supported: 1 2023-10-09T03:48:11.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: kBZip2Compression supported: 0 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.merge_operator: 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_filter: None 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T03:48:11.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e605e58fb8) 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks: 1 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_top_level_index_and_filter: 1 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_type: 0 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_index_type: 0 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_shortening: 1 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: hash_index_allow_collision: 1 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: checksum: 1 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: no_block_cache: 0 2023-10-09T03:48:11.575 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache: 0x55e605e8b090 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_name: BinnedLRUCache 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_options: 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: capacity : 536870912 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: num_shard_bits : 4 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: strict_capacity_limit : 0 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: high_pri_pool_ratio: 0.000 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_compressed: (nil) 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: persistent_cache: (nil) 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size: 4096 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size_deviation: 10 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_restart_interval: 16 2023-10-09T03:48:11.576 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_block_restart_interval: 1 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: metadata_block_size: 4096 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: partition_filters: 0 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: use_delta_encoding: 1 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: whole_key_filtering: 1 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: verify_compression: 0 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: read_amp_bytes_per_bit: 0 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: format_version: 4 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_index_compression: 1 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_align: 0 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T03:48:11.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression: NoCompression 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.num_levels: 7 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T03:48:11.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T03:48:11.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T03:48:11.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T03:48:11.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.table_properties_collectors: 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.inplace_update_support: 0 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T03:48:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.bloom_locality: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.max_successive_merges: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.ttl: 2592000 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.enable_blob_files: false 2023-10-09T03:48:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.min_blob_size: 0 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823291299077, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T03:48:11.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823291299760, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80156, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78424, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72632, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696823291, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "01771d03-7dbc-4eff-bfe5-33a3b7fdc374", "db_session_id": "LA3EA2LLZH0O0UY06JB8"}} 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823291302652, "job": 1, "event": "recovery_finished"} 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55e605e1f180 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: DB pointer 0x55e605ef4000 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** DB Stats ** 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:48:11.585 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.586 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative compaction: 0.00 GB write, 13.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval compaction: 0.00 GB write, 13.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:48:11.587 INFO:journalctl@ceph.mon.a.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:48:11.588 INFO:journalctl@ceph.mon.a.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.1 0.00 0.00 1 0.000 0 0 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative compaction: 0.00 GB write, 13.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:48:11.589 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: starting mon.a rank 0 at public addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] at bind addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 bash[100957]: 51bd91cc44f86a966a2531891bb4ed396d01b78001a7aa0859e50e17f9acec93 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???) e1 preinit fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).mds e1 new map 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).mds e1 print_map 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: e1 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: legacy client fscid: -1 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:48:11.590 INFO:journalctl@ceph.mon.a.smithi163.stdout: No filesystems configured 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expand map: {default=false} 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta from 'false' to 'false' 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expanded map: {default=false} 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expand map: {default=info} 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta from 'info' to 'info' 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expanded map: {default=info} 2023-10-09T03:48:11.591 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expand map: {default=daemon} 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expand map: {default=debug} 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: expand_channel_meta expanded map: {default=debug} 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 systemd[1]: Started Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: monmap e1: 1 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0]} removed_ranks: {} 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: fsmap 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T03:48:11.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:11 smithi163 ceph-mon[101022]: mgrmap e1: no daemons active 2023-10-09T03:48:12.432 INFO:teuthology.orchestra.run.smithi163.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T03:48:12.432 INFO:teuthology.orchestra.run.smithi163.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T03:48:12.433 INFO:teuthology.orchestra.run.smithi163.stdout:Creating mgr... 2023-10-09T03:48:12.433 INFO:teuthology.orchestra.run.smithi163.stdout:Verifying port 9283 ... 2023-10-09T03:48:12.553 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T03:48:12.553 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Failed to reset failed state of unit ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T03:48:12.642 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-8628d772-6656-11ee-8db5-212e2dc638e7.target.wants/ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-8628d772-6656-11ee-8db5-212e2dc638e7@.service. 2023-10-09T03:48:12.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:12 smithi163 systemd[1]: Starting Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:48:13.221 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:12 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2064824666' entity='client.admin' 2023-10-09T03:48:13.238 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T03:48:13.238 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout disabled 2023-10-09T03:48:13.245 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T03:48:13.245 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout inactive 2023-10-09T03:48:13.245 INFO:teuthology.orchestra.run.smithi163.stdout:firewalld.service is not enabled 2023-10-09T03:48:13.245 INFO:teuthology.orchestra.run.smithi163.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T03:48:13.253 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T03:48:13.253 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout disabled 2023-10-09T03:48:13.261 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T03:48:13.261 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout inactive 2023-10-09T03:48:13.261 INFO:teuthology.orchestra.run.smithi163.stdout:firewalld.service is not enabled 2023-10-09T03:48:13.261 INFO:teuthology.orchestra.run.smithi163.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T03:48:13.261 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for mgr to start... 2023-10-09T03:48:13.261 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for mgr... 2023-10-09T03:48:13.487 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 bash[101409]: ec02a34c21a59f771821f1ca617699f3e4012831bae51a1607f60772c0cf652d 2023-10-09T03:48:13.488 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 systemd[1]: Started Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:48:13.488 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 conmon[101450]: 2023-10-09T03:48:13.348+0000 7f7e67d8a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:48:13.488 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 conmon[101450]: 2023-10-09T03:48:13.427+0000 7f7e67d8a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:48:13.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 conmon[101450]: 2023-10-09T03:48:13.525+0000 7f7e67d8a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:48:13.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 conmon[101450]: 2023-10-09T03:48:13.620+0000 7f7e67d8a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:48:13.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:13 smithi163 conmon[101450]: 2023-10-09T03:48:13.794+0000 7f7e67d8a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:48:14.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:13 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3949256316' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T03:48:14.406 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:14.409 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:14.409 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsid": "8628d772-6656-11ee-8db5-212e2dc638e7", 2023-10-09T03:48:14.409 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 0 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T03:48:14.410 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T03:48:14.411 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T03:48:14.412 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T03:48:14.412 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T03:48:14.412 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T03:48:14.412 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T03:48:14.415 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.415 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T03:48:14.415 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T03:48:14.415 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T03:48:14.415 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T03:48:14.415 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T03:48:14.416 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T03:48:06.379819+0000", 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:14.417 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T03:48:14.418 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:14.418 INFO:teuthology.orchestra.run.smithi163.stdout:mgr not available, waiting (1/15)... 2023-10-09T03:48:14.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:14 smithi163 conmon[101450]: 2023-10-09T03:48:14.123+0000 7f7e67d8a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:48:14.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:14 smithi163 conmon[101450]: 2023-10-09T03:48:14.210+0000 7f7e67d8a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:48:14.418 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:14 smithi163 conmon[101450]: 2023-10-09T03:48:14.339+0000 7f7e67d8a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:48:14.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:14 smithi163 conmon[101450]: 2023-10-09T03:48:14.438+0000 7f7e67d8a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:48:15.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:14 smithi163 conmon[101450]: 2023-10-09T03:48:14.854+0000 7f7e67d8a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:48:15.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:15 smithi163 conmon[101450]: 2023-10-09T03:48:15.425+0000 7f7e67d8a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:48:15.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:15 smithi163 conmon[101450]: 2023-10-09T03:48:15.593+0000 7f7e67d8a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:48:16.621 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:16 smithi163 conmon[101450]: 2023-10-09T03:48:16.358+0000 7f7e67d8a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:48:16.621 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:16 smithi163 conmon[101450]: 2023-10-09T03:48:16.461+0000 7f7e67d8a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:48:16.926 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:16 smithi163 conmon[101450]: 2023-10-09T03:48:16.638+0000 7f7e67d8a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:48:17.308 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:16 smithi163 conmon[101450]: 2023-10-09T03:48:16.925+0000 7f7e67d8a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:48:17.309 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:17 smithi163 conmon[101450]: 2023-10-09T03:48:17.014+0000 7f7e67d8a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsid": "8628d772-6656-11ee-8db5-212e2dc638e7", 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.519 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 0 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T03:48:17.520 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T03:48:17.521 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T03:48:17.522 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T03:48:17.523 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:17.524 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T03:48:06.379819+0000", 2023-10-09T03:48:17.524 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:17.524 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:17.524 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T03:48:17.524 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:17.524 INFO:teuthology.orchestra.run.smithi163.stdout:mgr not available, waiting (2/15)... 2023-10-09T03:48:17.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:17 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3768943162' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T03:48:17.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:17 smithi163 conmon[101450]: 2023-10-09T03:48:17.320+0000 7f7e67d8a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:48:17.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:17 smithi163 conmon[101450]: 2023-10-09T03:48:17.580+0000 7f7e67d8a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:48:18.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:18 smithi163 conmon[101450]: 2023-10-09T03:48:18.280+0000 7f7e67d8a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:48:18.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:18 smithi163 conmon[101450]: 2023-10-09T03:48:18.408+0000 7f7e67d8a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:48:18.919 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:18 smithi163 conmon[101450]: 2023-10-09T03:48:18.594+0000 7f7e67d8a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:48:18.919 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:18 smithi163 conmon[101450]: 2023-10-09T03:48:18.706+0000 7f7e67d8a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:48:19.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:18 smithi163 conmon[101450]: 2023-10-09T03:48:18.918+0000 7f7e67d8a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:48:20.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:19 smithi163 conmon[101450]: 2023-10-09T03:48:19.804+0000 7f7e67d8a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: Activating manager daemon y 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: mgrmap e2: y(active, starting, since 0.00272089s) 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: Manager daemon y is now available 2023-10-09T03:48:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:48:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:48:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' 2023-10-09T03:48:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' 2023-10-09T03:48:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:19 smithi163 ceph-mon[101022]: from='mgr.14100 172.21.15.163:0/3376700087' entity='mgr.y' 2023-10-09T03:48:20.598 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsid": "8628d772-6656-11ee-8db5-212e2dc638e7", 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T03:48:20.599 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 0 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:20.600 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T03:48:20.601 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T03:48:20.601 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T03:48:20.604 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.605 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:20.606 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T03:48:06.379819+0000", 2023-10-09T03:48:20.607 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:20.607 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:20.607 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T03:48:20.607 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:20.607 INFO:teuthology.orchestra.run.smithi163.stdout:mgr not available, waiting (3/15)... 2023-10-09T03:48:21.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:20 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1921559647' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T03:48:21.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:20 smithi163 ceph-mon[101022]: mgrmap e3: y(active, since 1.00563s) 2023-10-09T03:48:23.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:23 smithi163 ceph-mon[101022]: mgrmap e4: y(active, since 2s) 2023-10-09T03:48:23.821 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:23.821 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:23.822 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsid": "8628d772-6656-11ee-8db5-212e2dc638e7", 2023-10-09T03:48:23.822 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T03:48:23.822 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T03:48:23.823 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T03:48:23.823 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T03:48:23.823 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.823 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 0 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:23.824 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T03:48:23.825 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T03:48:23.826 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T03:48:23.827 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ], 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T03:48:06.379819+0000", 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout }, 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:23.828 INFO:teuthology.orchestra.run.smithi163.stdout:mgr is available 2023-10-09T03:48:24.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:24 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3154493060' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T03:48:24.979 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:24.980 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout fsid = 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T03:48:24.981 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 2023-10-09T03:48:24.982 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T03:48:24.982 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T03:48:24.982 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T03:48:24.982 INFO:teuthology.orchestra.run.smithi163.stdout:Enabling cephadm module... 2023-10-09T03:48:25.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:25 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/69367293' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T03:48:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:26 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2025795692' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T03:48:26.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: ignoring --setuser ceph since I am not root 2023-10-09T03:48:26.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: ignoring --setgroup ceph since I am not root 2023-10-09T03:48:26.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: 2023-10-09T03:48:26.475+0000 7f710ce2a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:48:26.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: 2023-10-09T03:48:26.553+0000 7f710ce2a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:48:26.922 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: 2023-10-09T03:48:26.644+0000 7f710ce2a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:48:26.922 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: 2023-10-09T03:48:26.735+0000 7f710ce2a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:48:27.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:26 smithi163 conmon[101450]: 2023-10-09T03:48:26.921+0000 7f710ce2a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:48:27.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:27 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2025795692' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T03:48:27.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:27 smithi163 ceph-mon[101022]: mgrmap e5: y(active, since 6s) 2023-10-09T03:48:27.561 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:27 smithi163 conmon[101450]: 2023-10-09T03:48:27.244+0000 7f710ce2a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:48:27.562 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:27 smithi163 conmon[101450]: 2023-10-09T03:48:27.336+0000 7f710ce2a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:48:27.562 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:27 smithi163 conmon[101450]: 2023-10-09T03:48:27.467+0000 7f710ce2a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:48:27.562 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:27 smithi163 conmon[101450]: 2023-10-09T03:48:27.561+0000 7f710ce2a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:48:28.129 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:28.130 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T03:48:28.130 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T03:48:28.130 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T03:48:28.130 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T03:48:28.130 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:28.130 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for the mgr to restart... 2023-10-09T03:48:28.131 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for mgr epoch 5... 2023-10-09T03:48:28.317 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:27 smithi163 conmon[101450]: 2023-10-09T03:48:27.996+0000 7f710ce2a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:48:28.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:28 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4294104475' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T03:48:29.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:28 smithi163 conmon[101450]: 2023-10-09T03:48:28.595+0000 7f710ce2a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:48:29.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:28 smithi163 conmon[101450]: 2023-10-09T03:48:28.765+0000 7f710ce2a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:48:29.765 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:29 smithi163 conmon[101450]: 2023-10-09T03:48:29.513+0000 7f710ce2a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:48:29.765 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:29 smithi163 conmon[101450]: 2023-10-09T03:48:29.601+0000 7f710ce2a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:48:29.765 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:29 smithi163 conmon[101450]: 2023-10-09T03:48:29.765+0000 7f710ce2a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:48:30.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:30 smithi163 conmon[101450]: 2023-10-09T03:48:30.033+0000 7f710ce2a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:48:30.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:30 smithi163 conmon[101450]: 2023-10-09T03:48:30.118+0000 7f710ce2a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:48:30.676 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:30 smithi163 conmon[101450]: 2023-10-09T03:48:30.423+0000 7f710ce2a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:48:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:30 smithi163 conmon[101450]: 2023-10-09T03:48:30.676+0000 7f710ce2a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:48:31.678 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:31 smithi163 conmon[101450]: 2023-10-09T03:48:31.368+0000 7f710ce2a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:48:31.678 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:31 smithi163 conmon[101450]: 2023-10-09T03:48:31.495+0000 7f710ce2a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:48:31.996 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:31 smithi163 conmon[101450]: 2023-10-09T03:48:31.678+0000 7f710ce2a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:48:31.996 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:31 smithi163 conmon[101450]: 2023-10-09T03:48:31.789+0000 7f710ce2a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:48:31.997 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:31 smithi163 conmon[101450]: 2023-10-09T03:48:31.996+0000 7f710ce2a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:48:33.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:32 smithi163 conmon[101450]: 2023-10-09T03:48:32.834+0000 7f710ce2a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: Active manager daemon y restarted 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: Activating manager daemon y 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: mgrmap e6: y(active, starting, since 0.00559207s) 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:48:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:48:33.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: Manager daemon y is now available 2023-10-09T03:48:33.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:33.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:33.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:33.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:32 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:34.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:33 smithi163 ceph-mon[101022]: Found migration_current of "None". Setting to last migration. 2023-10-09T03:48:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:33 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:33 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:48:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:33 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:48:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:33 smithi163 ceph-mon[101022]: mgrmap e7: y(active, since 1.00498s) 2023-10-09T03:48:34.429 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:34.430 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T03:48:34.430 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T03:48:34.430 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:34.430 INFO:teuthology.orchestra.run.smithi163.stdout:mgr epoch 5 is available 2023-10-09T03:48:34.430 INFO:teuthology.orchestra.run.smithi163.stdout:Setting orchestrator backend to cephadm... 2023-10-09T03:48:35.267 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: [09/Oct/2023:03:48:33] ENGINE Bus STARTING 2023-10-09T03:48:35.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T03:48:35.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T03:48:35.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: [09/Oct/2023:03:48:33] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:48:35.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: [09/Oct/2023:03:48:33] ENGINE Bus STARTED 2023-10-09T03:48:35.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:35.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:34 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:36.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:35 smithi163 ceph-mon[101022]: mgrmap e8: y(active, since 2s) 2023-10-09T03:48:36.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:35 smithi163 ceph-mon[101022]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:36.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:35 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:36.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:35 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:36.760 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T03:48:36.761 INFO:teuthology.orchestra.run.smithi163.stdout:Generating ssh key... 2023-10-09T03:48:37.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:37 smithi163 ceph-mon[101022]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:37.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: Generating public/private rsa key pair. 2023-10-09T03:48:37.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: Your identification has been saved in /tmp/tmpa0cn8ce1/key. 2023-10-09T03:48:37.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: Your public key has been saved in /tmp/tmpa0cn8ce1/key.pub. 2023-10-09T03:48:37.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: The key fingerprint is: 2023-10-09T03:48:37.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: SHA256:1DJyoaF8x9DbqcA9PqM5EzFE1CUoQnxsmG6JQBHVtew ceph-8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: The key's randomart image is: 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: +---[RSA 3072]----+ 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: |.*==.o*=o.. | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: |. =.=o+=++ | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: |.o =o+++Bo.. | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: |. + .*=+oo | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: | . ESo | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: | . = | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: | + o | 2023-10-09T03:48:37.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: | = | 2023-10-09T03:48:37.584 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: | o | 2023-10-09T03:48:37.584 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:37 smithi163 conmon[101450]: +----[SHA256]-----+ 2023-10-09T03:48:38.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:38 smithi163 ceph-mon[101022]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:38.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:38 smithi163 ceph-mon[101022]: Generating ssh key... 2023-10-09T03:48:38.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:38 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:38.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:38 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:39.110 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuMN/MJ2D6sPKD3VttQeJlerntv57cxL2IWoQ6BQKj6QEYJnTajUHAzajDtawobzGy3A6DuqGhkNt/BnSxEOlawQAK7Mw9AaRz6h2b2iuDgpD/wLzMQNfXV79APwNDLgfJrOF3F9FF3iK3eUPQy6PyI6cekiTiVcdrjnJBuqZQUozPZ7Ph44Z3GBxBymIKVpVW+NNe3qB/YuiM+J8LjpX2vW0sJMalZZQYgBCYDB1E5uVrETcfbbQao/FIX4N4EDK7zLf8VugZrxPXL3KCZDV4Xun1ZGKMJ8RS94mhBioXzCd90+u1784zSInEER5+5mudsWx6i33jVNBqeeSRNHv4DUFbVi1HO47AzAaH1F+RRGUbwY9KFTVt7rMx6KH3/jA0f7weyNV7rRdknKfGudJ2IHddhPdYWUVVrrEWJh8LG++m5/5SDzGKOLMH/a8igdUlBb3yyy9YMMAoiuN9VaouV4SwC2DH3LLr7y8NXalyqCsNyPf0yYhuS++PtPE8Kv0= ceph-8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:48:39.111 INFO:teuthology.orchestra.run.smithi163.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T03:48:39.111 INFO:teuthology.orchestra.run.smithi163.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T03:48:39.112 INFO:teuthology.orchestra.run.smithi163.stdout:Adding host smithi163... 2023-10-09T03:48:39.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:39 smithi163 ceph-mon[101022]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:40.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:40 smithi163 ceph-mon[101022]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi163", "addr": "172.21.15.163", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:40.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:40 smithi163 ceph-mon[101022]: Deploying cephadm binary to smithi163 2023-10-09T03:48:42.043 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout Added host 'smithi163' with addr '172.21.15.163' 2023-10-09T03:48:42.043 INFO:teuthology.orchestra.run.smithi163.stdout:Deploying unmanaged mon service... 2023-10-09T03:48:42.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:42 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:42.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:42 smithi163 ceph-mon[101022]: Added host smithi163 2023-10-09T03:48:42.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:42 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:43.772 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T03:48:43.772 INFO:teuthology.orchestra.run.smithi163.stdout:Deploying unmanaged mgr service... 2023-10-09T03:48:43.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:43 smithi163 ceph-mon[101022]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:43.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:43 smithi163 ceph-mon[101022]: Saving service mon spec with placement count:5 2023-10-09T03:48:43.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:43 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:44.959 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T03:48:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:44 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:44 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:44 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:46.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:45 smithi163 ceph-mon[101022]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:45 smithi163 ceph-mon[101022]: Saving service mgr spec with placement count:2 2023-10-09T03:48:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:45 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1501218573' entity='client.admin' 2023-10-09T03:48:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:45 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:47.300 INFO:teuthology.orchestra.run.smithi163.stdout:Enabling the dashboard module... 2023-10-09T03:48:47.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:47 smithi163 ceph-mon[101022]: from='mgr.14120 172.21.15.163:0/3255649578' entity='mgr.y' 2023-10-09T03:48:47.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:47 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1380798464' entity='client.admin' 2023-10-09T03:48:48.662 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:48 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/746913219' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T03:48:48.982 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:48 smithi163 conmon[101450]: ignoring --setuser ceph since I am not root 2023-10-09T03:48:48.982 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:48 smithi163 conmon[101450]: ignoring --setgroup ceph since I am not root 2023-10-09T03:48:48.983 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:48 smithi163 conmon[101450]: 2023-10-09T03:48:48.804+0000 7ffbe85f41c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:48:48.983 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:48 smithi163 conmon[101450]: 2023-10-09T03:48:48.883+0000 7ffbe85f41c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:48:49.235 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:48 smithi163 conmon[101450]: 2023-10-09T03:48:48.970+0000 7ffbe85f41c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:48:49.235 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:49 smithi163 conmon[101450]: 2023-10-09T03:48:49.061+0000 7ffbe85f41c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:48:49.569 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:49 smithi163 conmon[101450]: 2023-10-09T03:48:49.234+0000 7ffbe85f41c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:48:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:49 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/746913219' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T03:48:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:49 smithi163 ceph-mon[101022]: mgrmap e9: y(active, since 15s) 2023-10-09T03:48:49.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:49 smithi163 conmon[101450]: 2023-10-09T03:48:49.568+0000 7ffbe85f41c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:48:49.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:49 smithi163 conmon[101450]: 2023-10-09T03:48:49.659+0000 7ffbe85f41c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:48:49.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:49 smithi163 conmon[101450]: 2023-10-09T03:48:49.801+0000 7ffbe85f41c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:48:50.157 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:49 smithi163 conmon[101450]: 2023-10-09T03:48:49.896+0000 7ffbe85f41c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:48:50.550 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:50.550 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T03:48:50.550 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T03:48:50.550 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T03:48:50.551 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T03:48:50.551 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:50.551 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for the mgr to restart... 2023-10-09T03:48:50.551 INFO:teuthology.orchestra.run.smithi163.stdout:Waiting for mgr epoch 9... 2023-10-09T03:48:50.557 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:50 smithi163 conmon[101450]: 2023-10-09T03:48:50.337+0000 7ffbe85f41c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:48:50.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:50 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1163347960' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T03:48:51.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:50 smithi163 conmon[101450]: 2023-10-09T03:48:50.934+0000 7ffbe85f41c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:48:51.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:51 smithi163 conmon[101450]: 2023-10-09T03:48:51.105+0000 7ffbe85f41c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:48:52.118 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:51 smithi163 conmon[101450]: 2023-10-09T03:48:51.860+0000 7ffbe85f41c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:48:52.119 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:51 smithi163 conmon[101450]: 2023-10-09T03:48:51.950+0000 7ffbe85f41c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:48:52.119 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:52 smithi163 conmon[101450]: 2023-10-09T03:48:52.117+0000 7ffbe85f41c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:48:52.764 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:52 smithi163 conmon[101450]: 2023-10-09T03:48:52.390+0000 7ffbe85f41c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:48:52.765 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:52 smithi163 conmon[101450]: 2023-10-09T03:48:52.475+0000 7ffbe85f41c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:48:53.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:52 smithi163 conmon[101450]: 2023-10-09T03:48:52.763+0000 7ffbe85f41c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:48:53.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:53 smithi163 conmon[101450]: 2023-10-09T03:48:53.017+0000 7ffbe85f41c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:48:54.035 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:53 smithi163 conmon[101450]: 2023-10-09T03:48:53.720+0000 7ffbe85f41c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:48:54.035 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:53 smithi163 conmon[101450]: 2023-10-09T03:48:53.847+0000 7ffbe85f41c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:48:54.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:54 smithi163 conmon[101450]: 2023-10-09T03:48:54.033+0000 7ffbe85f41c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:48:54.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:54 smithi163 conmon[101450]: 2023-10-09T03:48:54.145+0000 7ffbe85f41c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:48:54.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:54 smithi163 conmon[101450]: 2023-10-09T03:48:54.356+0000 7ffbe85f41c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:48:55 smithi163 conmon[101450]: 2023-10-09T03:48:55.201+0000 7ffbe85f41c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: Active manager daemon y restarted 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: Activating manager daemon y 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: mgrmap e10: y(active, starting, since 0.00307717s) 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:48:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:48:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:48:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:48:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:48:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: Manager daemon y is now available 2023-10-09T03:48:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:55 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:48:56.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:48:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:48:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:56 smithi163 ceph-mon[101022]: mgrmap e11: y(active, since 1.00535s) 2023-10-09T03:48:56.807 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout { 2023-10-09T03:48:56.808 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T03:48:56.808 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T03:48:56.808 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout } 2023-10-09T03:48:56.808 INFO:teuthology.orchestra.run.smithi163.stdout:mgr epoch 9 is available 2023-10-09T03:48:56.808 INFO:teuthology.orchestra.run.smithi163.stdout:Generating a dashboard self-signed certificate... 2023-10-09T03:48:57.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:57 smithi163 ceph-mon[101022]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T03:48:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:57 smithi163 ceph-mon[101022]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T03:48:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:48:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:48:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:48:58.282 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T03:48:58.282 INFO:teuthology.orchestra.run.smithi163.stdout:Creating initial admin user... 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: [09/Oct/2023:03:48:56] ENGINE Bus STARTING 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: [09/Oct/2023:03:48:56] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: [09/Oct/2023:03:48:56] ENGINE Bus STARTED 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:48:58.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:48:58 smithi163 ceph-mon[101022]: mgrmap e12: y(active, since 2s) 2023-10-09T03:48:59.831 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Do.S8x0sgcjwfFKJoHEtFeix38W8WfyaQMBe01mkLS4sbWOlI2h/G", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696823339, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T03:48:59.832 INFO:teuthology.orchestra.run.smithi163.stdout:Fetching dashboard port number... 2023-10-09T03:49:00.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:00 smithi163 ceph-mon[101022]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:00.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:00 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:01.023 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T03:49:01.034 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T03:49:01.034 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout disabled 2023-10-09T03:49:01.041 INFO:teuthology.orchestra.run.smithi163.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T03:49:01.041 INFO:teuthology.orchestra.run.smithi163.stdout:systemctl: stdout inactive 2023-10-09T03:49:01.042 INFO:teuthology.orchestra.run.smithi163.stdout:firewalld.service is not enabled 2023-10-09T03:49:01.042 INFO:teuthology.orchestra.run.smithi163.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T03:49:01.043 INFO:teuthology.orchestra.run.smithi163.stdout:Ceph Dashboard is now available at: 2023-10-09T03:49:01.043 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:01.043 INFO:teuthology.orchestra.run.smithi163.stdout: URL: https://smithi163:8443/ 2023-10-09T03:49:01.043 INFO:teuthology.orchestra.run.smithi163.stdout: User: admin 2023-10-09T03:49:01.043 INFO:teuthology.orchestra.run.smithi163.stdout: Password: woxsbtlpuf 2023-10-09T03:49:01.043 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:01.044 INFO:teuthology.orchestra.run.smithi163.stdout:Saving cluster configuration to /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config directory 2023-10-09T03:49:01.044 INFO:teuthology.orchestra.run.smithi163.stdout:Enabling autotune for osd_memory_target 2023-10-09T03:49:01.670 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:01 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/544103231' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T03:49:02.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:02 smithi163 ceph-mon[101022]: mgrmap e13: y(active, since 6s) 2023-10-09T03:49:03.473 INFO:teuthology.orchestra.run.smithi163.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T03:49:03.473 INFO:teuthology.orchestra.run.smithi163.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: ceph telemetry on 2023-10-09T03:49:03.474 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.475 INFO:teuthology.orchestra.run.smithi163.stdout:For more information see: 2023-10-09T03:49:03.475 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.475 INFO:teuthology.orchestra.run.smithi163.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-10-09T03:49:03.475 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:03.475 INFO:teuthology.orchestra.run.smithi163.stdout:Bootstrap complete. 2023-10-09T03:49:03.513 INFO:tasks.cephadm:Fetching config... 2023-10-09T03:49:03.513 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:03.514 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T03:49:03.531 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T03:49:03.531 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:03.531 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T03:49:03.588 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T03:49:03.589 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:03.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T03:49:03.657 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T03:49:03.657 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:03.657 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T03:49:03.714 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T03:49:03.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuMN/MJ2D6sPKD3VttQeJlerntv57cxL2IWoQ6BQKj6QEYJnTajUHAzajDtawobzGy3A6DuqGhkNt/BnSxEOlawQAK7Mw9AaRz6h2b2iuDgpD/wLzMQNfXV79APwNDLgfJrOF3F9FF3iK3eUPQy6PyI6cekiTiVcdrjnJBuqZQUozPZ7Ph44Z3GBxBymIKVpVW+NNe3qB/YuiM+J8LjpX2vW0sJMalZZQYgBCYDB1E5uVrETcfbbQao/FIX4N4EDK7zLf8VugZrxPXL3KCZDV4Xun1ZGKMJ8RS94mhBioXzCd90+u1784zSInEER5+5mudsWx6i33jVNBqeeSRNHv4DUFbVi1HO47AzAaH1F+RRGUbwY9KFTVt7rMx6KH3/jA0f7weyNV7rRdknKfGudJ2IHddhPdYWUVVrrEWJh8LG++m5/5SDzGKOLMH/a8igdUlBb3yyy9YMMAoiuN9VaouV4SwC2DH3LLr7y8NXalyqCsNyPf0yYhuS++PtPE8Kv0= ceph-8628d772-6656-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T03:49:03.795 INFO:teuthology.orchestra.run.smithi163.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuMN/MJ2D6sPKD3VttQeJlerntv57cxL2IWoQ6BQKj6QEYJnTajUHAzajDtawobzGy3A6DuqGhkNt/BnSxEOlawQAK7Mw9AaRz6h2b2iuDgpD/wLzMQNfXV79APwNDLgfJrOF3F9FF3iK3eUPQy6PyI6cekiTiVcdrjnJBuqZQUozPZ7Ph44Z3GBxBymIKVpVW+NNe3qB/YuiM+J8LjpX2vW0sJMalZZQYgBCYDB1E5uVrETcfbbQao/FIX4N4EDK7zLf8VugZrxPXL3KCZDV4Xun1ZGKMJ8RS94mhBioXzCd90+u1784zSInEER5+5mudsWx6i33jVNBqeeSRNHv4DUFbVi1HO47AzAaH1F+RRGUbwY9KFTVt7rMx6KH3/jA0f7weyNV7rRdknKfGudJ2IHddhPdYWUVVrrEWJh8LG++m5/5SDzGKOLMH/a8igdUlBb3yyy9YMMAoiuN9VaouV4SwC2DH3LLr7y8NXalyqCsNyPf0yYhuS++PtPE8Kv0= ceph-8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:03.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuMN/MJ2D6sPKD3VttQeJlerntv57cxL2IWoQ6BQKj6QEYJnTajUHAzajDtawobzGy3A6DuqGhkNt/BnSxEOlawQAK7Mw9AaRz6h2b2iuDgpD/wLzMQNfXV79APwNDLgfJrOF3F9FF3iK3eUPQy6PyI6cekiTiVcdrjnJBuqZQUozPZ7Ph44Z3GBxBymIKVpVW+NNe3qB/YuiM+J8LjpX2vW0sJMalZZQYgBCYDB1E5uVrETcfbbQao/FIX4N4EDK7zLf8VugZrxPXL3KCZDV4Xun1ZGKMJ8RS94mhBioXzCd90+u1784zSInEER5+5mudsWx6i33jVNBqeeSRNHv4DUFbVi1HO47AzAaH1F+RRGUbwY9KFTVt7rMx6KH3/jA0f7weyNV7rRdknKfGudJ2IHddhPdYWUVVrrEWJh8LG++m5/5SDzGKOLMH/a8igdUlBb3yyy9YMMAoiuN9VaouV4SwC2DH3LLr7y8NXalyqCsNyPf0yYhuS++PtPE8Kv0= ceph-8628d772-6656-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T03:49:03.852 INFO:teuthology.orchestra.run.smithi181.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuMN/MJ2D6sPKD3VttQeJlerntv57cxL2IWoQ6BQKj6QEYJnTajUHAzajDtawobzGy3A6DuqGhkNt/BnSxEOlawQAK7Mw9AaRz6h2b2iuDgpD/wLzMQNfXV79APwNDLgfJrOF3F9FF3iK3eUPQy6PyI6cekiTiVcdrjnJBuqZQUozPZ7Ph44Z3GBxBymIKVpVW+NNe3qB/YuiM+J8LjpX2vW0sJMalZZQYgBCYDB1E5uVrETcfbbQao/FIX4N4EDK7zLf8VugZrxPXL3KCZDV4Xun1ZGKMJ8RS94mhBioXzCd90+u1784zSInEER5+5mudsWx6i33jVNBqeeSRNHv4DUFbVi1HO47AzAaH1F+RRGUbwY9KFTVt7rMx6KH3/jA0f7weyNV7rRdknKfGudJ2IHddhPdYWUVVrrEWJh8LG++m5/5SDzGKOLMH/a8igdUlBb3yyy9YMMAoiuN9VaouV4SwC2DH3LLr7y8NXalyqCsNyPf0yYhuS++PtPE8Kv0= ceph-8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:03.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T03:49:04.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:03 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1513803466' entity='client.admin' 2023-10-09T03:49:04.208 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:07.152 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T03:49:07.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T03:49:07.607 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:07.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:07 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2499118316' entity='client.admin' 2023-10-09T03:49:08.763 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:08.763 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:08.763 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi163", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:49:08.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:08.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:08.764 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:09.846 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi181 2023-10-09T03:49:09.846 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:49:09.846 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.conf 2023-10-09T03:49:09.867 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:49:09.867 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:49:09.926 INFO:tasks.cephadm:Adding host smithi181 to orchestrator... 2023-10-09T03:49:09.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi181 2023-10-09T03:49:10.289 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:10.462 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:10.462 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:10.462 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:10.462 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:10.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:10.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:49:10.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:10 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:11.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:11 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:49:11.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:11 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:49:11.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:11.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:11.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:13.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:12 smithi163 ceph-mon[101022]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:13.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:12 smithi163 ceph-mon[101022]: Deploying cephadm binary to smithi181 2023-10-09T03:49:13.598 INFO:teuthology.orchestra.run.smithi163.stdout:Added host 'smithi181' with addr '172.21.15.181' 2023-10-09T03:49:14.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T03:49:14.539 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:14.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:14.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:14 smithi163 ceph-mon[101022]: Added host smithi181 2023-10-09T03:49:14.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:14.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:14.893 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:16.059 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:49:16.059 INFO:teuthology.orchestra.run.smithi163.stdout:[{"addr": "172.21.15.163", "hostname": "smithi163", "labels": [], "status": ""}, {"addr": "172.21.15.181", "hostname": "smithi181", "labels": [], "status": ""}] 2023-10-09T03:49:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:16 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:16 smithi163 ceph-mon[101022]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:16.609 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T03:49:16.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T03:49:16.942 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:17.461 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:17 smithi163 ceph-mon[101022]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:19.330 INFO:teuthology.orchestra.run.smithi163.stderr:adjusted tunables profile to default 2023-10-09T03:49:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:49:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3296709132' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T03:49:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:49:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:19.893 INFO:tasks.cephadm:Adding mon.a on smithi163 2023-10-09T03:49:19.893 INFO:tasks.cephadm:Adding mon.c on smithi163 2023-10-09T03:49:19.893 INFO:tasks.cephadm:Adding mon.b on smithi181 2023-10-09T03:49:19.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b' 2023-10-09T03:49:20.263 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:20 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:49:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:20 smithi163 ceph-mon[101022]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:20 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3296709132' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T03:49:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:20 smithi163 ceph-mon[101022]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:21.792 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mon update... 2023-10-09T03:49:22.385 DEBUG:teuthology.orchestra.run.smithi163:mon.c> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T03:49:22.388 DEBUG:teuthology.orchestra.run.smithi181:mon.b> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T03:49:22.390 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T03:49:22.391 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T03:49:22.420 INFO:journalctl@ceph.mon.b.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:49:22.424 INFO:journalctl@ceph.mon.c.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:23.369 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:23.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:23 smithi163 ceph-mon[101022]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:23.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:23 smithi163 ceph-mon[101022]: Saving service mon spec with placement smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b;count:3 2023-10-09T03:49:23.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:23 smithi163 ceph-mon[101022]: Deploying daemon mon.b on smithi181 2023-10-09T03:49:24.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:24 smithi163 ceph-mon[101022]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:25.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 systemd[1]: Starting Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:49:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 bash[105290]: 4a4d2d630200392dac4ea0b9e22e30faf7f1ccb21a2e2456db2652f26aef0657 2023-10-09T03:49:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 systemd[1]: Started Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:49:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T03:49:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T03:49:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: pidfile_write: ignore empty --pid-file 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: load: jerasure load: lrc 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: RocksDB version: 6.15.5 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Compile date Oct 8 2023 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: DB SUMMARY 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: DB Session ID: VMG83BYNDTM4XF0QAFNM 2023-10-09T03:49:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: CURRENT file: CURRENT 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.error_if_exists: 0 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.create_if_missing: 0 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.paranoid_checks: 1 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.env: 0x55c5a0540a80 2023-10-09T03:49:26.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.fs: Posix File System 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.info_log: 0x55c5a11d65a0 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.statistics: (nil) 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.use_fsync: 0 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_log_file_size: 0 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.allow_fallocate: 1 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T03:49:26.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T03:49:26.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.use_direct_reads: 0 2023-10-09T03:49:26.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T03:49:26.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T03:49:26.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.db_log_dir: 2023-10-09T03:49:26.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T03:49:26.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.write_buffer_manager: 0x55c5a1f48900 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T03:49:26.026 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.unordered_write: 0 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.row_cache: None 2023-10-09T03:49:26.027 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.wal_filter: None 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.preserve_deletes: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.two_write_queues: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.atomic_flush: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T03:49:26.028 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.log_readahead_size: 0 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_background_jobs: 2 2023-10-09T03:49:26.029 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_background_compactions: -1 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_subcompactions: 1 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T03:49:26.030 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_open_files: -1 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_background_flushes: -1 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Compression algorithms supported: 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T03:49:26.031 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kZSTD supported: 0 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kXpressCompression supported: 0 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kZlibCompression supported: 1 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kSnappyCompression supported: 1 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kLZ4Compression supported: 1 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: kBZip2Compression supported: 0 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.merge_operator: 2023-10-09T03:49:26.032 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_filter: None 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c5a1228fb8) 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: cache_index_and_filter_blocks: 1 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: pin_top_level_index_and_filter: 1 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: index_type: 0 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: data_block_index_type: 0 2023-10-09T03:49:26.033 INFO:journalctl@ceph.mon.b.smithi181.stdout: index_shortening: 1 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: hash_index_allow_collision: 1 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: checksum: 1 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: no_block_cache: 0 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache: 0x55c5a125b090 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache_name: BinnedLRUCache 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache_options: 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: capacity : 536870912 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: num_shard_bits : 4 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: strict_capacity_limit : 0 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: high_pri_pool_ratio: 0.000 2023-10-09T03:49:26.034 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache_compressed: (nil) 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: persistent_cache: (nil) 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_size: 4096 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_size_deviation: 10 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_restart_interval: 16 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: index_block_restart_interval: 1 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: metadata_block_size: 4096 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: partition_filters: 0 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: use_delta_encoding: 1 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: whole_key_filtering: 1 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: verify_compression: 0 2023-10-09T03:49:26.035 INFO:journalctl@ceph.mon.b.smithi181.stdout: read_amp_bytes_per_bit: 0 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout: format_version: 4 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout: enable_index_compression: 1 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_align: 0 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression: NoCompression 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.num_levels: 7 2023-10-09T03:49:26.036 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T03:49:26.037 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T03:49:26.038 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T03:49:26.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T03:49:26.040 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.table_properties_collectors: 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.inplace_update_support: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.bloom_locality: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.max_successive_merges: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T03:49:26.041 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.ttl: 2592000 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.enable_blob_files: false 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.min_blob_size: 0 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T03:49:26.042 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823365682964, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823365683229, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696823365, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "65c8ac4d-9284-4af0-95ca-64d0c9984012", "db_session_id": "VMG83BYNDTM4XF0QAFNM"}} 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823365683553, "job": 1, "event": "recovery_finished"} 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c5a11ef180 2023-10-09T03:49:26.043 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: DB pointer 0x55c5a12c4000 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** DB Stats ** 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.044 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.045 INFO:journalctl@ceph.mon.b.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.046 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:49:26.047 INFO:journalctl@ceph.mon.b.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T03:49:26.048 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: using public_addr v2:172.21.15.181:0/0 -> [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: starting mon.b rank -1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(???) e0 preinit fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).mds e1 new map 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout: e1 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout: legacy client fscid: -1 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout: No filesystems configured 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T03:49:26.049 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3296709132' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:26.050 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: Saving service mon spec with placement smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b;count:3 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: Deploying daemon mon.b on smithi181 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:26.051 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T03:49:26.052 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:25 smithi181 ceph-mon[105375]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-10-09T03:49:30.761 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:49:30.762 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":2,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","modified":"2023-10-09T03:49:25.704789Z","created":"2023-10-09T03:48:03.831950Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T03:49:30.765 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 2 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: mon.a calling monitor election 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/1934667979' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: mon.b calling monitor election 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: fsmap 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: mgrmap e13: y(active, since 35s) 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: overall HEALTH_OK 2023-10-09T03:49:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:31.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:31.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:31.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:31.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: mon.a calling monitor election 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/1934667979' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: mon.b calling monitor election 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: fsmap 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: mgrmap e13: y(active, since 35s) 2023-10-09T03:49:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: overall HEALTH_OK 2023-10-09T03:49:31.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:31.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:31.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:31.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:31.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:32.384 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:32 smithi163 ceph-mon[101022]: Deploying daemon mon.c on smithi163 2023-10-09T03:49:32.385 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:32 smithi163 ceph-mon[101022]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:32.385 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:32.407 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T03:49:32.407 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T03:49:32.750 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:49:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:32 smithi181 ceph-mon[105375]: Deploying daemon mon.c on smithi163 2023-10-09T03:49:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:32 smithi181 ceph-mon[105375]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:33.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 systemd[1]: Starting Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:49:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T03:49:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T03:49:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pidfile_write: ignore empty --pid-file 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: load: jerasure load: lrc 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: RocksDB version: 6.15.5 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Compile date Oct 8 2023 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: DB SUMMARY 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: DB Session ID: 15WK827XW0YZZJJEWB3I 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: CURRENT file: CURRENT 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T03:49:34.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.error_if_exists: 0 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.create_if_missing: 0 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.paranoid_checks: 1 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.env: 0x55c6b3090a80 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.fs: Posix File System 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.info_log: 0x55c6b542e5a0 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.statistics: (nil) 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.use_fsync: 0 2023-10-09T03:49:34.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_log_file_size: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.allow_fallocate: 1 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.use_direct_reads: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T03:49:34.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.db_log_dir: 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.write_buffer_manager: 0x55c6b61a0930 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T03:49:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.unordered_write: 0 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T03:49:34.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.row_cache: None 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.wal_filter: None 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.preserve_deletes: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.two_write_queues: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.atomic_flush: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T03:49:34.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.log_readahead_size: 0 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_background_jobs: 2 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_background_compactions: -1 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_subcompactions: 1 2023-10-09T03:49:34.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_open_files: -1 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T03:49:34.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_background_flushes: -1 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Compression algorithms supported: 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kZSTD supported: 0 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kXpressCompression supported: 0 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kZlibCompression supported: 1 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kSnappyCompression supported: 1 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kLZ4Compression supported: 1 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: kBZip2Compression supported: 0 2023-10-09T03:49:34.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.merge_operator: 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_filter: None 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c6b5480fb8) 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout: cache_index_and_filter_blocks: 1 2023-10-09T03:49:34.342 INFO:journalctl@ceph.mon.c.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: pin_top_level_index_and_filter: 1 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_type: 0 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_block_index_type: 0 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_shortening: 1 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: hash_index_allow_collision: 1 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: checksum: 1 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: no_block_cache: 0 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache: 0x55c6b54b3090 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_name: BinnedLRUCache 2023-10-09T03:49:34.343 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_options: 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: capacity : 536870912 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: num_shard_bits : 4 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: strict_capacity_limit : 0 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: high_pri_pool_ratio: 0.000 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_compressed: (nil) 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: persistent_cache: (nil) 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_size: 4096 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_size_deviation: 10 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_restart_interval: 16 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_block_restart_interval: 1 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: metadata_block_size: 4096 2023-10-09T03:49:34.344 INFO:journalctl@ceph.mon.c.smithi163.stdout: partition_filters: 0 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: use_delta_encoding: 1 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: whole_key_filtering: 1 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: verify_compression: 0 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: read_amp_bytes_per_bit: 0 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: format_version: 4 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: enable_index_compression: 1 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_align: 0 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression: NoCompression 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T03:49:34.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.num_levels: 7 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:49:34.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T03:49:34.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T03:49:34.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T03:49:34.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T03:49:34.349 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T03:49:34.350 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.table_properties_collectors: 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.inplace_update_support: 0 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T03:49:34.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.bloom_locality: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.max_successive_merges: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.ttl: 2592000 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.enable_blob_files: false 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.min_blob_size: 0 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T03:49:34.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823373858748, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823373859014, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696823373, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1abff181-15c2-414e-a468-41b0b1c2d725", "db_session_id": "15WK827XW0YZZJJEWB3I"}} 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T03:49:34.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823373859426, "job": 1, "event": "recovery_finished"} 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c6b5447180 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: DB pointer 0x55c6b551c000 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** DB Stats ** 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:49:34.354 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.355 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:34.356 INFO:journalctl@ceph.mon.c.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:49:34.357 INFO:journalctl@ceph.mon.c.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.4 0.00 0.00 1 0.000 0 0 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:49:34.358 INFO:journalctl@ceph.mon.c.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: using public_addrv [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: starting mon.c rank -1 at public addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] at bind addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(???) e0 preinit fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 bash[118742]: 72ef10caabbbdc9cddc0f33ff941648df08b26db51151a9480a9d693d45a66dd 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 systemd[1]: Started Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).mds e1 new map 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout: e1 2023-10-09T03:49:34.359 INFO:journalctl@ceph.mon.c.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout: legacy client fscid: -1 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout: No filesystems configured 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T03:49:34.360 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3296709132' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:34.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: Saving service mon spec with placement smithi163:172.21.15.163=a;smithi163:[v2:172.21.15.163:3301,v1:172.21.15.163:6790]=c;smithi181:172.21.15.181=b;count:3 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: Deploying daemon mon.b on smithi181 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.a calling monitor election 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/1934667979' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T03:49:34.362 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.b calling monitor election 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: monmap e2: 2 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: fsmap 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:34.363 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mgrmap e13: y(active, since 35s) 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: overall HEALTH_OK 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: Deploying daemon mon.c on smithi163 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T03:49:34.364 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:33 smithi163 ceph-mon[118807]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-10-09T03:49:39.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: mon.a calling monitor election 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: mon.b calling monitor election 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: mon.c calling monitor election 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: fsmap 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: mgrmap e13: y(active, since 43s) 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: overall HEALTH_OK 2023-10-09T03:49:39.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:39.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: mon.a calling monitor election 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: mon.b calling monitor election 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: mon.c calling monitor election 2023-10-09T03:49:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: fsmap 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: mgrmap e13: y(active, since 43s) 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: overall HEALTH_OK 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: mon.a calling monitor election 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: mon.b calling monitor election 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: mon.c calling monitor election 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: fsmap 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: mgrmap e13: y(active, since 43s) 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: overall HEALTH_OK 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:39.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:39 smithi181 ceph-mon[105375]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:40.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[118807]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[101022]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:49:40.336 INFO:teuthology.orchestra.run.smithi181.stdout: 2023-10-09T03:49:40.336 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":3,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","modified":"2023-10-09T03:49:33.883566Z","created":"2023-10-09T03:48:03.831950Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3300","nonce":0},{"type":"v1","addr":"172.21.15.163:6789","nonce":0}]},"addr":"172.21.15.163:6789/0","public_addr":"172.21.15.163:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:3301","nonce":0},{"type":"v1","addr":"172.21.15.163:6790","nonce":0}]},"addr":"172.21.15.163:6790/0","public_addr":"172.21.15.163:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T03:49:40.338 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 3 2023-10-09T03:49:40.974 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T03:49:40.974 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T03:49:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/4057822808' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:41.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/4057822808' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:41.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/4057822808' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:41.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:41.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:41.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:42.087 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: Reconfiguring daemon mon.a on smithi163 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:42.354 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: Reconfiguring daemon mon.a on smithi163 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:42.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: Reconfiguring daemon mon.a on smithi163 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:43.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:43 smithi163 ceph-mon[101022]: Reconfiguring mon.c (monmap changed)... 2023-10-09T03:49:43.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:43 smithi163 ceph-mon[101022]: Reconfiguring daemon mon.c on smithi163 2023-10-09T03:49:43.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:43 smithi163 ceph-mon[118807]: Reconfiguring mon.c (monmap changed)... 2023-10-09T03:49:43.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:43 smithi163 ceph-mon[118807]: Reconfiguring daemon mon.c on smithi163 2023-10-09T03:49:43.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:43 smithi181 ceph-mon[105375]: Reconfiguring mon.c (monmap changed)... 2023-10-09T03:49:43.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:43 smithi181 ceph-mon[105375]: Reconfiguring daemon mon.c on smithi163 2023-10-09T03:49:43.855 INFO:teuthology.orchestra.run.smithi163.stdout:# minimal ceph.conf for 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:43.855 INFO:teuthology.orchestra.run.smithi163.stdout:[global] 2023-10-09T03:49:43.855 INFO:teuthology.orchestra.run.smithi163.stdout: fsid = 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:49:43.855 INFO:teuthology.orchestra.run.smithi163.stdout: mon_host = [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] 2023-10-09T03:49:44.357 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T03:49:44.358 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:44.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T03:49:44.388 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:44.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:49:44.454 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:49:44.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T03:49:44.487 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:49:44.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:49:44.553 INFO:tasks.cephadm:Adding mgr.y on smithi163 2023-10-09T03:49:44.554 INFO:tasks.cephadm:Adding mgr.x on smithi181 2023-10-09T03:49:44.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi163=y;smithi181=x' 2023-10-09T03:49:44.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:44 smithi163 ceph-mon[118807]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:44.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:44 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4118782547' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:44 smithi163 ceph-mon[101022]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:44 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4118782547' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:44.618 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:44 smithi181 ceph-mon[105375]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:44.618 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:44 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4118782547' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:45.477 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:49:45.649 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:45.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:45.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: Reconfiguring mon.b (monmap changed)... 2023-10-09T03:49:45.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:45.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:45.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:45.650 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:45 smithi181 ceph-mon[105375]: Reconfiguring daemon mon.b on smithi181 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: Reconfiguring mon.b (monmap changed)... 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[101022]: Reconfiguring daemon mon.b on smithi181 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: Reconfiguring mon.b (monmap changed)... 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:45 smithi163 ceph-mon[118807]: Reconfiguring daemon mon.b on smithi181 2023-10-09T03:49:46.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:46.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:46.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:46.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:46.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:46 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:46.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:46 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:47.003 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mgr update... 2023-10-09T03:49:47.579 DEBUG:teuthology.orchestra.run.smithi181:mgr.x> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T03:49:47.621 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T03:49:47.621 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:49:47.621 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T03:49:47.643 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T03:49:47.643 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_1 2023-10-09T03:49:47.649 INFO:journalctl@ceph.mgr.x.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 6h/6d Inode: 392061 Links: 1 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 03:49:33.275796798 +0000 2023-10-09T03:49:47.703 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 03:41:24.002482460 +0000 2023-10-09T03:49:47.704 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 03:41:24.002482460 +0000 2023-10-09T03:49:47.704 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T03:49:47.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T03:49:47.773 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T03:49:47.773 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T03:49:47.773 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000282452 s, 1.8 MB/s 2023-10-09T03:49:47.774 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T03:49:47.842 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_2 2023-10-09T03:49:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T03:49:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:47.905 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 6h/6d Inode: 396290 Links: 1 2023-10-09T03:49:47.906 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:47.906 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:47.906 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 03:49:33.275796798 +0000 2023-10-09T03:49:47.906 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 03:41:24.508470984 +0000 2023-10-09T03:49:47.906 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 03:41:24.508470984 +0000 2023-10-09T03:49:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T03:49:47.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T03:49:47.978 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T03:49:47.978 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T03:49:47.978 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000220717 s, 2.3 MB/s 2023-10-09T03:49:47.979 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T03:49:48.043 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_3 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: Saving service mgr spec with placement smithi163=y;smithi181=x;count:2 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: Deploying daemon mgr.x on smithi181 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[118807]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:48.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: Saving service mgr spec with placement smithi163=y;smithi181=x;count:2 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: Deploying daemon mgr.x on smithi181 2023-10-09T03:49:48.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:48 smithi163 ceph-mon[101022]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:48.103 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T03:49:48.103 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:48.103 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 6h/6d Inode: 392626 Links: 1 2023-10-09T03:49:48.103 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:48.103 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:48.103 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 03:49:33.275796798 +0000 2023-10-09T03:49:48.104 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 03:41:25.011459576 +0000 2023-10-09T03:49:48.104 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 03:41:25.011459576 +0000 2023-10-09T03:49:48.104 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T03:49:48.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T03:49:48.182 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T03:49:48.182 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T03:49:48.183 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000272817 s, 1.9 MB/s 2023-10-09T03:49:48.184 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T03:49:48.247 DEBUG:teuthology.orchestra.run.smithi163:> stat /dev/vg_nvme/lv_4 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi163=y;smithi181=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: Saving service mgr spec with placement smithi163=y;smithi181=x;count:2 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:49:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T03:49:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:49:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: Deploying daemon mgr.x on smithi181 2023-10-09T03:49:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:48 smithi181 ceph-mon[105375]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:48.309 INFO:teuthology.orchestra.run.smithi163.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T03:49:48.309 INFO:teuthology.orchestra.run.smithi163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:48.309 INFO:teuthology.orchestra.run.smithi163.stdout:Device: 6h/6d Inode: 395535 Links: 1 2023-10-09T03:49:48.309 INFO:teuthology.orchestra.run.smithi163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:48.309 INFO:teuthology.orchestra.run.smithi163.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:48.309 INFO:teuthology.orchestra.run.smithi163.stdout:Access: 2023-10-09 03:49:33.275796798 +0000 2023-10-09T03:49:48.310 INFO:teuthology.orchestra.run.smithi163.stdout:Modify: 2023-10-09 03:41:25.540447578 +0000 2023-10-09T03:49:48.310 INFO:teuthology.orchestra.run.smithi163.stdout:Change: 2023-10-09 03:41:25.540447578 +0000 2023-10-09T03:49:48.310 INFO:teuthology.orchestra.run.smithi163.stdout: Birth: - 2023-10-09T03:49:48.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T03:49:48.379 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records in 2023-10-09T03:49:48.379 INFO:teuthology.orchestra.run.smithi163.stderr:1+0 records out 2023-10-09T03:49:48.379 INFO:teuthology.orchestra.run.smithi163.stderr:512 bytes copied, 0.000178821 s, 2.9 MB/s 2023-10-09T03:49:48.381 DEBUG:teuthology.orchestra.run.smithi163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T03:49:48.445 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:49:48.445 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T03:49:48.474 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T03:49:48.474 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_1 2023-10-09T03:49:48.539 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T03:49:48.539 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:48.539 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 6h/6d Inode: 385947 Links: 1 2023-10-09T03:49:48.539 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:48.539 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:48.539 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 03:49:48.498544188 +0000 2023-10-09T03:49:48.540 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 03:41:10.110008821 +0000 2023-10-09T03:49:48.540 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 03:41:10.110008821 +0000 2023-10-09T03:49:48.540 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T03:49:48.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T03:49:48.681 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T03:49:48.681 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T03:49:48.681 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000249567 s, 2.1 MB/s 2023-10-09T03:49:48.683 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T03:49:48.702 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_2 2023-10-09T03:49:48.764 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T03:49:48.764 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:48.764 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 6h/6d Inode: 392672 Links: 1 2023-10-09T03:49:48.764 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:48.765 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:48.765 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 03:49:48.633541653 +0000 2023-10-09T03:49:48.765 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 03:41:10.619999132 +0000 2023-10-09T03:49:48.765 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 03:41:10.619999132 +0000 2023-10-09T03:49:48.765 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T03:49:48.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T03:49:48.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:48 smithi181 systemd[1]: Starting Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:49:48.796 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T03:49:48.796 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T03:49:48.796 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000233725 s, 2.2 MB/s 2023-10-09T03:49:48.798 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T03:49:48.858 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_3 2023-10-09T03:49:48.916 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T03:49:48.916 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:48.916 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 6h/6d Inode: 395408 Links: 1 2023-10-09T03:49:48.916 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:48.916 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:48.917 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 03:49:48.633541653 +0000 2023-10-09T03:49:48.917 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 03:41:11.118989653 +0000 2023-10-09T03:49:48.917 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 03:41:11.118989653 +0000 2023-10-09T03:49:48.917 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T03:49:48.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T03:49:48.983 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T03:49:48.983 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T03:49:48.983 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000208903 s, 2.5 MB/s 2023-10-09T03:49:48.984 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T03:49:49.041 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_4 2023-10-09T03:49:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T03:49:49.098 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T03:49:49.098 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 6h/6d Inode: 396420 Links: 1 2023-10-09T03:49:49.098 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T03:49:49.098 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T03:49:49.098 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2023-10-09 03:49:48.633541653 +0000 2023-10-09T03:49:49.099 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2023-10-09 03:41:11.640979737 +0000 2023-10-09T03:49:49.099 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2023-10-09 03:41:11.640979737 +0000 2023-10-09T03:49:49.099 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: - 2023-10-09T03:49:49.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T03:49:49.164 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2023-10-09T03:49:49.164 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2023-10-09T03:49:49.164 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000199352 s, 2.6 MB/s 2023-10-09T03:49:49.165 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T03:49:49.222 INFO:tasks.cephadm:Deploying osd.0 on smithi163 with /dev/vg_nvme/lv_4... 2023-10-09T03:49:49.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T03:49:49.489 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 bash[108042]: 1bddf6f7cdf3b54006a0ea88cfa23d88535b289a07652957d62c9f818b2a3434 2023-10-09T03:49:49.489 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 systemd[1]: Started Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:49:49.490 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 conmon[108147]: 2023-10-09T03:49:49.407+0000 7fabfb1121c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:49:49.490 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 conmon[108147]: 2023-10-09T03:49:49.488+0000 7fabfb1121c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:49:49.579 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:49.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 conmon[108147]: 2023-10-09T03:49:49.583+0000 7fabfb1121c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:49:49.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 conmon[108147]: 2023-10-09T03:49:49.678+0000 7fabfb1121c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:49:50.227 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:49 smithi181 conmon[108147]: 2023-10-09T03:49:49.880+0000 7fabfb1121c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:49:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:50 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:50 smithi181 conmon[108147]: 2023-10-09T03:49:50.225+0000 7fabfb1121c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:50 smithi181 conmon[108147]: 2023-10-09T03:49:50.315+0000 7fabfb1121c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:49:50.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:50 smithi181 conmon[108147]: 2023-10-09T03:49:50.454+0000 7fabfb1121c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:49:50.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:50 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:50.962 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:50 smithi181 conmon[108147]: 2023-10-09T03:49:50.548+0000 7fabfb1121c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:49:51.392 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T03:49:51.393 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T03:49:51.393 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T03:49:51.393 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T03:49:51.393 INFO:teuthology.orchestra.run.smithi163.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111568 s, 940 MB/s 2023-10-09T03:49:51.393 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T03:49:51.694 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:50 smithi181 conmon[108147]: 2023-10-09T03:49:50.961+0000 7fabfb1121c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:51 smithi181 conmon[108147]: 2023-10-09T03:49:51.525+0000 7fabfb1121c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:51 smithi181 conmon[108147]: 2023-10-09T03:49:51.693+0000 7fabfb1121c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:51.695 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T03:49:51.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:49:51.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:49:51.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:51.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:51 smithi181 ceph-mon[105375]: Reconfiguring daemon mgr.y on smithi163 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:49:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[118807]: Reconfiguring daemon mgr.y on smithi163 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T03:49:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:49:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:49:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:51 smithi163 ceph-mon[101022]: Reconfiguring daemon mgr.y on smithi163 2023-10-09T03:49:52.365 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_4 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.697 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:52 smithi181 conmon[108147]: 2023-10-09T03:49:52.442+0000 7fabfb1121c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:49:52.697 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:52 smithi181 conmon[108147]: 2023-10-09T03:49:52.532+0000 7fabfb1121c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:49:52.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:49:52.702 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:49:52.966 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:52 smithi181 conmon[108147]: 2023-10-09T03:49:52.697+0000 7fabfb1121c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:49:53.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:52 smithi181 conmon[108147]: 2023-10-09T03:49:52.966+0000 7fabfb1121c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:49:53.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:53 smithi181 conmon[108147]: 2023-10-09T03:49:53.050+0000 7fabfb1121c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:49:53.585 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:53 smithi181 conmon[108147]: 2023-10-09T03:49:53.334+0000 7fabfb1121c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:49:54.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:53 smithi181 conmon[108147]: 2023-10-09T03:49:53.584+0000 7fabfb1121c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:49:54.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:54 smithi181 conmon[108147]: 2023-10-09T03:49:54.282+0000 7fabfb1121c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:49:54.596 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:54 smithi181 conmon[108147]: 2023-10-09T03:49:54.412+0000 7fabfb1121c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:49:54.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:54 smithi181 ceph-mon[105375]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:54.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:49:54.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:49:54.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:54.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[118807]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[101022]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:49:54.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:49:54.915 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:54 smithi181 conmon[108147]: 2023-10-09T03:49:54.596+0000 7fabfb1121c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:49:54.915 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:54 smithi181 conmon[108147]: 2023-10-09T03:49:54.707+0000 7fabfb1121c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:49:55.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:54 smithi181 conmon[108147]: 2023-10-09T03:49:54.915+0000 7fabfb1121c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:49:55.767 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:55 smithi181 ceph-mon[105375]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:55.806 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:55 smithi163 ceph-mon[101022]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:55.806 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:55 smithi163 ceph-mon[118807]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:49:56.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:49:55 smithi181 conmon[108147]: 2023-10-09T03:49:55.767+0000 7fabfb1121c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:49:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:56 smithi181 ceph-mon[105375]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:56 smithi181 ceph-mon[105375]: Standby manager daemon x started 2023-10-09T03:49:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:56 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:49:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:56 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:49:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:56 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:49:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:56 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:49:56.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[101022]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[101022]: Standby manager daemon x started 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[118807]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[118807]: Standby manager daemon x started 2023-10-09T03:49:56.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:49:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:49:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:49:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:56 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3912213234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:49:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T03:49:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:49:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3848795466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]: dispatch 2023-10-09T03:49:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]: dispatch 2023-10-09T03:49:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]': finished 2023-10-09T03:49:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T03:49:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:49:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:57 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/912356762' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3848795466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]: dispatch 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]: dispatch 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]': finished 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:49:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/912356762' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: mgrmap e14: y(active, since 61s), standbys: x 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3848795466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]: dispatch 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]: dispatch 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f39300f-5c82-4020-b75d-28b9705365f3"}]': finished 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:49:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:57 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/912356762' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:49:58.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:49:58 smithi163 ceph-mon[101022]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:49:58 smithi163 ceph-mon[118807]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:49:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:49:58 smithi181 ceph-mon[105375]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:00.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:00 smithi163 ceph-mon[101022]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:00 smithi163 ceph-mon[101022]: overall HEALTH_OK 2023-10-09T03:50:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:00 smithi163 ceph-mon[118807]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:00 smithi163 ceph-mon[118807]: overall HEALTH_OK 2023-10-09T03:50:01.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:00 smithi181 ceph-mon[105375]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:01.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:00 smithi181 ceph-mon[105375]: overall HEALTH_OK 2023-10-09T03:50:02.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:02 smithi163 ceph-mon[101022]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:02 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T03:50:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:02 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:02 smithi163 ceph-mon[118807]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:02 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T03:50:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:02 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:02 smithi181 ceph-mon[105375]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:02 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T03:50:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:02 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:03.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:03 smithi163 ceph-mon[101022]: Deploying daemon osd.0 on smithi163 2023-10-09T03:50:03.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:03 smithi163 ceph-mon[118807]: Deploying daemon osd.0 on smithi163 2023-10-09T03:50:04.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:03 smithi181 ceph-mon[105375]: Deploying daemon osd.0 on smithi163 2023-10-09T03:50:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:04 smithi163 ceph-mon[101022]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:04 smithi163 ceph-mon[118807]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:04 smithi181 ceph-mon[105375]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:06 smithi163 ceph-mon[101022]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:06 smithi163 ceph-mon[118807]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:07.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:06 smithi181 ceph-mon[105375]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[118807]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[101022]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:08 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:08.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:08 smithi181 ceph-mon[105375]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:08 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:08 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:08 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:10.306 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 0 on host 'smithi163' 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[101022]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[118807]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:10.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:10.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:10.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:10 smithi181 ceph-mon[105375]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:10.900 DEBUG:teuthology.orchestra.run.smithi163:osd.0> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T03:50:10.902 INFO:tasks.cephadm:Deploying osd.1 on smithi163 with /dev/vg_nvme/lv_3... 2023-10-09T03:50:10.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T03:50:10.934 INFO:journalctl@ceph.osd.0.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:50:11.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:11.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:11.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:11.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[118807]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T03:50:11.513 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:11.513 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:11.513 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:11.513 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:11 smithi163 ceph-mon[101022]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T03:50:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:11 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:11 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:11 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:11 smithi181 ceph-mon[105375]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T03:50:11.946 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:50:12.527 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:50:12 smithi163 conmon[125110]: 2023-10-09T03:50:12.283+0000 7fda18538700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T03:50:12.527 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:50:12 smithi163 conmon[125110]: 2023-10-09T03:50:12.285+0000 7fda13b34700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:50:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[101022]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:12.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[101022]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[101022]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[101022]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[118807]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[118807]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[118807]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:12.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:12 smithi163 ceph-mon[118807]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:12 smithi181 ceph-mon[105375]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:12 smithi181 ceph-mon[105375]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T03:50:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:12 smithi181 ceph-mon[105375]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T03:50:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:12 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:12 smithi181 ceph-mon[105375]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:13.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[101022]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[101022]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[118807]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[118807]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:13.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:13 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:13.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:13 smithi181 ceph-mon[105375]: from='osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:13.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:13 smithi181 ceph-mon[105375]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T03:50:13.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:13 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:13 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:13.877 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T03:50:13.877 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T03:50:13.877 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T03:50:13.877 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T03:50:13.877 INFO:teuthology.orchestra.run.smithi163.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0169401 s, 619 MB/s 2023-10-09T03:50:13.877 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T03:50:14.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_3 2023-10-09T03:50:14.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717] boot 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717] boot 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.739 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: osd.0 [v2:172.21.15.163:6802/3842166717,v1:172.21.15.163:6803/3842166717] boot 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:14.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:14.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:14.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:15 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi163 2023-10-09T03:50:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:15 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi163 to 16144M 2023-10-09T03:50:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:15 smithi163 ceph-mon[101022]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T03:50:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:15 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi163 2023-10-09T03:50:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:15 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi163 to 16144M 2023-10-09T03:50:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:15 smithi163 ceph-mon[118807]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T03:50:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:15 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi163 2023-10-09T03:50:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:15 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi163 to 16144M 2023-10-09T03:50:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:15 smithi181 ceph-mon[105375]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T03:50:16.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:16 smithi163 ceph-mon[101022]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:16 smithi163 ceph-mon[118807]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:16.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:16 smithi181 ceph-mon[105375]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[101022]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:50:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:50:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:50:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[118807]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:50:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:50:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:50:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:17.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:17 smithi181 ceph-mon[105375]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:50:17.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:50:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:50:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:18.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:18 smithi181 ceph-mon[105375]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:18 smithi163 ceph-mon[101022]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:18 smithi163 ceph-mon[118807]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:19 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2983347054' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "07dc6811-c7f1-40ce-83b3-46a33ff57f0a"}]: dispatch 2023-10-09T03:50:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:19 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2983347054' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "07dc6811-c7f1-40ce-83b3-46a33ff57f0a"}]': finished 2023-10-09T03:50:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:19 smithi181 ceph-mon[105375]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T03:50:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:19 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/73159555' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:50:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2983347054' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "07dc6811-c7f1-40ce-83b3-46a33ff57f0a"}]: dispatch 2023-10-09T03:50:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2983347054' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "07dc6811-c7f1-40ce-83b3-46a33ff57f0a"}]': finished 2023-10-09T03:50:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[101022]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T03:50:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/73159555' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:50:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2983347054' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "07dc6811-c7f1-40ce-83b3-46a33ff57f0a"}]: dispatch 2023-10-09T03:50:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2983347054' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "07dc6811-c7f1-40ce-83b3-46a33ff57f0a"}]': finished 2023-10-09T03:50:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[118807]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T03:50:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:19 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/73159555' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:50:21.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:20 smithi181 ceph-mon[105375]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:21.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:20 smithi163 ceph-mon[101022]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:20 smithi163 ceph-mon[118807]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:22.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:22 smithi163 ceph-mon[101022]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:22.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:22 smithi163 ceph-mon[118807]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:23.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:22 smithi181 ceph-mon[105375]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:25.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:24 smithi181 ceph-mon[105375]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:25.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:24 smithi163 ceph-mon[101022]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:24 smithi163 ceph-mon[118807]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:25.995 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:25 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T03:50:25.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:25 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:25.996 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:25 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T03:50:25.996 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:25 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:26.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T03:50:26.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:25 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:26 smithi181 ceph-mon[105375]: Deploying daemon osd.1 on smithi163 2023-10-09T03:50:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:26 smithi181 ceph-mon[105375]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:27.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:26 smithi163 ceph-mon[118807]: Deploying daemon osd.1 on smithi163 2023-10-09T03:50:27.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:26 smithi163 ceph-mon[118807]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:27.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:26 smithi163 ceph-mon[101022]: Deploying daemon osd.1 on smithi163 2023-10-09T03:50:27.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:26 smithi163 ceph-mon[101022]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:29.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:28 smithi181 ceph-mon[105375]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:28 smithi163 ceph-mon[101022]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:28 smithi163 ceph-mon[118807]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[101022]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[118807]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:30 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:30 smithi181 ceph-mon[105375]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.362 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 1 on host 'smithi163' 2023-10-09T03:50:32.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:32 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:32.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:32 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:32.895 DEBUG:teuthology.orchestra.run.smithi163:osd.1> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T03:50:32.897 INFO:tasks.cephadm:Deploying osd.2 on smithi163 with /dev/vg_nvme/lv_2... 2023-10-09T03:50:32.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T03:50:32.927 INFO:journalctl@ceph.osd.1.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:50:33.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:33 smithi181 ceph-mon[105375]: from='osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T03:50:33.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:33 smithi181 ceph-mon[105375]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T03:50:33.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:33 smithi163 ceph-mon[118807]: from='osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T03:50:33.873 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:33 smithi163 ceph-mon[118807]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T03:50:33.873 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:33 smithi163 ceph-mon[101022]: from='osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T03:50:33.873 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:33 smithi163 ceph-mon[101022]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T03:50:33.927 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:50:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:34 smithi181 ceph-mon[105375]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:34.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:34 smithi181 ceph-mon[105375]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T03:50:34.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:34 smithi181 ceph-mon[105375]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T03:50:34.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:34 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:34.973 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:34 smithi181 ceph-mon[105375]: from='osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:34.973 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:34 smithi181 ceph-mon[105375]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:34.974 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:50:34 smithi163 conmon[134840]: 2023-10-09T03:50:34.452+0000 7f19a973a700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T03:50:34.974 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:50:34 smithi163 conmon[134840]: 2023-10-09T03:50:34.455+0000 7f19a3d34700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[118807]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[118807]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[118807]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[118807]: from='osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[118807]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[101022]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:34.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[101022]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T03:50:34.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[101022]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T03:50:34.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:34.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[101022]: from='osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:34.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:34 smithi163 ceph-mon[101022]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:35.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[101022]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:35.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[101022]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T03:50:35.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:35.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:35.565 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[118807]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:35.565 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[118807]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T03:50:35.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:35.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:35 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:35.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:35 smithi181 ceph-mon[105375]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:35 smithi181 ceph-mon[105375]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T03:50:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:35 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:35 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:35.839 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T03:50:35.839 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T03:50:35.840 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T03:50:35.840 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T03:50:35.840 INFO:teuthology.orchestra.run.smithi163.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127623 s, 822 MB/s 2023-10-09T03:50:35.840 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 2023-10-09T03:50:35.840 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T03:50:36.420 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_2 2023-10-09T03:50:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:50:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:50:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175] boot 2023-10-09T03:50:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T03:50:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:36.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:36.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:36.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:50:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:50:36.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:36.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175] boot 2023-10-09T03:50:36.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T03:50:36.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:36.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.761 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:50:36.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: osd.1 [v2:172.21.15.163:6810/147795175,v1:172.21.15.163:6811/147795175] boot 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:37 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi163 2023-10-09T03:50:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:37 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi163 to 8072M 2023-10-09T03:50:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:37 smithi163 ceph-mon[118807]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T03:50:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:37 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi163 2023-10-09T03:50:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:37 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi163 to 8072M 2023-10-09T03:50:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:37 smithi163 ceph-mon[101022]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T03:50:37.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:37 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi163 2023-10-09T03:50:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:37 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi163 to 8072M 2023-10-09T03:50:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:37 smithi181 ceph-mon[105375]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T03:50:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:38 smithi181 ceph-mon[105375]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:50:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:50:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:38 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[101022]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:50:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:50:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[118807]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:50:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:50:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:38 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:39.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:39 smithi163 ceph-mon[101022]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:50:39.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:39 smithi163 ceph-mon[118807]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:50:39.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:39 smithi181 ceph-mon[105375]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:50:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:40 smithi181 ceph-mon[105375]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:40 smithi163 ceph-mon[101022]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:40 smithi163 ceph-mon[118807]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:41.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:41 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/162211273' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d5a25867-ce16-4bfd-8356-c346de1fcbf6"}]: dispatch 2023-10-09T03:50:41.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:41 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/162211273' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d5a25867-ce16-4bfd-8356-c346de1fcbf6"}]': finished 2023-10-09T03:50:41.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:41 smithi181 ceph-mon[105375]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T03:50:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:41 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1804778168' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:50:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/162211273' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d5a25867-ce16-4bfd-8356-c346de1fcbf6"}]: dispatch 2023-10-09T03:50:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/162211273' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d5a25867-ce16-4bfd-8356-c346de1fcbf6"}]': finished 2023-10-09T03:50:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[101022]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T03:50:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1804778168' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:50:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/162211273' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d5a25867-ce16-4bfd-8356-c346de1fcbf6"}]: dispatch 2023-10-09T03:50:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/162211273' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d5a25867-ce16-4bfd-8356-c346de1fcbf6"}]': finished 2023-10-09T03:50:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[118807]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T03:50:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:41 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1804778168' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:50:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:42 smithi181 ceph-mon[105375]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:42.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:42 smithi163 ceph-mon[101022]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:42 smithi163 ceph-mon[118807]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:44 smithi163 ceph-mon[101022]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:44.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:44 smithi163 ceph-mon[118807]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:44 smithi181 ceph-mon[105375]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:46 smithi163 ceph-mon[101022]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:46 smithi163 ceph-mon[118807]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:46 smithi181 ceph-mon[105375]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:47 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T03:50:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:47 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:47.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:47 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T03:50:47.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:47 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:47.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:47 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T03:50:47.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:47 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:48.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:48 smithi181 ceph-mon[105375]: Deploying daemon osd.2 on smithi163 2023-10-09T03:50:48.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:48 smithi181 ceph-mon[105375]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:48.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:48 smithi163 ceph-mon[118807]: Deploying daemon osd.2 on smithi163 2023-10-09T03:50:48.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:48 smithi163 ceph-mon[118807]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:48.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:48 smithi163 ceph-mon[101022]: Deploying daemon osd.2 on smithi163 2023-10-09T03:50:48.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:48 smithi163 ceph-mon[101022]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:50 smithi181 ceph-mon[105375]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:50.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:50 smithi163 ceph-mon[101022]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:50 smithi163 ceph-mon[118807]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:52.828 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[118807]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:52.828 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:52.828 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:52.829 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:52.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[101022]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:52.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:52.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:52.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:52 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:53.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:52 smithi181 ceph-mon[105375]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:52 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:54.348 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 2 on host 'smithi163' 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:54 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:54 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:50:54.941 DEBUG:teuthology.orchestra.run.smithi163:osd.2> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T03:50:54.943 INFO:tasks.cephadm:Deploying osd.3 on smithi163 with /dev/vg_nvme/lv_1... 2023-10-09T03:50:54.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T03:50:54.974 INFO:journalctl@ceph.osd.2.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:50:55.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:55 smithi163 ceph-mon[101022]: from='osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T03:50:55.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:55 smithi163 ceph-mon[101022]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T03:50:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:55 smithi181 ceph-mon[105375]: from='osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T03:50:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:55 smithi181 ceph-mon[105375]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T03:50:55.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:55 smithi163 ceph-mon[118807]: from='osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T03:50:55.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:55 smithi163 ceph-mon[118807]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T03:50:55.940 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:50:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[101022]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[101022]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T03:50:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[101022]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T03:50:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[101022]: from='osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[101022]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[118807]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[118807]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[118807]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[118807]: from='osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:56 smithi163 ceph-mon[118807]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:56.584 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:50:56 smithi163 conmon[143762]: 2023-10-09T03:50:56.509+0000 7f692514e700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T03:50:56.584 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:50:56 smithi163 conmon[143762]: 2023-10-09T03:50:56.513+0000 7f691df45700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:50:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:56 smithi181 ceph-mon[105375]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:56 smithi181 ceph-mon[105375]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T03:50:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:56 smithi181 ceph-mon[105375]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T03:50:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:56 smithi181 ceph-mon[105375]: from='osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:56 smithi181 ceph-mon[105375]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:50:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:57 smithi181 ceph-mon[105375]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:57 smithi181 ceph-mon[105375]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T03:50:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:57 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:57 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:57.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[101022]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:57.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[101022]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T03:50:57.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:57.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:57.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[118807]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:50:57.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[118807]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T03:50:57.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:57.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:57 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:57.836 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T03:50:57.837 INFO:teuthology.orchestra.run.smithi163.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T03:50:57.837 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10+0 records in 2023-10-09T03:50:57.837 INFO:teuthology.orchestra.run.smithi163.stderr:10+0 records out 2023-10-09T03:50:57.837 INFO:teuthology.orchestra.run.smithi163.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110068 s, 953 MB/s 2023-10-09T03:50:57.837 INFO:teuthology.orchestra.run.smithi163.stderr:--> Zapping successful for: 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924] boot 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T03:50:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924] boot 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:58.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:58.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: osd.2 [v2:172.21.15.163:6818/2150896924,v1:172.21.15.163:6819/2150896924] boot 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:50:58.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:50:58.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:50:58.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:50:58.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi163:vg_nvme/lv_1 2023-10-09T03:50:59.141 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:50:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:59 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi163 2023-10-09T03:50:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:59 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi163 to 5381M 2023-10-09T03:50:59.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:50:59 smithi163 ceph-mon[101022]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T03:50:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:59 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi163 2023-10-09T03:50:59.719 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:59 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi163 to 5381M 2023-10-09T03:50:59.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:50:59 smithi163 ceph-mon[118807]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T03:51:00.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:59 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi163 2023-10-09T03:51:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:59 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi163 to 5381M 2023-10-09T03:51:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:50:59 smithi181 ceph-mon[105375]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[118807]: pgmap v71: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[118807]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[101022]: pgmap v71: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T03:51:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T03:51:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[101022]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T03:51:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:00 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:51:01.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:00 smithi181 ceph-mon[105375]: pgmap v71: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:00 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T03:51:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:00 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T03:51:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:00 smithi181 ceph-mon[105375]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T03:51:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:00 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:51:01.249 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:00 smithi163 sudo[150616]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T03:51:01.250 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:00 smithi163 sudo[150616]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:01.250 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:00 smithi163 sudo[150616]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:01.250 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:00 smithi163 sudo[150616]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:01.526 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150775]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T03:51:01.526 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150775]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:01.526 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150775]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:01.526 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150775]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:01.799 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150798]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T03:51:01.799 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150798]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:01.799 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150798]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:01.799 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150798]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:01.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T03:51:01.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[118807]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[101022]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:01.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:02.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T03:51:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:01 smithi181 ceph-mon[105375]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T03:51:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:02.056 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150853]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150853]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150853]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 03:51:01 smithi163 sudo[150853]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150856]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150856]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150856]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:02.057 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150856]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:02.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150868]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T03:51:02.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150868]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:02.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150868]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:02.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150868]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:02.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150922]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T03:51:02.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150922]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:02.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150922]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:02.829 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150922]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:03.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:02 smithi163 ceph-mon[118807]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:03.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:02 smithi163 ceph-mon[118807]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:03.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:02 smithi163 ceph-mon[118807]: osdmap e22: 3 total, 3 up, 3 in 2023-10-09T03:51:03.099 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150952]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T03:51:03.099 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150952]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:03.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150952]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:03.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 sudo[150952]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:03.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 ceph-mon[101022]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:03.100 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 ceph-mon[101022]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:03.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:02 smithi163 ceph-mon[101022]: osdmap e22: 3 total, 3 up, 3 in 2023-10-09T03:51:03.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:02 smithi181 ceph-mon[105375]: from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi163:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:03.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:02 smithi181 ceph-mon[105375]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:03.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:02 smithi181 ceph-mon[105375]: osdmap e22: 3 total, 3 up, 3 in 2023-10-09T03:51:03.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109116]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T03:51:03.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109116]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:03.336 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109116]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:03.336 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109116]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:03.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109119]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T03:51:04.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109119]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:04.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109119]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:04.039 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 sudo[109119]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:04.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151002]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T03:51:04.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151002]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:04.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151002]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:04.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151002]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3487783233' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d47d9fe-d7d7-4505-abf8-22ac463ee305"}]: dispatch 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3487783233' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d47d9fe-d7d7-4505-abf8-22ac463ee305"}]': finished 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T03:51:04.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:04.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:04.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151025]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151025]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151025]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 sudo[151025]: pam_unix(sudo:session): session closed for user root 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3487783233' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d47d9fe-d7d7-4505-abf8-22ac463ee305"}]: dispatch 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3487783233' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d47d9fe-d7d7-4505-abf8-22ac463ee305"}]': finished 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:04.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3487783233' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d47d9fe-d7d7-4505-abf8-22ac463ee305"}]: dispatch 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3487783233' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d47d9fe-d7d7-4505-abf8-22ac463ee305"}]': finished 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:51:04.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:51:04.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:51:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:04 smithi181 ceph-mon[105375]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:04 smithi181 ceph-mon[105375]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:04 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1926522462' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:04 smithi163 ceph-mon[118807]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:04 smithi163 ceph-mon[118807]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:04 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1926522462' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:05.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:04 smithi163 ceph-mon[101022]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T03:51:05.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:04 smithi163 ceph-mon[101022]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T03:51:05.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:04 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1926522462' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:06.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:05 smithi163 ceph-mon[118807]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:06.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:05 smithi163 ceph-mon[101022]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:06.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:05 smithi181 ceph-mon[105375]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:08.408 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:08 smithi163 ceph-mon[118807]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.2 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-10-09T03:51:08.409 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:08 smithi163 ceph-mon[101022]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.2 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-10-09T03:51:08.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:08 smithi181 ceph-mon[105375]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.2 KiB/s rd, 139 KiB/s wr, 9 op/s 2023-10-09T03:51:09.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:09 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T03:51:09.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:09 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:09 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T03:51:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:09 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:09 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T03:51:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:09 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:10.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:10 smithi163 ceph-mon[101022]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.4 KiB/s rd, 103 KiB/s wr, 7 op/s 2023-10-09T03:51:10.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:10 smithi163 ceph-mon[101022]: Deploying daemon osd.3 on smithi163 2023-10-09T03:51:10.555 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:10 smithi163 ceph-mon[118807]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.4 KiB/s rd, 103 KiB/s wr, 7 op/s 2023-10-09T03:51:10.555 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:10 smithi163 ceph-mon[118807]: Deploying daemon osd.3 on smithi163 2023-10-09T03:51:10.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:10 smithi181 ceph-mon[105375]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.4 KiB/s rd, 103 KiB/s wr, 7 op/s 2023-10-09T03:51:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:10 smithi181 ceph-mon[105375]: Deploying daemon osd.3 on smithi163 2023-10-09T03:51:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:12 smithi181 ceph-mon[105375]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.9 KiB/s rd, 81 KiB/s wr, 5 op/s 2023-10-09T03:51:12.814 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:12 smithi163 ceph-mon[101022]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.9 KiB/s rd, 81 KiB/s wr, 5 op/s 2023-10-09T03:51:12.814 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:12 smithi163 ceph-mon[118807]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.9 KiB/s rd, 81 KiB/s wr, 5 op/s 2023-10-09T03:51:14.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[118807]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T03:51:14.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:14.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:14.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:14.612 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[101022]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T03:51:14.612 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:14.612 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:14.612 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:14 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:14 smithi181 ceph-mon[105375]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T03:51:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:14 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:16 smithi181 ceph-mon[105375]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2023-10-09T03:51:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:16 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:16 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:16.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:16 smithi163 ceph-mon[118807]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2023-10-09T03:51:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:16 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:16 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:16 smithi163 ceph-mon[101022]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2023-10-09T03:51:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:16 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:16 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:16.991 INFO:teuthology.orchestra.run.smithi163.stdout:Created osd(s) 3 on host 'smithi163' 2023-10-09T03:51:17.542 DEBUG:teuthology.orchestra.run.smithi163:osd.3> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T03:51:17.544 INFO:tasks.cephadm:Deploying osd.4 on smithi181 with /dev/vg_nvme/lv_4... 2023-10-09T03:51:17.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T03:51:17.577 INFO:journalctl@ceph.osd.3.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:51:17.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:17.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:17.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.618 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[118807]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:17.620 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:17 smithi163 ceph-mon[101022]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T03:51:17.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:17.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:17.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:17 smithi181 ceph-mon[105375]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T03:51:17.904 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:51:18.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:18 smithi181 ceph-mon[105375]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T03:51:18.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:18 smithi181 ceph-mon[105375]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T03:51:18.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:18 smithi181 ceph-mon[105375]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T03:51:18.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:18 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:18.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:18 smithi181 ceph-mon[105375]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:51:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[118807]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T03:51:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[118807]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T03:51:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[118807]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T03:51:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[118807]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:51:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[101022]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T03:51:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[101022]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T03:51:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[101022]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T03:51:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:18 smithi163 ceph-mon[101022]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T03:51:19.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 03:51:18 smithi163 conmon[153552]: 2023-10-09T03:51:18.961+0000 7f8159b0b700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T03:51:19.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 03:51:18 smithi163 conmon[153552]: 2023-10-09T03:51:18.965+0000 7f8154105700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:51:19.349 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T03:51:19.349 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T03:51:19.349 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T03:51:19.349 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T03:51:19.349 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110203 s, 951 MB/s 2023-10-09T03:51:19.349 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T03:51:19.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_4 2023-10-09T03:51:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[101022]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:51:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[101022]: osdmap e25: 4 total, 3 up, 4 in 2023-10-09T03:51:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[101022]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[118807]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[118807]: osdmap e25: 4 total, 3 up, 4 in 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:20.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:19 smithi163 ceph-mon[118807]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:20.179 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:51:20.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:19 smithi181 ceph-mon[105375]: from='osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]': finished 2023-10-09T03:51:20.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:19 smithi181 ceph-mon[105375]: osdmap e25: 4 total, 3 up, 4 in 2023-10-09T03:51:20.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:20.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:20.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:19 smithi181 ceph-mon[105375]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T03:51:21.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115] boot 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi163 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi163 to 4036M 2023-10-09T03:51:21.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:21.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:21.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115] boot 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi163 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi163 to 4036M 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: osd.3 [v2:172.21.15.163:6826/1216700115,v1:172.21.15.163:6827/1216700115] boot 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi163 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi163 to 4036M 2023-10-09T03:51:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:22.242 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:21 smithi181 ceph-mon[105375]: osdmap e27: 4 total, 4 up, 4 in 2023-10-09T03:51:22.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:21 smithi181 ceph-mon[105375]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:22.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:21 smithi181 ceph-mon[105375]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:22.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:21 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:22.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:21 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:22.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:21 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[118807]: osdmap e27: 4 total, 4 up, 4 in 2023-10-09T03:51:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[118807]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[118807]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[101022]: osdmap e27: 4 total, 4 up, 4 in 2023-10-09T03:51:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[101022]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[101022]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:21 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[101022]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/1010949725' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]: dispatch 2023-10-09T03:51:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]: dispatch 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]': finished 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[101022]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[118807]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/1010949725' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]: dispatch 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]: dispatch 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]': finished 2023-10-09T03:51:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[118807]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T03:51:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:24 smithi181 ceph-mon[105375]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:24 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/1010949725' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]: dispatch 2023-10-09T03:51:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:24 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]: dispatch 2023-10-09T03:51:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:24 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad58407a-b12d-4543-8092-f0ba514729f5"}]': finished 2023-10-09T03:51:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:24 smithi181 ceph-mon[105375]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T03:51:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:25.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:25 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/2631940703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:25 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/2631940703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:25.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:25 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/2631940703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:26 smithi163 ceph-mon[118807]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:26 smithi163 ceph-mon[101022]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:26 smithi181 ceph-mon[105375]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:28.647 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:28 smithi181 ceph-mon[105375]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:28 smithi163 ceph-mon[101022]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:28.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:28 smithi163 ceph-mon[118807]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:30.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:30 smithi181 ceph-mon[105375]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:30.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T03:51:30.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:30 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:30 smithi163 ceph-mon[101022]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T03:51:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:30 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:30.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:30 smithi163 ceph-mon[118807]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:30.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:30 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T03:51:30.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:30 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:31.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:31 smithi181 ceph-mon[105375]: Deploying daemon osd.4 on smithi181 2023-10-09T03:51:31.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:31 smithi163 ceph-mon[101022]: Deploying daemon osd.4 on smithi181 2023-10-09T03:51:31.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:31 smithi163 ceph-mon[118807]: Deploying daemon osd.4 on smithi181 2023-10-09T03:51:32.419 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:32 smithi181 ceph-mon[105375]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:32 smithi163 ceph-mon[101022]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:32 smithi163 ceph-mon[118807]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:34 smithi181 ceph-mon[105375]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:34 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:34 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:34 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[118807]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[101022]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:34 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:36.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:36 smithi181 ceph-mon[105375]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:36.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:36.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:36 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:36.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:36 smithi163 ceph-mon[118807]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:36.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:36.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:36 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:36.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:36 smithi163 ceph-mon[101022]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:36.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:36.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:36 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.309 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 4 on host 'smithi181' 2023-10-09T03:51:37.491 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:37 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:37.492 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:37 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:37.492 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:37 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.492 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:37 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.492 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:37 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.492 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:37 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:37.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:37.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:37.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:37 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:37.871 DEBUG:teuthology.orchestra.run.smithi181:osd.4> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T03:51:37.873 INFO:tasks.cephadm:Deploying osd.5 on smithi181 with /dev/vg_nvme/lv_3... 2023-10-09T03:51:37.873 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T03:51:37.902 INFO:journalctl@ceph.osd.4.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:51:38.401 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:51:38.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:38 smithi163 ceph-mon[101022]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:38 smithi163 ceph-mon[101022]: from='osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T03:51:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:38 smithi163 ceph-mon[101022]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T03:51:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:38 smithi163 ceph-mon[118807]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:38 smithi163 ceph-mon[118807]: from='osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T03:51:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:38 smithi163 ceph-mon[118807]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T03:51:38.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:38 smithi181 ceph-mon[105375]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:38.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:38 smithi181 ceph-mon[105375]: from='osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T03:51:38.991 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:38 smithi181 ceph-mon[105375]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T03:51:39.754 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 03:51:39 smithi181 conmon[112997]: 2023-10-09T03:51:39.570+0000 7f474635c700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T03:51:39.755 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 03:51:39 smithi181 conmon[112997]: 2023-10-09T03:51:39.575+0000 7f474315b700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:51:39.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:39 smithi181 ceph-mon[105375]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T03:51:39.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:39 smithi181 ceph-mon[105375]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T03:51:39.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:39.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:39 smithi181 ceph-mon[105375]: from='osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:51:39.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:39 smithi181 ceph-mon[105375]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[101022]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[101022]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[101022]: from='osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[101022]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[118807]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T03:51:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[118807]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T03:51:39.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:39.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[118807]: from='osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:51:39.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:39 smithi163 ceph-mon[118807]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:51:40.506 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T03:51:40.506 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T03:51:40.506 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T03:51:40.507 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T03:51:40.507 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130601 s, 803 MB/s 2023-10-09T03:51:40.507 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[101022]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[101022]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[101022]: osdmap e30: 5 total, 4 up, 5 in 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[118807]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[118807]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:51:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[118807]: osdmap e30: 5 total, 4 up, 5 in 2023-10-09T03:51:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:40 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:40 smithi181 ceph-mon[105375]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:40 smithi181 ceph-mon[105375]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:51:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:40 smithi181 ceph-mon[105375]: osdmap e30: 5 total, 4 up, 5 in 2023-10-09T03:51:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:40 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:40 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:41.359 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_3 2023-10-09T03:51:41.701 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:51:41.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[101022]: osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687] boot 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[101022]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[101022]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:51:41.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[118807]: osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687] boot 2023-10-09T03:51:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[118807]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T03:51:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:41 smithi163 ceph-mon[118807]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T03:51:41.874 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:41 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:51:41.874 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:41 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:51:41.874 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:41 smithi181 ceph-mon[105375]: osd.4 [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687] boot 2023-10-09T03:51:41.874 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:41 smithi181 ceph-mon[105375]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T03:51:41.874 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:51:41.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:41 smithi181 ceph-mon[105375]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T03:51:42.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:42.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi181 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi181 to 17168M 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.829 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:42 smithi181 ceph-mon[105375]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T03:51:42.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:42.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi181 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi181 to 17168M 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[118807]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T03:51:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi181 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi181 to 17168M 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:42 smithi163 ceph-mon[101022]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T03:51:43.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:43 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:43.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:43 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:43.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:43 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:43.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:43 smithi181 ceph-mon[105375]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T03:51:44.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[101022]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:44.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:43 smithi163 ceph-mon[118807]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T03:51:44.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:44 smithi181 ceph-mon[105375]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:44.736 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:44 smithi181 ceph-mon[105375]: pgmap v107: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:51:44.737 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:44 smithi181 ceph-mon[105375]: osdmap e35: 5 total, 5 up, 5 in 2023-10-09T03:51:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:44 smithi163 ceph-mon[101022]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:44 smithi163 ceph-mon[101022]: pgmap v107: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:51:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:44 smithi163 ceph-mon[101022]: osdmap e35: 5 total, 5 up, 5 in 2023-10-09T03:51:45.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:44 smithi163 ceph-mon[118807]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:51:45.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:44 smithi163 ceph-mon[118807]: pgmap v107: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:51:45.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:44 smithi163 ceph-mon[118807]: osdmap e35: 5 total, 5 up, 5 in 2023-10-09T03:51:46.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:45 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/1943763704' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]: dispatch 2023-10-09T03:51:46.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:45 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]: dispatch 2023-10-09T03:51:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:45 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]': finished 2023-10-09T03:51:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:45 smithi181 ceph-mon[105375]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T03:51:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:51:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/1943763704' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]: dispatch 2023-10-09T03:51:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]: dispatch 2023-10-09T03:51:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]': finished 2023-10-09T03:51:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[101022]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T03:51:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:51:46.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/1943763704' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]: dispatch 2023-10-09T03:51:46.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]: dispatch 2023-10-09T03:51:46.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21ccce9f-28b2-4624-8a70-2f726e7c45d1"}]': finished 2023-10-09T03:51:46.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[118807]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T03:51:46.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:51:47.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:46 smithi181 ceph-mon[105375]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:51:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:46 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/45355641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:46 smithi163 ceph-mon[101022]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:51:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:46 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/45355641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:46 smithi163 ceph-mon[118807]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:51:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:46 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/45355641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:51:48.878 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:48 smithi181 ceph-mon[105375]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 118 KiB/s, 0 objects/s recovering 2023-10-09T03:51:49.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:48 smithi163 ceph-mon[101022]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 118 KiB/s, 0 objects/s recovering 2023-10-09T03:51:49.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:48 smithi163 ceph-mon[118807]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 118 KiB/s, 0 objects/s recovering 2023-10-09T03:51:50.898 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:50 smithi181 ceph-mon[105375]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T03:51:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:50 smithi163 ceph-mon[101022]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T03:51:51.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:50 smithi163 ceph-mon[118807]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T03:51:51.941 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T03:51:51.942 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:51 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T03:51:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:51 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:52.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T03:51:52.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:51 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:52.702 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:52 smithi181 ceph-mon[105375]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-10-09T03:51:52.702 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:52 smithi181 ceph-mon[105375]: Deploying daemon osd.5 on smithi181 2023-10-09T03:51:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:52 smithi163 ceph-mon[118807]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-10-09T03:51:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:52 smithi163 ceph-mon[118807]: Deploying daemon osd.5 on smithi181 2023-10-09T03:51:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:52 smithi163 ceph-mon[101022]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-10-09T03:51:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:52 smithi163 ceph-mon[101022]: Deploying daemon osd.5 on smithi181 2023-10-09T03:51:55.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:54 smithi181 ceph-mon[105375]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-10-09T03:51:55.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:54 smithi163 ceph-mon[101022]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-10-09T03:51:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:54 smithi163 ceph-mon[118807]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-10-09T03:51:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:56 smithi181 ceph-mon[105375]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T03:51:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[101022]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[118807]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:57.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:58.449 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 5 on host 'smithi181' 2023-10-09T03:51:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:58 smithi181 ceph-mon[105375]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-10-09T03:51:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[101022]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[118807]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-10-09T03:51:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:51:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:51:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:58.979 DEBUG:teuthology.orchestra.run.smithi181:osd.5> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T03:51:58.981 INFO:tasks.cephadm:Deploying osd.6 on smithi181 with /dev/vg_nvme/lv_2... 2023-10-09T03:51:58.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T03:51:59.011 INFO:journalctl@ceph.osd.5.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:51:59.482 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:51:59.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:59 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:59.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:59 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:59 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:59 smithi181 ceph-mon[105375]: from='osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T03:51:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:51:59 smithi181 ceph-mon[105375]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[101022]: from='osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[101022]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:51:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[118807]: from='osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T03:51:59.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:51:59 smithi163 ceph-mon[118807]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T03:52:00.430 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:00 smithi181 ceph-mon[105375]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:52:00.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:00 smithi181 ceph-mon[105375]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T03:52:00.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:00 smithi181 ceph-mon[105375]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T03:52:00.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:00 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:00.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:00 smithi181 ceph-mon[105375]: from='osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:00.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:00 smithi181 ceph-mon[105375]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:00.769 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 03:52:00 smithi181 conmon[122624]: 2023-10-09T03:52:00.433+0000 7fc825302700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T03:52:00.769 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 03:52:00 smithi181 conmon[122624]: 2023-10-09T03:52:00.438+0000 7fc821900700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:52:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[101022]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:52:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[101022]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T03:52:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[101022]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T03:52:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[101022]: from='osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[101022]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[118807]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:52:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[118807]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T03:52:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[118807]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T03:52:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[118807]: from='osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:00 smithi163 ceph-mon[118807]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:01.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:01.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: osdmap e38: 6 total, 5 up, 6 in 2023-10-09T03:52:01.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307] boot 2023-10-09T03:52:01.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T03:52:01.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:01 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.684 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T03:52:01.684 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T03:52:01.684 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T03:52:01.684 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T03:52:01.684 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0145627 s, 720 MB/s 2023-10-09T03:52:01.684 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: osdmap e38: 6 total, 5 up, 6 in 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307] boot 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:01.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: osdmap e38: 6 total, 5 up, 6 in 2023-10-09T03:52:01.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:01.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: osd.5 [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307] boot 2023-10-09T03:52:01.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T03:52:01.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:01 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:52:02.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_2 2023-10-09T03:52:02.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:02 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:52:02.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:02 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:52:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:02 smithi181 ceph-mon[105375]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:52:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:02 smithi181 ceph-mon[105375]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[101022]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[101022]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[118807]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T03:52:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:02 smithi163 ceph-mon[118807]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T03:52:02.896 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:52:03.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi181 2023-10-09T03:52:03.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:03.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:03.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:03.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi181 to 8584M 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:03.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:03 smithi181 ceph-mon[105375]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi181 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi181 to 8584M 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[101022]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi181 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi181 to 8584M 2023-10-09T03:52:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:04.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:03 smithi163 ceph-mon[118807]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T03:52:05.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:04 smithi181 ceph-mon[105375]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:05.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:04 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:52:05.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:04 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:52:05.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:04 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:05.162 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:04 smithi181 ceph-mon[105375]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[101022]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[101022]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[118807]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:52:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:52:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:04 smithi163 ceph-mon[118807]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T03:52:06.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:06 smithi181 ceph-mon[105375]: pgmap v127: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:06 smithi181 ceph-mon[105375]: osdmap e43: 6 total, 6 up, 6 in 2023-10-09T03:52:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:06 smithi163 ceph-mon[101022]: pgmap v127: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:06 smithi163 ceph-mon[101022]: osdmap e43: 6 total, 6 up, 6 in 2023-10-09T03:52:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:06 smithi163 ceph-mon[118807]: pgmap v127: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:06 smithi163 ceph-mon[118807]: osdmap e43: 6 total, 6 up, 6 in 2023-10-09T03:52:07.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:07 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/2735493620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]: dispatch 2023-10-09T03:52:07.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:07 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]: dispatch 2023-10-09T03:52:07.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:07 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]': finished 2023-10-09T03:52:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:07 smithi181 ceph-mon[105375]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T03:52:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:07 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:07 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/1690820609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:52:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/2735493620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]: dispatch 2023-10-09T03:52:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]: dispatch 2023-10-09T03:52:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]': finished 2023-10-09T03:52:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[101022]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T03:52:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/1690820609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:52:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/2735493620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]: dispatch 2023-10-09T03:52:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]: dispatch 2023-10-09T03:52:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "69caed62-5f69-4fcd-bbbd-5369ec7bc6e7"}]': finished 2023-10-09T03:52:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[118807]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T03:52:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:07 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/1690820609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:52:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:08 smithi181 ceph-mon[105375]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:08 smithi163 ceph-mon[101022]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:08.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:08 smithi163 ceph-mon[118807]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:10.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:10 smithi181 ceph-mon[105375]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:10.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:10 smithi163 ceph-mon[101022]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:10 smithi163 ceph-mon[118807]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:12 smithi181 ceph-mon[105375]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:12 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T03:52:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:12 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:12 smithi163 ceph-mon[101022]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:12 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T03:52:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:12 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:12 smithi163 ceph-mon[118807]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:12 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T03:52:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:12 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:13.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:13 smithi181 ceph-mon[105375]: Deploying daemon osd.6 on smithi181 2023-10-09T03:52:13.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:13 smithi163 ceph-mon[101022]: Deploying daemon osd.6 on smithi181 2023-10-09T03:52:13.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:13 smithi163 ceph-mon[118807]: Deploying daemon osd.6 on smithi181 2023-10-09T03:52:14.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:14 smithi181 ceph-mon[105375]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:14.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:14 smithi163 ceph-mon[101022]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:14.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:14 smithi163 ceph-mon[118807]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:16 smithi163 ceph-mon[101022]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:16 smithi163 ceph-mon[118807]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:16.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:16 smithi181 ceph-mon[105375]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:18.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:18.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:18.441 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:17 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:18.441 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:17 smithi181 ceph-mon[105375]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:18.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:18.442 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:18.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:18.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[118807]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:18.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:18.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:18.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:18.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:17 smithi163 ceph-mon[101022]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:19.709 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 6 on host 'smithi181' 2023-10-09T03:52:19.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:19.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:19.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:19.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:19.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:19 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:20.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:20.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:19 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:20.324 DEBUG:teuthology.orchestra.run.smithi181:osd.6> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T03:52:20.326 INFO:tasks.cephadm:Deploying osd.7 on smithi181 with /dev/vg_nvme/lv_1... 2023-10-09T03:52:20.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T03:52:20.358 INFO:journalctl@ceph.osd.6.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:52:20.957 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:52:21.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:20 smithi181 ceph-mon[105375]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:20 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:20 smithi181 ceph-mon[105375]: from='osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T03:52:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:20 smithi181 ceph-mon[105375]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T03:52:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[101022]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[101022]: from='osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T03:52:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[101022]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T03:52:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[118807]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[118807]: from='osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T03:52:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:20 smithi163 ceph-mon[118807]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T03:52:21.768 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 03:52:21 smithi181 conmon[132062]: 2023-10-09T03:52:21.710+0000 7fbdea740700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T03:52:21.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 03:52:21 smithi181 conmon[132062]: 2023-10-09T03:52:21.717+0000 7fbde553b700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:52:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:21 smithi181 ceph-mon[105375]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T03:52:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:21 smithi181 ceph-mon[105375]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T03:52:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:21 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:21 smithi181 ceph-mon[105375]: from='osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:21 smithi181 ceph-mon[105375]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:22.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[101022]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[101022]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[101022]: from='osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[101022]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[118807]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[118807]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T03:52:22.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:22.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[118807]: from='osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:22.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:21 smithi163 ceph-mon[118807]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:23.013 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T03:52:23.013 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T03:52:23.013 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2023-10-09T03:52:23.013 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2023-10-09T03:52:23.013 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135408 s, 774 MB/s 2023-10-09T03:52:23.014 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2023-10-09T03:52:23.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:22 smithi181 ceph-mon[105375]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:22 smithi181 ceph-mon[105375]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:22 smithi181 ceph-mon[105375]: osdmap e46: 7 total, 6 up, 7 in 2023-10-09T03:52:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:22 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:22 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:23.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[101022]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:23.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[101022]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:23.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[101022]: osdmap e46: 7 total, 6 up, 7 in 2023-10-09T03:52:23.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:23.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[118807]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T03:52:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[118807]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[118807]: osdmap e46: 7 total, 6 up, 7 in 2023-10-09T03:52:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:22 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:23.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi181:vg_nvme/lv_1 2023-10-09T03:52:24.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:23 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:52:24.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:23 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:52:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:23 smithi181 ceph-mon[105375]: osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149] boot 2023-10-09T03:52:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:23 smithi181 ceph-mon[105375]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T03:52:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:23 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:23 smithi181 ceph-mon[105375]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T03:52:24.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[101022]: osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149] boot 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[101022]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[101022]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:52:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:52:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[118807]: osd.6 [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149] boot 2023-10-09T03:52:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[118807]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T03:52:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:52:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:23 smithi163 ceph-mon[118807]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T03:52:24.287 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:24 smithi181 ceph-mon[105375]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[101022]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:24 smithi163 ceph-mon[118807]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T03:52:26.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:25 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi181 2023-10-09T03:52:26.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:25 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi181 to 5722M 2023-10-09T03:52:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:25 smithi181 ceph-mon[105375]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:25 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi181 2023-10-09T03:52:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:25 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi181 to 5722M 2023-10-09T03:52:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:25 smithi163 ceph-mon[101022]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:25 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi181 2023-10-09T03:52:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:25 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi181 to 5722M 2023-10-09T03:52:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:25 smithi163 ceph-mon[118807]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:26 smithi181 ceph-mon[105375]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T03:52:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:26 smithi181 ceph-mon[105375]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:26 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:52:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:26 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:52:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:26 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:27.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[101022]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T03:52:27.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[101022]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:27.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[118807]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[118807]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T03:52:27.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:26 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:28.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:27 smithi181 ceph-mon[105375]: osdmap e51: 7 total, 7 up, 7 in 2023-10-09T03:52:28.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:27 smithi181 ceph-mon[105375]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:28.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:27 smithi163 ceph-mon[101022]: osdmap e51: 7 total, 7 up, 7 in 2023-10-09T03:52:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:27 smithi163 ceph-mon[101022]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:27 smithi163 ceph-mon[118807]: osdmap e51: 7 total, 7 up, 7 in 2023-10-09T03:52:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:27 smithi163 ceph-mon[118807]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:29.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:28 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/2748444001' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]: dispatch 2023-10-09T03:52:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:28 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]: dispatch 2023-10-09T03:52:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:28 smithi181 ceph-mon[105375]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]': finished 2023-10-09T03:52:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:28 smithi181 ceph-mon[105375]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T03:52:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:28 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:28 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/3815042259' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:52:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/2748444001' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]: dispatch 2023-10-09T03:52:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]: dispatch 2023-10-09T03:52:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[101022]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]': finished 2023-10-09T03:52:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[101022]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T03:52:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/3815042259' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:52:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/2748444001' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]: dispatch 2023-10-09T03:52:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]: dispatch 2023-10-09T03:52:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[118807]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3dc4fa53-2d90-441e-a825-74bd4da41f4a"}]': finished 2023-10-09T03:52:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[118807]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T03:52:29.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:29.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:28 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/3815042259' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T03:52:30.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:29 smithi181 ceph-mon[105375]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T03:52:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:29 smithi163 ceph-mon[101022]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T03:52:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:29 smithi163 ceph-mon[118807]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T03:52:32.745 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:32 smithi181 ceph-mon[105375]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T03:52:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:32 smithi163 ceph-mon[101022]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T03:52:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:32 smithi163 ceph-mon[118807]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T03:52:34.732 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:34 smithi181 ceph-mon[105375]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2023-10-09T03:52:34.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:34 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T03:52:34.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:34 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:34 smithi163 ceph-mon[101022]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2023-10-09T03:52:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:34 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T03:52:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:34 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:34 smithi163 ceph-mon[118807]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2023-10-09T03:52:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:34 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T03:52:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:34 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:35.656 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:35 smithi181 ceph-mon[105375]: Deploying daemon osd.7 on smithi181 2023-10-09T03:52:35.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:35 smithi163 ceph-mon[101022]: Deploying daemon osd.7 on smithi181 2023-10-09T03:52:35.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:35 smithi163 ceph-mon[118807]: Deploying daemon osd.7 on smithi181 2023-10-09T03:52:36.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:36 smithi181 ceph-mon[105375]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2023-10-09T03:52:36.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:36 smithi163 ceph-mon[101022]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2023-10-09T03:52:36.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:36 smithi163 ceph-mon[118807]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2023-10-09T03:52:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:38 smithi181 ceph-mon[105375]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T03:52:38.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:38 smithi163 ceph-mon[101022]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T03:52:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:38 smithi163 ceph-mon[118807]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T03:52:39.723 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:39.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:39.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:39 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:39.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:39 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:39 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:40 smithi181 ceph-mon[105375]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:40 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:40 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:40 smithi163 ceph-mon[101022]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:40 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:40.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:40 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:40 smithi163 ceph-mon[118807]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:40 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:40 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:41.463 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 7 on host 'smithi181' 2023-10-09T03:52:41.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:41.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:41.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:41.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:41 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:41.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:41.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:41.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:41.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:41.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:41 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:42.060 DEBUG:teuthology.orchestra.run.smithi181:osd.7> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T03:52:42.062 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T03:52:42.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T03:52:42.094 INFO:journalctl@ceph.osd.7.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:52:42.411 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:52:42.581 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:42 smithi181 ceph-mon[105375]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:42 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:42 smithi181 ceph-mon[105375]: from='osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T03:52:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:42 smithi181 ceph-mon[105375]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T03:52:42.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[101022]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:42.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:42.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:42.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[101022]: from='osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T03:52:42.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[101022]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T03:52:42.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[118807]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:42.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:42.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:42.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[118807]: from='osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T03:52:42.811 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:42 smithi163 ceph-mon[118807]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T03:52:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:43 smithi181 ceph-mon[105375]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T03:52:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:43 smithi181 ceph-mon[105375]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T03:52:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:43 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:43 smithi181 ceph-mon[105375]: from='osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:43 smithi181 ceph-mon[105375]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:43.519 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 03:52:43 smithi181 conmon[141718]: 2023-10-09T03:52:43.478+0000 7f1553051700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T03:52:43.519 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 03:52:43 smithi181 conmon[141718]: 2023-10-09T03:52:43.485+0000 7f154de4c700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T03:52:43.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[118807]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T03:52:43.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[118807]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T03:52:43.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:43.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[118807]: from='osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:43.646 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[118807]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:43.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[101022]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T03:52:43.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[101022]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T03:52:43.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:43.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[101022]: from='osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:43.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:43 smithi163 ceph-mon[101022]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2023-10-09T03:52:43.967 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:52:44.566 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1696823542,"num_in_osds":8,"osd_in_since":1696823548,"num_remapped_pgs":0} 2023-10-09T03:52:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:44 smithi181 ceph-mon[105375]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:44 smithi181 ceph-mon[105375]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:44 smithi181 ceph-mon[105375]: osdmap e54: 8 total, 7 up, 8 in 2023-10-09T03:52:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:44 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:44 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:44 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2120844126' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T03:52:44.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[118807]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[118807]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[118807]: osdmap e54: 8 total, 7 up, 8 in 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2120844126' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[101022]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[101022]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2023-10-09T03:52:44.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[101022]: osdmap e54: 8 total, 7 up, 8 in 2023-10-09T03:52:44.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:44.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:44.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:44 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2120844126' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T03:52:45.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T03:52:45.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: purged_snaps scrub starts 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: purged_snaps scrub ok 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668] boot 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:45.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:45.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:45 smithi181 ceph-mon[105375]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: purged_snaps scrub starts 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: purged_snaps scrub ok 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668] boot 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[101022]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: purged_snaps scrub starts 2023-10-09T03:52:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: purged_snaps scrub ok 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: osd.7 [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668] boot 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:45.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:45.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:45.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:45 smithi163 ceph-mon[118807]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T03:52:45.915 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:52:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:46 smithi181 ceph-mon[105375]: Detected new or changed devices on smithi181 2023-10-09T03:52:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:46 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi181 to 4292M 2023-10-09T03:52:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:46 smithi181 ceph-mon[105375]: pgmap v160: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:46.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:46 smithi163 ceph-mon[101022]: Detected new or changed devices on smithi181 2023-10-09T03:52:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:46 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi181 to 4292M 2023-10-09T03:52:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:46 smithi163 ceph-mon[101022]: pgmap v160: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:46 smithi163 ceph-mon[118807]: Detected new or changed devices on smithi181 2023-10-09T03:52:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:46 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi181 to 4292M 2023-10-09T03:52:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:46 smithi163 ceph-mon[118807]: pgmap v160: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:47.587 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:52:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:47 smithi181 ceph-mon[105375]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T03:52:47.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:47 smithi163 ceph-mon[118807]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T03:52:47.775 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:47 smithi163 ceph-mon[101022]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T03:52:48.224 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1696823564,"num_in_osds":8,"osd_in_since":1696823548,"num_remapped_pgs":0} 2023-10-09T03:52:48.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T03:52:48.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:48 smithi163 ceph-mon[101022]: pgmap v163: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:48.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:48 smithi163 ceph-mon[101022]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T03:52:48.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:48 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2365603883' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T03:52:48.570 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:52:48.766 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:48 smithi163 ceph-mon[118807]: pgmap v163: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:48.766 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:48 smithi163 ceph-mon[118807]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T03:52:48.766 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:48 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2365603883' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T03:52:48.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:48 smithi181 ceph-mon[105375]: pgmap v163: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:48.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:48 smithi181 ceph-mon[105375]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T03:52:48.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:48 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2365603883' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T03:52:49.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:49 smithi181 ceph-mon[105375]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T03:52:49.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:49 smithi163 ceph-mon[118807]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T03:52:49.818 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:49 smithi163 ceph-mon[101022]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T03:52:50.206 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:52:50.206 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":59,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T03:52:48.486446+0000","last_up_change":"2023-10-09T03:52:44.469504+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6803","nonce":3842166717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6805","nonce":3842166717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6809","nonce":3842166717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6807","nonce":3842166717}]},"public_addr":"172.21.15.163:6803/3842166717","cluster_addr":"172.21.15.163:6805/3842166717","heartbeat_back_addr":"172.21.15.163:6809/3842166717","heartbeat_front_addr":"172.21.15.163:6807/3842166717","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6811","nonce":147795175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6813","nonce":147795175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6817","nonce":147795175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6815","nonce":147795175}]},"public_addr":"172.21.15.163:6811/147795175","cluster_addr":"172.21.15.163:6813/147795175","heartbeat_back_addr":"172.21.15.163:6817/147795175","heartbeat_front_addr":"172.21.15.163:6815/147795175","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6819","nonce":2150896924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6821","nonce":2150896924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6825","nonce":2150896924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6823","nonce":2150896924}]},"public_addr":"172.21.15.163:6819/2150896924","cluster_addr":"172.21.15.163:6821/2150896924","heartbeat_back_addr":"172.21.15.163:6825/2150896924","heartbeat_front_addr":"172.21.15.163:6823/2150896924","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6827","nonce":1216700115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6829","nonce":1216700115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6833","nonce":1216700115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6831","nonce":1216700115}]},"public_addr":"172.21.15.163:6827/1216700115","cluster_addr":"172.21.15.163:6829/1216700115","heartbeat_back_addr":"172.21.15.163:6833/1216700115","heartbeat_front_addr":"172.21.15.163:6831/1216700115","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T03:52:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:50 smithi181 ceph-mon[105375]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:50 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/28852607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:52:50.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:50 smithi163 ceph-mon[118807]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:50 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/28852607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:52:50.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:50 smithi163 ceph-mon[101022]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:52:50.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:50 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/28852607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:52:50.899 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-09T03:50:59.539545+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-10-09T03:52:50.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-09T03:52:51.241 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:52:52.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:52 smithi181 ceph-mon[105375]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-10-09T03:52:52.833 INFO:teuthology.orchestra.run.smithi163.stdout:pg_num: 1 2023-10-09T03:52:52.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:52 smithi163 ceph-mon[101022]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-10-09T03:52:52.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:52 smithi163 ceph-mon[118807]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-10-09T03:52:53.464 INFO:tasks.cephadm:Adding prometheus.a on smithi181 2023-10-09T03:52:53.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi181=a' 2023-10-09T03:52:53.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:53 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2898150394' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T03:52:53.816 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:52:53.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:53 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2898150394' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T03:52:53.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:53 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2898150394' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T03:52:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:54 smithi181 ceph-mon[105375]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-10-09T03:52:54.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:54 smithi163 ceph-mon[101022]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-10-09T03:52:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:54 smithi163 ceph-mon[118807]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 85 KiB/s, 0 objects/s recovering 2023-10-09T03:52:55.428 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled prometheus update... 2023-10-09T03:52:56.111 DEBUG:teuthology.orchestra.run.smithi181:prometheus.a> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T03:52:56.113 INFO:tasks.cephadm:Adding node-exporter.a on smithi163 2023-10-09T03:52:56.113 INFO:tasks.cephadm:Adding node-exporter.b on smithi181 2023-10-09T03:52:56.113 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi163=a;smithi181=b' 2023-10-09T03:52:56.143 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:52:56.474 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:52:56.703 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: Saving service prometheus spec with placement smithi181=a;count:1 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:56.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:56 smithi181 ceph-mon[105375]: Deploying daemon prometheus.a on smithi181 2023-10-09T03:52:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T03:52:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: Saving service prometheus spec with placement smithi181=a;count:1 2023-10-09T03:52:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[101022]: Deploying daemon prometheus.a on smithi181 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: Saving service prometheus spec with placement smithi181=a;count:1 2023-10-09T03:52:56.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:56.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:52:56.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:52:56.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:52:56.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:56.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:56 smithi163 ceph-mon[118807]: Deploying daemon prometheus.a on smithi181 2023-10-09T03:52:58.427 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled node-exporter update... 2023-10-09T03:52:58.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:58 smithi181 ceph-mon[105375]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-10-09T03:52:58.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:58 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:58.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:58 smithi163 ceph-mon[101022]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-10-09T03:52:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:58 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:58 smithi163 ceph-mon[118807]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-10-09T03:52:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:58 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:52:59.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:59 smithi181 ceph-mon[105375]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi163=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:59.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:52:59 smithi181 ceph-mon[105375]: Saving service node-exporter spec with placement smithi163=a;smithi181=b;count:2 2023-10-09T03:52:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:59 smithi163 ceph-mon[101022]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi163=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:52:59 smithi163 ceph-mon[101022]: Saving service node-exporter spec with placement smithi163=a;smithi181=b;count:2 2023-10-09T03:52:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:59 smithi163 ceph-mon[118807]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi163=a;smithi181=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:52:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:52:59 smithi163 ceph-mon[118807]: Saving service node-exporter spec with placement smithi163=a;smithi181=b;count:2 2023-10-09T03:52:59.961 DEBUG:teuthology.orchestra.run.smithi163:node-exporter.a> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T03:52:59.964 DEBUG:teuthology.orchestra.run.smithi181:node-exporter.b> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T03:52:59.966 INFO:tasks.cephadm:Adding alertmanager.a on smithi163 2023-10-09T03:52:59.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi163=a' 2023-10-09T03:52:59.998 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:53:00.001 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:53:00.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:00 smithi181 ceph-mon[105375]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-10-09T03:53:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:00 smithi163 ceph-mon[101022]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-10-09T03:53:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:00 smithi163 ceph-mon[118807]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-10-09T03:53:02.454 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:53:02.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:02 smithi181 ceph-mon[105375]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T03:53:02.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:02 smithi163 ceph-mon[118807]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T03:53:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:02 smithi163 ceph-mon[101022]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T03:53:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:04 smithi181 ceph-mon[105375]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:04 smithi163 ceph-mon[101022]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:04 smithi163 ceph-mon[118807]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:06.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:06 smithi181 ceph-mon[105375]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:06.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:06 smithi163 ceph-mon[118807]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:06 smithi163 ceph-mon[101022]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:06.972 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled alertmanager update... 2023-10-09T03:53:07.784 DEBUG:teuthology.orchestra.run.smithi163:alertmanager.a> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T03:53:07.787 INFO:tasks.cephadm:Adding grafana.a on smithi181 2023-10-09T03:53:07.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi181=a' 2023-10-09T03:53:07.820 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:-- Logs begin at Mon 2023-10-09 03:31:37 UTC. -- 2023-10-09T03:53:08.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:07 smithi181 ceph-mon[105375]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:53:08.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:07 smithi181 ceph-mon[105375]: Saving service alertmanager spec with placement smithi163=a;count:1 2023-10-09T03:53:08.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:07 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:08.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:07 smithi181 ceph-mon[105375]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[118807]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[118807]: Saving service alertmanager spec with placement smithi163=a;count:1 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[118807]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[101022]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi163=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[101022]: Saving service alertmanager spec with placement smithi163=a;count:1 2023-10-09T03:53:08.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:08.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:07 smithi163 ceph-mon[101022]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:08.253 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:53:09.386 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:53:09 smithi181 bash[149940]: 4c5e87623c24c0c559a0f1053d9627fba476d7a3f3d3977f27502752b8d1c2bb 2023-10-09T03:53:09.386 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:53:09 smithi181 systemd[1]: Started Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:53:10.017 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled grafana update... 2023-10-09T03:53:10.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:10.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: ignoring --setuser ceph since I am not root 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: ignoring --setgroup ceph since I am not root 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 ceph-mgr[108232]: -- 172.21.15.181:0/75679826 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5639782389c0 con 0x563978210c00 2023-10-09T03:53:10.576 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: 2023-10-09T03:53:10.532+0000 7fe8950cd1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:53:10.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: ignoring --setuser ceph since I am not root 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: ignoring --setgroup ceph since I am not root 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 ceph-mgr[101474]: -- 172.21.15.163:0/4288799550 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555e7f5009c0 con 0x555e7f4d9000 2023-10-09T03:53:10.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: 2023-10-09T03:53:10.534+0000 7fabd08b91c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:53:10.593 DEBUG:teuthology.orchestra.run.smithi181:grafana.a> sudo journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T03:53:10.596 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T03:53:10.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T03:53:10.626 INFO:journalctl@ceph.grafana.a.smithi181.stdout:-- Logs begin at Mon 2023-10-09 03:30:54 UTC. -- 2023-10-09T03:53:10.948 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:10.961 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: 2023-10-09T03:53:10.619+0000 7fe8950cd1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:53:10.962 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: 2023-10-09T03:53:10.706+0000 7fe8950cd1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:53:10.962 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: 2023-10-09T03:53:10.797+0000 7fe8950cd1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:53:10.997 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: 2023-10-09T03:53:10.613+0000 7fabd08b91c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:53:10.997 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: 2023-10-09T03:53:10.705+0000 7fabd08b91c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:53:10.997 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: 2023-10-09T03:53:10.802+0000 7fabd08b91c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:53:10.997 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:10 smithi163 conmon[101450]: 2023-10-09T03:53:10.996+0000 7fabd08b91c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:53:11.263 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:10 smithi181 conmon[108147]: 2023-10-09T03:53:10.960+0000 7fe8950cd1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:53:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:11 smithi181 ceph-mon[105375]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T03:53:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:11 smithi181 ceph-mon[105375]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T03:53:11.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:11 smithi181 conmon[108147]: 2023-10-09T03:53:11.262+0000 7fe8950cd1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:53:11.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:11 smithi181 conmon[108147]: 2023-10-09T03:53:11.349+0000 7fe8950cd1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:53:11.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:11 smithi181 conmon[108147]: 2023-10-09T03:53:11.476+0000 7fe8950cd1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:53:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:11 smithi163 ceph-mon[101022]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T03:53:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:11 smithi163 ceph-mon[101022]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T03:53:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:11 smithi163 ceph-mon[118807]: from='mgr.14152 172.21.15.163:0/2146264268' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T03:53:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:11 smithi163 ceph-mon[118807]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T03:53:11.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:11 smithi163 conmon[101450]: 2023-10-09T03:53:11.327+0000 7fabd08b91c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:53:11.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:11 smithi163 conmon[101450]: 2023-10-09T03:53:11.423+0000 7fabd08b91c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:53:11.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:11 smithi163 conmon[101450]: 2023-10-09T03:53:11.558+0000 7fabd08b91c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:53:11.985 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:11 smithi181 conmon[108147]: 2023-10-09T03:53:11.569+0000 7fe8950cd1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:53:11.985 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:11 smithi181 conmon[108147]: 2023-10-09T03:53:11.983+0000 7fe8950cd1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:53:12.051 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:11 smithi163 conmon[101450]: 2023-10-09T03:53:11.653+0000 7fabd08b91c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:53:12.316 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:12 smithi163 conmon[101450]: 2023-10-09T03:53:12.076+0000 7fabd08b91c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:53:12.825 INFO:teuthology.orchestra.run.smithi163.stdout:[client.0] 2023-10-09T03:53:12.825 INFO:teuthology.orchestra.run.smithi163.stdout: key = AQAoeSNl33AAMRAAtM797Klt1gpN5U6kNRwXWw== 2023-10-09T03:53:12.876 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:12 smithi181 conmon[108147]: 2023-10-09T03:53:12.537+0000 7fe8950cd1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:53:12.876 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:12 smithi181 conmon[108147]: 2023-10-09T03:53:12.706+0000 7fe8950cd1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:53:13.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:12 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1245923478' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:13.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:12 smithi163 ceph-mon[118807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:13.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:12 smithi163 ceph-mon[118807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T03:53:13.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:12 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1245923478' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:13.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:12 smithi163 ceph-mon[101022]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:13.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:12 smithi163 ceph-mon[101022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T03:53:13.022 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:12 smithi163 conmon[101450]: 2023-10-09T03:53:12.674+0000 7fabd08b91c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:53:13.022 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:12 smithi163 conmon[101450]: 2023-10-09T03:53:12.849+0000 7fabd08b91c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:53:13.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:12 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1245923478' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:13.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:12 smithi181 ceph-mon[105375]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:12 smithi181 ceph-mon[105375]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T03:53:13.462 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:53:13.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T03:53:13.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T03:53:13.507 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T03:53:13.731 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:13 smithi163 conmon[101450]: 2023-10-09T03:53:13.640+0000 7fabd08b91c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:53:13.733 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:13 smithi181 conmon[108147]: 2023-10-09T03:53:13.460+0000 7fe8950cd1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:53:13.733 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:13 smithi181 conmon[108147]: 2023-10-09T03:53:13.552+0000 7fe8950cd1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:53:13.858 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T03:53:14.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:13 smithi181 conmon[108147]: 2023-10-09T03:53:13.732+0000 7fe8950cd1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:53:14.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:13 smithi163 conmon[101450]: 2023-10-09T03:53:13.730+0000 7fabd08b91c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:53:14.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:13 smithi163 conmon[101450]: 2023-10-09T03:53:13.900+0000 7fabd08b91c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:53:14.442 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:14 smithi181 conmon[108147]: 2023-10-09T03:53:14.040+0000 7fe8950cd1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:53:14.442 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:14 smithi181 conmon[108147]: 2023-10-09T03:53:14.136+0000 7fe8950cd1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:53:14.442 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:14 smithi181 conmon[108147]: 2023-10-09T03:53:14.440+0000 7fe8950cd1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:53:14.549 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:14 smithi163 conmon[101450]: 2023-10-09T03:53:14.174+0000 7fabd08b91c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:53:14.549 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:14 smithi163 conmon[101450]: 2023-10-09T03:53:14.261+0000 7fabd08b91c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:53:14.806 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:14 smithi163 conmon[101450]: 2023-10-09T03:53:14.548+0000 7fabd08b91c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:53:15.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:14 smithi181 conmon[108147]: 2023-10-09T03:53:14.699+0000 7fe8950cd1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:53:15.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:14 smithi163 conmon[101450]: 2023-10-09T03:53:14.805+0000 7fabd08b91c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:53:15.581 INFO:teuthology.orchestra.run.smithi181.stdout:[client.1] 2023-10-09T03:53:15.581 INFO:teuthology.orchestra.run.smithi181.stdout: key = AQAreSNlw4dxIhAAnKx+pA+2QZw2G6ek5no/dA== 2023-10-09T03:53:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:15 smithi181 ceph-mon[105375]: from='client.? 172.21.15.181:0/3536064085' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:15 smithi181 ceph-mon[105375]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:15 smithi181 ceph-mon[105375]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T03:53:15.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:15 smithi181 conmon[108147]: 2023-10-09T03:53:15.445+0000 7fe8950cd1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:53:15.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:15 smithi181 conmon[108147]: 2023-10-09T03:53:15.577+0000 7fe8950cd1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:53:15.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:15 smithi163 conmon[101450]: 2023-10-09T03:53:15.511+0000 7fabd08b91c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:53:15.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:15 smithi163 conmon[101450]: 2023-10-09T03:53:15.644+0000 7fabd08b91c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:53:15.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:15 smithi163 ceph-mon[101022]: from='client.? 172.21.15.181:0/3536064085' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:15.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:15 smithi163 ceph-mon[101022]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:15.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:15 smithi163 ceph-mon[101022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T03:53:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:15 smithi163 ceph-mon[118807]: from='client.? 172.21.15.181:0/3536064085' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:15 smithi163 ceph-mon[118807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T03:53:15.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:15 smithi163 ceph-mon[118807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T03:53:16.114 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:15 smithi181 conmon[108147]: 2023-10-09T03:53:15.773+0000 7fe8950cd1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:53:16.114 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:15 smithi181 conmon[108147]: 2023-10-09T03:53:15.893+0000 7fe8950cd1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:53:16.160 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:15 smithi163 conmon[101450]: 2023-10-09T03:53:15.832+0000 7fabd08b91c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:53:16.160 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:15 smithi163 conmon[101450]: 2023-10-09T03:53:15.946+0000 7fabd08b91c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:53:16.200 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2023-10-09T03:53:16.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T03:53:16.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T03:53:16.242 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T03:53:16.243 DEBUG:teuthology.orchestra.run.smithi163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T03:53:16.518 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:16 smithi181 conmon[108147]: 2023-10-09T03:53:16.112+0000 7fe8950cd1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:53:16.562 INFO:teuthology.misc.health.smithi163.stdout: 2023-10-09T03:53:16.562 INFO:teuthology.misc.health.smithi163.stdout:{"epoch":59,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T03:52:48.486446+0000","last_up_change":"2023-10-09T03:52:44.469504+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6803","nonce":3842166717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6805","nonce":3842166717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6809","nonce":3842166717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6807","nonce":3842166717}]},"public_addr":"172.21.15.163:6803/3842166717","cluster_addr":"172.21.15.163:6805/3842166717","heartbeat_back_addr":"172.21.15.163:6809/3842166717","heartbeat_front_addr":"172.21.15.163:6807/3842166717","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6811","nonce":147795175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6813","nonce":147795175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6817","nonce":147795175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6815","nonce":147795175}]},"public_addr":"172.21.15.163:6811/147795175","cluster_addr":"172.21.15.163:6813/147795175","heartbeat_back_addr":"172.21.15.163:6817/147795175","heartbeat_front_addr":"172.21.15.163:6815/147795175","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6819","nonce":2150896924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6821","nonce":2150896924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6825","nonce":2150896924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6823","nonce":2150896924}]},"public_addr":"172.21.15.163:6819/2150896924","cluster_addr":"172.21.15.163:6821/2150896924","heartbeat_back_addr":"172.21.15.163:6825/2150896924","heartbeat_front_addr":"172.21.15.163:6823/2150896924","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6827","nonce":1216700115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6829","nonce":1216700115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6833","nonce":1216700115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6831","nonce":1216700115}]},"public_addr":"172.21.15.163:6827/1216700115","cluster_addr":"172.21.15.163:6829/1216700115","heartbeat_back_addr":"172.21.15.163:6833/1216700115","heartbeat_front_addr":"172.21.15.163:6831/1216700115","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T03:53:16.575 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T03:53:16.575 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T03:53:16.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T03:53:16.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:16 smithi163 conmon[101450]: 2023-10-09T03:53:16.159+0000 7fabd08b91c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:53:16.937 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:17.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:16 smithi181 conmon[108147]: 2023-10-09T03:53:16.979+0000 7fe8950cd1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:53:17.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:16 smithi181 conmon[108147]: [09/Oct/2023:03:53:16] ENGINE Bus STARTING 2023-10-09T03:53:17.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:17 smithi181 conmon[108147]: CherryPy Checker: 2023-10-09T03:53:17.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:17 smithi181 conmon[108147]: The Application mounted at '' has an empty config. 2023-10-09T03:53:17.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:17 smithi181 conmon[108147]: 2023-10-09T03:53:17.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:17 smithi181 conmon[108147]: [09/Oct/2023:03:53:17] ENGINE Serving on http://:::9283 2023-10-09T03:53:17.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:17 smithi181 conmon[108147]: [09/Oct/2023:03:53:17] ENGINE Bus STARTED 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: 2023-10-09T03:53:17.073+0000 7fabd08b91c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: [09/Oct/2023:03:53:17] ENGINE Bus STARTING 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: CherryPy Checker: 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: The Application mounted at '' has an empty config. 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: [09/Oct/2023:03:53:17] ENGINE Serving on http://:::9283 2023-10-09T03:53:17.327 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:17 smithi163 conmon[101450]: [09/Oct/2023:03:53:17] ENGINE Bus STARTED 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3484630339' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: Standby manager daemon x restarted 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: Standby manager daemon x started 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:53:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: Active manager daemon y restarted 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: Activating manager daemon y 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: mgrmap e17: y(active, starting, since 0.00886504s), standbys: x 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:53:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: Manager daemon y is now available 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3484630339' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: Standby manager daemon x restarted 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: Standby manager daemon x started 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: Active manager daemon y restarted 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: Activating manager daemon y 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: mgrmap e17: y(active, starting, since 0.00886504s), standbys: x 2023-10-09T03:53:17.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:53:17.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: Manager daemon y is now available 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:53:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:53:17.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:53:17.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3484630339' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: Standby manager daemon x restarted 2023-10-09T03:53:17.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: Standby manager daemon x started 2023-10-09T03:53:17.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3859421984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: Active manager daemon y restarted 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: Activating manager daemon y 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: mgrmap e17: y(active, starting, since 0.00886504s), standbys: x 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:53:17.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:53:17.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: Manager daemon y is now available 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:53:17.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:53:19.109 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'rbd' created 2023-10-09T03:53:19.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: [09/Oct/2023:03:53:17] ENGINE Bus STARTING 2023-10-09T03:53:19.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: [09/Oct/2023:03:53:17] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:53:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: [09/Oct/2023:03:53:17] ENGINE Bus STARTED 2023-10-09T03:53:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: mgrmap e18: y(active, since 1.01281s), standbys: x 2023-10-09T03:53:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:18 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3377991482' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: [09/Oct/2023:03:53:17] ENGINE Bus STARTING 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: [09/Oct/2023:03:53:17] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: [09/Oct/2023:03:53:17] ENGINE Bus STARTED 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: mgrmap e18: y(active, since 1.01281s), standbys: x 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:19.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3377991482' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: [09/Oct/2023:03:53:17] ENGINE Bus STARTING 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: [09/Oct/2023:03:53:17] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: [09/Oct/2023:03:53:17] ENGINE Bus STARTED 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: mgrmap e18: y(active, since 1.01281s), standbys: x 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:19.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:18 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3377991482' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T03:53:19.818 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T03:53:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3377991482' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3377991482' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T03:53:20.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:20 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.236 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3377991482' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:53:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:20 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi181 to 4036M 2023-10-09T03:53:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:53:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:53:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:53:21.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:53:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: mgrmap e19: y(active, since 2s), standbys: x 2023-10-09T03:53:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T03:53:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:53:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:53:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi181 to 4036M 2023-10-09T03:53:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:53:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:53:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:53:21.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:53:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: mgrmap e19: y(active, since 2s), standbys: x 2023-10-09T03:53:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T03:53:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:53:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:53:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:21 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi181 to 4036M 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: mgrmap e19: y(active, since 2s), standbys: x 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:53:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:53:21.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:21.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:21 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:53:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:53:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:53:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: Deploying daemon alertmanager.a on smithi163 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2676594581' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:53:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[118807]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:53:22.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:53:22.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:53:22.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:53:22.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:53:22.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: Deploying daemon alertmanager.a on smithi163 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2676594581' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:53:22.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:22 smithi163 ceph-mon[101022]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:53:22.380 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T03:53:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:53:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:53:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:53:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:53:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:22.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: Deploying daemon alertmanager.a on smithi163 2023-10-09T03:53:22.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:22.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T03:53:22.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2676594581' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:53:22.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:22 smithi181 ceph-mon[105375]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:53:23.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:23 smithi163 ceph-mon[101022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:53:23.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:23 smithi163 ceph-mon[101022]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T03:53:23.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:23 smithi163 ceph-mon[101022]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T03:53:23.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:23 smithi163 ceph-mon[118807]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:53:23.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:23 smithi163 ceph-mon[118807]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T03:53:23.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:23 smithi163 ceph-mon[118807]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T03:53:23.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:23 smithi181 ceph-mon[105375]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:53:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:23 smithi181 ceph-mon[105375]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T03:53:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:23 smithi181 ceph-mon[105375]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T03:53:23.544 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T03:53:23.544 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T03:53:23.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T03:53:24.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:24 smithi163 ceph-mon[118807]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:24.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:24 smithi163 ceph-mon[101022]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:24.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:24 smithi181 ceph-mon[105375]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:25.178 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:25.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:25 smithi163 ceph-mon[118807]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:53:25.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:25 smithi163 ceph-mon[118807]: Cluster is now healthy 2023-10-09T03:53:25.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:25 smithi163 ceph-mon[101022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:53:25.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:25 smithi163 ceph-mon[101022]: Cluster is now healthy 2023-10-09T03:53:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:25 smithi181 ceph-mon[105375]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:53:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:25 smithi181 ceph-mon[105375]: Cluster is now healthy 2023-10-09T03:53:26.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:25 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:53:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:53:26.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:26 smithi163 ceph-mon[118807]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:26.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:26 smithi163 ceph-mon[101022]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:26.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:26 smithi181 ceph-mon[105375]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:28.370 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:53:28.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:28 smithi163 ceph-mon[118807]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:28 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3224109272' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T03:53:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:28 smithi163 ceph-mon[101022]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:28 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3224109272' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T03:53:28.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:28 smithi181 ceph-mon[105375]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:28.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:28 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3224109272' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T03:53:29.267 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":19,"active_gid":14457,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":55760003},{"type":"v1","addr":"172.21.15.163:6801","nonce":55760003}]},"active_addr":"172.21.15.163:6801/55760003","active_change":"2023-10-09T03:53:17.088335+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24364,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":2462309240}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1847962187}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3524034575}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1361101496}]}]} 2023-10-09T03:53:29.271 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T03:53:29.272 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T03:53:29.272 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T03:53:29.582 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 systemd[1]: Starting Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:53:29.960 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:30.198 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:30 smithi163 conmon[101450]: [09/Oct/2023:03:53:30] ENGINE Bus STOPPING 2023-10-09T03:53:30.198 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:30 smithi163 conmon[101450]: [09/Oct/2023:03:53:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:53:30.198 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:30 smithi163 conmon[101450]: [09/Oct/2023:03:53:30] ENGINE Bus STOPPED 2023-10-09T03:53:30.198 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:30 smithi163 conmon[101450]: [09/Oct/2023:03:53:30] ENGINE Bus STARTING 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.944Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.944Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.945Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.946Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 bash[165978]: ce053ea1fbdc7736d9a9ad353c8d944b93218afa50bff1c724633f447e493f7a 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 systemd[1]: Started Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.973Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.973Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.977Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T03:53:30.199 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:29 smithi163 conmon[166098]: ts=2023-10-09T03:53:29.977Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T03:53:30.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:30 smithi163 conmon[101450]: [09/Oct/2023:03:53:30] ENGINE Serving on http://:::9283 2023-10-09T03:53:30.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:53:30 smithi163 conmon[101450]: [09/Oct/2023:03:53:30] ENGINE Bus STARTED 2023-10-09T03:53:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T03:53:30.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:30 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:30.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T03:53:30.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:30 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:31.411 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:31 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T03:53:31.412 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:31 smithi163 ceph-mon[118807]: Deploying daemon grafana.a on smithi181 2023-10-09T03:53:31.412 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:31 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T03:53:31.412 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:31 smithi163 ceph-mon[101022]: Deploying daemon grafana.a on smithi181 2023-10-09T03:53:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:31 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T03:53:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:31 smithi181 ceph-mon[105375]: Deploying daemon grafana.a on smithi181 2023-10-09T03:53:31.819 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:53:31.819 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":64,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T03:53:22.366174+0000","last_up_change":"2023-10-09T03:52:44.469504+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6803","nonce":3842166717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6805","nonce":3842166717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6809","nonce":3842166717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6807","nonce":3842166717}]},"public_addr":"172.21.15.163:6803/3842166717","cluster_addr":"172.21.15.163:6805/3842166717","heartbeat_back_addr":"172.21.15.163:6809/3842166717","heartbeat_front_addr":"172.21.15.163:6807/3842166717","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6811","nonce":147795175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6813","nonce":147795175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6817","nonce":147795175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6815","nonce":147795175}]},"public_addr":"172.21.15.163:6811/147795175","cluster_addr":"172.21.15.163:6813/147795175","heartbeat_back_addr":"172.21.15.163:6817/147795175","heartbeat_front_addr":"172.21.15.163:6815/147795175","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6819","nonce":2150896924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6821","nonce":2150896924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6825","nonce":2150896924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6823","nonce":2150896924}]},"public_addr":"172.21.15.163:6819/2150896924","cluster_addr":"172.21.15.163:6821/2150896924","heartbeat_back_addr":"172.21.15.163:6825/2150896924","heartbeat_front_addr":"172.21.15.163:6823/2150896924","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6827","nonce":1216700115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6829","nonce":1216700115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6833","nonce":1216700115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6831","nonce":1216700115}]},"public_addr":"172.21.15.163:6827/1216700115","cluster_addr":"172.21.15.163:6829/1216700115","heartbeat_back_addr":"172.21.15.163:6833/1216700115","heartbeat_front_addr":"172.21.15.163:6831/1216700115","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T03:53:32.331 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:31 smithi163 conmon[166098]: ts=2023-10-09T03:53:31.946Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00026408s 2023-10-09T03:53:32.401 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T03:53:32.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T03:53:32.746 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:32 smithi181 ceph-mon[105375]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:32 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2558273784' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:32 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[101022]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2558273784' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[118807]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2558273784' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:32 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:34.325 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:53:34.325 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":64,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T03:53:22.366174+0000","last_up_change":"2023-10-09T03:52:44.469504+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6803","nonce":3842166717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6805","nonce":3842166717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6809","nonce":3842166717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6807","nonce":3842166717}]},"public_addr":"172.21.15.163:6803/3842166717","cluster_addr":"172.21.15.163:6805/3842166717","heartbeat_back_addr":"172.21.15.163:6809/3842166717","heartbeat_front_addr":"172.21.15.163:6807/3842166717","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6811","nonce":147795175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6813","nonce":147795175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6817","nonce":147795175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6815","nonce":147795175}]},"public_addr":"172.21.15.163:6811/147795175","cluster_addr":"172.21.15.163:6813/147795175","heartbeat_back_addr":"172.21.15.163:6817/147795175","heartbeat_front_addr":"172.21.15.163:6815/147795175","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6819","nonce":2150896924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6821","nonce":2150896924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6825","nonce":2150896924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6823","nonce":2150896924}]},"public_addr":"172.21.15.163:6819/2150896924","cluster_addr":"172.21.15.163:6821/2150896924","heartbeat_back_addr":"172.21.15.163:6825/2150896924","heartbeat_front_addr":"172.21.15.163:6823/2150896924","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6827","nonce":1216700115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6829","nonce":1216700115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6833","nonce":1216700115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6831","nonce":1216700115}]},"public_addr":"172.21.15.163:6827/1216700115","cluster_addr":"172.21.15.163:6829/1216700115","heartbeat_back_addr":"172.21.15.163:6833/1216700115","heartbeat_front_addr":"172.21.15.163:6831/1216700115","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T03:53:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:34 smithi181 ceph-mon[105375]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:34 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3787727637' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:34.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:34 smithi163 ceph-mon[118807]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:34 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3787727637' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:34 smithi163 ceph-mon[101022]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:34 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3787727637' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:53:34.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T03:53:34.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T03:53:34.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T03:53:34.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T03:53:34.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T03:53:34.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T03:53:34.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T03:53:34.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T03:53:35.984 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:36.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:35 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:53:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:53:36.036 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:36.240 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:36.338 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:36.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:36 smithi163 ceph-mon[118807]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:36.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:36 smithi163 ceph-mon[101022]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:36.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:36 smithi181 ceph-mon[105375]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:38.148 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:38.148 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:38.270 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:38.427 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:38.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:38 smithi163 ceph-mon[101022]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:38.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:38 smithi163 ceph-mon[118807]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:38 smithi181 ceph-mon[105375]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:40.202 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:53:39 smithi163 conmon[166098]: ts=2023-10-09T03:53:39.948Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002047053s 2023-10-09T03:53:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:40 smithi163 ceph-mon[118807]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:40.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:40 smithi163 ceph-mon[101022]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:40 smithi181 ceph-mon[105375]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:41.798 INFO:teuthology.orchestra.run.smithi163.stdout:167503724565 2023-10-09T03:53:41.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T03:53:41.965 INFO:teuthology.orchestra.run.smithi163.stdout:77309411363 2023-10-09T03:53:41.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T03:53:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:42 smithi163 ceph-mon[118807]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:42.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:42 smithi163 ceph-mon[101022]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:42 smithi181 ceph-mon[105375]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:44.000 INFO:teuthology.orchestra.run.smithi163.stdout:236223201292 2023-10-09T03:53:44.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T03:53:44.039 INFO:teuthology.orchestra.run.smithi163.stdout:34359738412 2023-10-09T03:53:44.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T03:53:44.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:44 smithi163 ceph-mon[101022]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:44.652 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:44 smithi163 ceph-mon[118807]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:44 smithi181 ceph-mon[105375]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:44.813 INFO:teuthology.orchestra.run.smithi163.stdout:111669149727 2023-10-09T03:53:44.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T03:53:45.746 INFO:teuthology.orchestra.run.smithi163.stdout:201863462930 2023-10-09T03:53:45.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T03:53:46.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:45 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:53:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:53:46.178 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:46.461 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:46.571 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:46.652 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:46 smithi163 ceph-mon[118807]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:46.652 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:46 smithi163 ceph-mon[101022]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:46.653 INFO:teuthology.orchestra.run.smithi163.stdout:55834574888 2023-10-09T03:53:46.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T03:53:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:46 smithi181 ceph-mon[105375]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:47.137 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:47.617 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:47 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:47.617 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:47 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:47.653 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:47 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:53:48.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:48 smithi163 ceph-mon[118807]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:48.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:48 smithi163 ceph-mon[101022]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:48 smithi181 ceph-mon[105375]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:48.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:48 smithi181 systemd[1]: Starting Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:53:49.061 INFO:teuthology.orchestra.run.smithi163.stdout:77309411364 2023-10-09T03:53:49.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 bash[153180]: be20d4ebd53fee5217392a14b66b184c802b899f6eff9e171709f913fa54552b 2023-10-09T03:53:49.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 systemd[1]: Started Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:53:49.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.27581114Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T03:53:49.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.275991818Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T03:53:49.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276008728Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T03:53:49.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276015363Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276021086Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276027048Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276032453Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276037984Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276043517Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276048938Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276054098Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276059344Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T03:53:49.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276064779Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=settings t=2023-10-09T03:53:49.276070261Z level=info msg="App mode production" 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=sqlstore t=2023-10-09T03:53:49.276170441Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=sqlstore t=2023-10-09T03:53:49.276187155Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.277077039Z level=info msg="Starting DB migrations" 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.277155254Z level=info msg="Executing migration" id="create migration_log table" 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.330779715Z level=info msg="Executing migration" id="create user table" 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.355940901Z level=info msg="Executing migration" id="add unique index user.login" 2023-10-09T03:53:49.416 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.380977002Z level=info msg="Executing migration" id="add unique index user.email" 2023-10-09T03:53:49.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2219577989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T03:53:49.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.418 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.683 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:49 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2219577989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T03:53:49.683 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.683 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.683 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.683 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.431330403Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-10-09T03:53:49.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.465234662Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-10-09T03:53:49.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.490147071Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-10-09T03:53:49.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.51557716Z level=info msg="Executing migration" id="create user table v2" 2023-10-09T03:53:49.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.557419413Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-10-09T03:53:49.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.607671622Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-10-09T03:53:49.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.632874682Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-10-09T03:53:49.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.658067331Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-10-09T03:53:49.684 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.68325879Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-10-09T03:53:49.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2219577989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T03:53:49.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:49.935 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.716989803Z level=info msg="Executing migration" id="Update user table charset" 2023-10-09T03:53:49.935 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.758693739Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-10-09T03:53:49.935 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.792303825Z level=info msg="Executing migration" id="Add missing user data" 2023-10-09T03:53:49.935 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.809130556Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-10-09T03:53:49.936 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.825920003Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-10-09T03:53:49.936 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.842997236Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-10-09T03:53:49.936 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.859750588Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-10-09T03:53:49.936 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.910227681Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-10-09T03:53:49.936 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.935696826Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-10-09T03:53:50.187 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.960774946Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T03:53:50.187 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:49 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:49.986007533Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-10-09T03:53:50.187 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.027984845Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-10-09T03:53:50.187 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.053335773Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-10-09T03:53:50.188 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.078408817Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-10-09T03:53:50.188 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.10355323Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-10-09T03:53:50.188 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.137185376Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-10-09T03:53:50.188 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.162437997Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-10-09T03:53:50.188 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.1875874Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T03:53:50.199 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411364 for osd.2 2023-10-09T03:53:50.199 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:50.298 INFO:teuthology.orchestra.run.smithi163.stdout:133143986202 2023-10-09T03:53:50.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T03:53:50.447 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:50 smithi181 ceph-mon[105375]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:50.447 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:50 smithi181 ceph-mon[105375]: Deploying daemon node-exporter.a on smithi163 2023-10-09T03:53:50.448 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.21287506Z level=info msg="Executing migration" id="create temp_user v2" 2023-10-09T03:53:50.448 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.238105017Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-10-09T03:53:50.448 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.263423072Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-10-09T03:53:50.448 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.305302501Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-10-09T03:53:50.448 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.33034627Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-10-09T03:53:50.449 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.355625712Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-10-09T03:53:50.449 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.397491611Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-10-09T03:53:50.449 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.414281142Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T03:53:50.449 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.447962955Z level=info msg="Executing migration" id="create star table" 2023-10-09T03:53:50.544 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:50 smithi163 ceph-mon[118807]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:50 smithi163 ceph-mon[118807]: Deploying daemon node-exporter.a on smithi163 2023-10-09T03:53:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:50 smithi163 ceph-mon[101022]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:50 smithi163 ceph-mon[101022]: Deploying daemon node-exporter.a on smithi163 2023-10-09T03:53:50.585 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:50.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.490037444Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-10-09T03:53:50.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.507108298Z level=info msg="Executing migration" id="create org table v1" 2023-10-09T03:53:50.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.540712537Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-10-09T03:53:50.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.607463727Z level=info msg="Executing migration" id="create org_user table v1" 2023-10-09T03:53:50.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.624269738Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-10-09T03:53:50.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.64111003Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T03:53:50.734 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.666445649Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-10-09T03:53:50.734 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.69146717Z level=info msg="Executing migration" id="Update org table charset" 2023-10-09T03:53:50.734 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.733403503Z level=info msg="Executing migration" id="Update org_user table charset" 2023-10-09T03:53:50.744 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:51.002 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.775628459Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-10-09T03:53:51.003 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.817582171Z level=info msg="Executing migration" id="create dashboard table" 2023-10-09T03:53:51.003 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.859419885Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-10-09T03:53:51.003 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.901506212Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-10-09T03:53:51.003 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.943287792Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-10-09T03:53:51.004 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:50 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:50.968548254Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T03:53:51.004 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.002282755Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T03:53:51.068 INFO:teuthology.orchestra.run.smithi163.stdout:167503724567 2023-10-09T03:53:51.268 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.052658483Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T03:53:51.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.094599616Z level=info msg="Executing migration" id="create dashboard v2" 2023-10-09T03:53:51.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.136540624Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-10-09T03:53:51.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.161638395Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T03:53:51.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.203690164Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-10-09T03:53:51.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.228704366Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-10-09T03:53:51.290 INFO:teuthology.orchestra.run.smithi163.stdout:34359738413 2023-10-09T03:53:51.335 INFO:teuthology.orchestra.run.smithi163.stdout:236223201294 2023-10-09T03:53:51.555 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:51 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3699224723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T03:53:51.556 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:51 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1886325282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T03:53:51.556 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:51 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1562828435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T03:53:51.556 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.270659207Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-10-09T03:53:51.556 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.295958104Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-10-09T03:53:51.556 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.321125567Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-10-09T03:53:51.557 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.363090152Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-10-09T03:53:51.557 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.404974054Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-10-09T03:53:51.557 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.446947553Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-10-09T03:53:51.557 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.472115273Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-10-09T03:53:51.557 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.514050094Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-10-09T03:53:51.557 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.555944986Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-10-09T03:53:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:51 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3699224723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T03:53:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:51 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1886325282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T03:53:51.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:51 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1562828435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T03:53:51.741 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:51 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3699224723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T03:53:51.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:51 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1886325282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T03:53:51.742 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:51 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1562828435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T03:53:51.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.598084585Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-10-09T03:53:51.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.623174902Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-10-09T03:53:51.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.648469141Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-10-09T03:53:51.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.67355018Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-10-09T03:53:51.816 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.698833585Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-10-09T03:53:51.817 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.723881161Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-10-09T03:53:51.817 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.749140303Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-10-09T03:53:51.817 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.77441023Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-10-09T03:53:51.817 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.81631728Z level=info msg="Executing migration" id="Update dashboard title length" 2023-10-09T03:53:51.961 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724565 got 167503724567 for osd.5 2023-10-09T03:53:51.961 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:52.059 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:52.067 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.858514175Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.900570232Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.925609055Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.950829052Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:51 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:51.975937031Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.001115199Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.042947959Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-10-09T03:53:52.068 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.068199043Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T03:53:52.353 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.102012354Z level=info msg="Executing migration" id="Add check_sum column" 2023-10-09T03:53:52.353 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.152410142Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-10-09T03:53:52.353 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.194222891Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-10-09T03:53:52.353 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.236382232Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-10-09T03:53:52.354 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.269889394Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-10-09T03:53:52.354 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.286815184Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-10-09T03:53:52.613 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:52 smithi181 ceph-mon[105375]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:52.614 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:52 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:52.614 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.353592626Z level=info msg="Executing migration" id="create data_source table" 2023-10-09T03:53:52.614 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.387479707Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-10-09T03:53:52.614 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.404236095Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-10-09T03:53:52.614 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.437832187Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-10-09T03:53:52.614 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.454689174Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T03:53:52.614 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.496767465Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-10-09T03:53:52.615 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.538611791Z level=info msg="Executing migration" id="create data_source table v2" 2023-10-09T03:53:52.615 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.5637901Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-10-09T03:53:52.615 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.588972459Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-10-09T03:53:52.615 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.614024771Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-10-09T03:53:52.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:52 smithi163 ceph-mon[118807]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:52.997 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:52 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:52.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:52 smithi163 ceph-mon[101022]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:52.998 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:52 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:52.998 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.664701117Z level=info msg="Executing migration" id="Add column with_credentials" 2023-10-09T03:53:52.998 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.689881024Z level=info msg="Executing migration" id="Add secure json data column" 2023-10-09T03:53:52.999 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.715014394Z level=info msg="Executing migration" id="Update data_source table charset" 2023-10-09T03:53:52.999 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.740245633Z level=info msg="Executing migration" id="Update initial version to 1" 2023-10-09T03:53:52.999 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.765591957Z level=info msg="Executing migration" id="Add read_only data column" 2023-10-09T03:53:52.999 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.807450158Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-10-09T03:53:52.999 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.849173046Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-10-09T03:53:53.000 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.891094565Z level=info msg="Executing migration" id="Add uid column" 2023-10-09T03:53:53.074 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738412 got 34359738413 for osd.0 2023-10-09T03:53:53.074 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:53.159 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.924991394Z level=info msg="Executing migration" id="Update uid value" 2023-10-09T03:53:53.160 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.950065668Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-10-09T03:53:53.160 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:52 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:52.975326842Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-10-09T03:53:53.160 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.017138803Z level=info msg="Executing migration" id="create api_key table" 2023-10-09T03:53:53.160 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.042499544Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-10-09T03:53:53.160 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.067642928Z level=info msg="Executing migration" id="add index api_key.key" 2023-10-09T03:53:53.161 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.092711339Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-10-09T03:53:53.161 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.13463032Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-10-09T03:53:53.161 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.159898073Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-10-09T03:53:53.304 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201292 got 236223201294 for osd.7 2023-10-09T03:53:53.304 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:53.420 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.210834581Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T03:53:53.420 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.252645462Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-10-09T03:53:53.421 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.277887097Z level=info msg="Executing migration" id="create api_key table v2" 2023-10-09T03:53:53.421 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.319787743Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-10-09T03:53:53.421 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.344924224Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-10-09T03:53:53.421 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.370178968Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-10-09T03:53:53.421 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.395337557Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-10-09T03:53:53.421 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.420615324Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-10-09T03:53:53.582 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:53 smithi163 bash[172900]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T03:53:53.680 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.462511726Z level=info msg="Executing migration" id="Update api_key table charset" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.487624457Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.512931842Z level=info msg="Executing migration" id="Add service account foreign key" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.554798972Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.579957041Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.605132728Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.630395646Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.655536913Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-10-09T03:53:53.681 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.68088497Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-10-09T03:53:53.940 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.714541652Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.739631644Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.781510912Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.806837325Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.831952621Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.857018835Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.899186957Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-10-09T03:53:53.941 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.940973573Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T03:53:54.081 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:53 smithi163 bash[172900]: Getting image source signatures 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:53 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:53.983032928Z level=info msg="Executing migration" id="create quota table v1" 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.000078606Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.058596155Z level=info msg="Executing migration" id="Update quota table charset" 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.075265147Z level=info msg="Executing migration" id="create plugin_setting table" 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.092154308Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.117641691Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-10-09T03:53:54.226 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.142621052Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-10-09T03:53:54.227 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.18461853Z level=info msg="Executing migration" id="create session table" 2023-10-09T03:53:54.227 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.226404347Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-10-09T03:53:54.480 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:54 smithi181 ceph-mon[105375]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:54.480 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.276954271Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.321124665Z level=info msg="Executing migration" id="create playlist table v2" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.346151934Z level=info msg="Executing migration" id="create playlist item table v2" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.371540813Z level=info msg="Executing migration" id="Update playlist table charset" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.396561324Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.413462592Z level=info msg="Executing migration" id="drop preferences table v2" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.447131974Z level=info msg="Executing migration" id="drop preferences table v3" 2023-10-09T03:53:54.481 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.48066756Z level=info msg="Executing migration" id="create preferences table v3" 2023-10-09T03:53:54.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:54 smithi163 ceph-mon[101022]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:54.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:54 smithi163 ceph-mon[118807]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:54.518 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:54 smithi163 bash[172900]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T03:53:54.518 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:54 smithi163 bash[172900]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T03:53:54.518 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:54 smithi163 bash[172900]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T03:53:54.700 INFO:teuthology.orchestra.run.smithi163.stdout:201863462932 2023-10-09T03:53:54.732 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.514366947Z level=info msg="Executing migration" id="Update preferences table charset" 2023-10-09T03:53:54.732 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.539615019Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.564909968Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.589917741Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.615243633Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.640358035Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.665565873Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.690872327Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-10-09T03:53:54.733 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.732775944Z level=info msg="Executing migration" id="create alert table v1" 2023-10-09T03:53:54.992 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.783036808Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-10-09T03:53:54.992 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.808448061Z level=info msg="Executing migration" id="add index alert state" 2023-10-09T03:53:54.992 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.83370266Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-10-09T03:53:54.993 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.858873061Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-10-09T03:53:54.993 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.900675032Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T03:53:54.993 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.942577923Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T03:53:54.993 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.967845993Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T03:53:54.993 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:54 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:54.992849897Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-10-09T03:53:55.082 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:54 smithi163 bash[172900]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T03:53:55.252 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.04357756Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T03:53:55.253 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.093799052Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-10-09T03:53:55.253 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.118957896Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-10-09T03:53:55.253 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.144108574Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-10-09T03:53:55.253 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.185967474Z level=info msg="Executing migration" id="Add column is_default" 2023-10-09T03:53:55.253 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.227799836Z level=info msg="Executing migration" id="Add column frequency" 2023-10-09T03:53:55.254 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.252975691Z level=info msg="Executing migration" id="Add column send_reminder" 2023-10-09T03:53:55.513 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:55 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/600524163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T03:53:55.513 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.278259967Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-10-09T03:53:55.513 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.320069371Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-10-09T03:53:55.513 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.345287155Z level=info msg="Executing migration" id="Update alert table charset" 2023-10-09T03:53:55.513 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.370430435Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-10-09T03:53:55.513 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.395646635Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-10-09T03:53:55.514 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.44607176Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T03:53:55.514 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.471298207Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-10-09T03:53:55.514 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.51331101Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-10-09T03:53:55.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:55 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/600524163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T03:53:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:55 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/600524163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T03:53:55.582 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:55 smithi163 bash[172900]: Writing manifest to image destination 2023-10-09T03:53:55.582 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:55 smithi163 bash[172900]: Storing signatures 2023-10-09T03:53:55.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:53:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:53:55.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.555237253Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T03:53:55.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.597126582Z level=info msg="Executing migration" id="Add for to alert table" 2023-10-09T03:53:55.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.622445613Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-10-09T03:53:55.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.647539413Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-10-09T03:53:55.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.689460137Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-10-09T03:53:55.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.731296655Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.790213014Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.815272191Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.840625857Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.882400947Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.924136764Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.949420895Z level=info msg="Executing migration" id="create annotation table v5" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.974773246Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:55 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:55.999859241Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-10-09T03:53:56.050 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.025029924Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-10-09T03:53:56.051 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.050103721Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-10-09T03:53:56.274 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462932 for osd.6 2023-10-09T03:53:56.274 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.100614947Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.125805229Z level=info msg="Executing migration" id="Update annotation table charset" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.150953914Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.176165169Z level=info msg="Executing migration" id="Drop category_id index" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.217788433Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.243009976Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.268390695Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T03:53:56.335 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.293476961Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T03:53:56.336 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.33526586Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T03:53:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:56 smithi163 ceph-mon[101022]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:56 smithi163 ceph-mon[118807]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:56.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:56 smithi181 ceph-mon[105375]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:56.595 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.369152979Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-10-09T03:53:56.595 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.410963771Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T03:53:56.595 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.452768668Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-10-09T03:53:56.595 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.477890909Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-10-09T03:53:56.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.503088083Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-10-09T03:53:56.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.528272842Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-10-09T03:53:56.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.570254926Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-10-09T03:53:56.596 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.595568649Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-10-09T03:53:56.847 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.620735292Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-10-09T03:53:56.847 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.662906637Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-10-09T03:53:56.847 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.688148005Z level=info msg="Executing migration" id="Add epoch_end column" 2023-10-09T03:53:56.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.713354478Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-10-09T03:53:56.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.755208646Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-10-09T03:53:56.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.780359903Z level=info msg="Executing migration" id="Move region to single row" 2023-10-09T03:53:56.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.822227008Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-10-09T03:53:56.848 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.84744953Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T03:53:57.098 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.906198559Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T03:53:57.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.956460204Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T03:53:57.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:56 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:56.981549096Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T03:53:57.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.006832491Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-10-09T03:53:57.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.031857499Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-10-09T03:53:57.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.057034306Z level=info msg="Executing migration" id="create test_data table" 2023-10-09T03:53:57.099 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.099117149Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-10-09T03:53:57.139 INFO:teuthology.orchestra.run.smithi163.stdout:111669149729 2023-10-09T03:53:57.280 INFO:teuthology.orchestra.run.smithi163.stdout:55834574890 2023-10-09T03:53:57.350 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.124358506Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-10-09T03:53:57.350 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.166175888Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T03:53:57.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.191348909Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-10-09T03:53:57.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.233306137Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-10-09T03:53:57.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.275359788Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-10-09T03:53:57.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.292151113Z level=info msg="Executing migration" id="create team table" 2023-10-09T03:53:57.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.325607725Z level=info msg="Executing migration" id="add index team.org_id" 2023-10-09T03:53:57.351 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.35090212Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-10-09T03:53:57.602 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:57 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/746423252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T03:53:57.602 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:57 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/545508907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.401342944Z level=info msg="Executing migration" id="create team member table" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.418171116Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.451666572Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.468721299Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.527253631Z level=info msg="Executing migration" id="Add column email to team table" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.552419726Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.585948826Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-10-09T03:53:57.603 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.602885362Z level=info msg="Executing migration" id="create dashboard acl table" 2023-10-09T03:53:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:57 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/746423252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T03:53:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:57 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/545508907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T03:53:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:57 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/746423252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T03:53:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:57 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/545508907' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T03:53:57.863 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.64482067Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.661861171Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.695396957Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.712201208Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.728974715Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.745865913Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.76298096Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.779988009Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.813465113Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.830215809Z level=info msg="Executing migration" id="create tag table" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.846983218Z level=info msg="Executing migration" id="add index tag.key_value" 2023-10-09T03:53:57.864 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.863879122Z level=info msg="Executing migration" id="create login attempt table" 2023-10-09T03:53:58.144 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.897889944Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-10-09T03:53:58.144 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.931307521Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-10-09T03:53:58.144 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.956640451Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T03:53:58.144 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:57 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:57.998400271Z level=info msg="Executing migration" id="create login_attempt v2" 2023-10-09T03:53:58.144 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.023642154Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-10-09T03:53:58.145 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.048782497Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-10-09T03:53:58.145 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.073990463Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-10-09T03:53:58.145 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.099202809Z level=info msg="Executing migration" id="create user auth table" 2023-10-09T03:53:58.145 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.141242254Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T03:53:58.405 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.183400089Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.208405633Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.225261198Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.242143264Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.258925587Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.292582883Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.309465003Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.334756419Z level=info msg="Executing migration" id="create server_lock table" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.35146396Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.368291415Z level=info msg="Executing migration" id="create user auth token table" 2023-10-09T03:53:58.406 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.385116536Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-10-09T03:53:58.407 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.402086483Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-10-09T03:53:58.413 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149727 got 111669149729 for osd.3 2023-10-09T03:53:58.413 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:58.505 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574888 got 55834574890 for osd.1 2023-10-09T03:53:58.505 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:58.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:58 smithi181 ceph-mon[105375]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.444151028Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.460940713Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.477749722Z level=info msg="Executing migration" id="create cache_data table" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.494826799Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.511780708Z level=info msg="Executing migration" id="create short_url table v1" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.553601043Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.578696364Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.620735591Z level=info msg="Executing migration" id="delete alert_definition table" 2023-10-09T03:53:58.699 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.645747113Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-10-09T03:53:58.700 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.696326871Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-10-09T03:53:58.751 INFO:teuthology.orchestra.run.smithi163.stdout:133143986204 2023-10-09T03:53:58.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:58 smithi163 ceph-mon[118807]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:58.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:58 smithi163 ceph-mon[101022]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:58.771 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.560Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T03:53:58.771 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.560Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.561Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.561Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.561Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.561Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T03:53:58.772 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T03:53:58.773 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T03:53:58.774 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T03:53:58.775 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T03:53:58.776 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T03:53:58.776 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 conmon[173601]: ts=2023-10-09T03:53:58.562Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T03:53:58.776 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 bash[172900]: 8d858ae3dede3398a4617742d65ce069162aff4ac8f322906f126c2a52df8407 2023-10-09T03:53:58.776 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:53:58 smithi163 systemd[1]: Started Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:53:58.981 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.746841403Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.772165064Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.814147169Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.839252656Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.864461664Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.898388573Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.940040157Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-10-09T03:53:58.982 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:58 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:58.981781788Z level=info msg="Executing migration" id="drop alert_definition table" 2023-10-09T03:53:59.259 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.032536394Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-10-09T03:53:59.259 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.065838256Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-10-09T03:53:59.259 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.082816132Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T03:53:59.259 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.116662784Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T03:53:59.259 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.158556676Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T03:53:59.259 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.183733738Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-10-09T03:53:59.260 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.259259677Z level=info msg="Executing migration" id="create alert_instance table" 2023-10-09T03:53:59.278 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986202 got 133143986204 for osd.4 2023-10-09T03:53:59.278 DEBUG:teuthology.parallel:result is None 2023-10-09T03:53:59.279 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T03:53:59.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T03:53:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.309741935Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.35158029Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.376739222Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.41862725Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.443837314Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.468949414Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.494091231Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-10-09T03:53:59.520 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:53:59 smithi181 systemd[1]: Starting Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:53:59.622 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:53:59.772 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:53:59 smithi181 bash[153652]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T03:53:59.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:59 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:59.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:59 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:59.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:59 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:53:59.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:59 smithi181 ceph-mon[105375]: Deploying daemon node-exporter.b on smithi181 2023-10-09T03:53:59.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:59 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/358215181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T03:53:59.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:53:59 smithi181 ceph-mon[105375]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.519710935Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.544655092Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.586579314Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.628670645Z level=info msg="Executing migration" id="create alert_rule table" 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.653833795Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.6958846Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-10-09T03:53:59.773 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.737727334Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T03:53:59.774 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.771644409Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.822107214Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.847261953Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.889066536Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.93113991Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.956173524Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:53:59 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:53:59.99816498Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-10-09T03:54:00.023 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.023547041Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-10-09T03:54:00.029 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[118807]: Deploying daemon node-exporter.b on smithi181 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/358215181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[118807]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:00.030 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[101022]: Deploying daemon node-exporter.b on smithi181 2023-10-09T03:54:00.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/358215181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T03:54:00.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:53:59 smithi163 ceph-mon[101022]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:00.292 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:00 smithi181 bash[153652]: Getting image source signatures 2023-10-09T03:54:00.292 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.073930057Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T03:54:00.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.099040986Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-10-09T03:54:00.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.140940184Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-10-09T03:54:00.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.166239711Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-10-09T03:54:00.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.208026704Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-10-09T03:54:00.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.249936814Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T03:54:00.293 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.291923044Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T03:54:00.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.36094556Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T03:54:00.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.452587607Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-10-09T03:54:00.597 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.50983171Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-10-09T03:54:00.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:00 smithi181 bash[153652]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T03:54:00.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:00 smithi181 bash[153652]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T03:54:00.597 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:00 smithi181 bash[153652]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T03:54:00.861 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.597286114Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-10-09T03:54:00.861 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.635564151Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-10-09T03:54:00.861 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.711179171Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-10-09T03:54:00.861 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.786305492Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-10-09T03:54:00.861 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.861577593Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-10-09T03:54:01.116 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:00 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:00.936881836Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-10-09T03:54:01.116 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.04160379Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T03:54:01.116 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.110789392Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-10-09T03:54:01.116 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:00 smithi181 bash[153652]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T03:54:01.117 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:01 smithi181 bash[153652]: Writing manifest to image destination 2023-10-09T03:54:01.242 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:01.245 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T03:54:01.369 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:01 smithi181 bash[153652]: Storing signatures 2023-10-09T03:54:01.369 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.180369665Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-10-09T03:54:01.369 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.205729332Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-10-09T03:54:01.370 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.247805741Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-10-09T03:54:01.370 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.315646268Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-10-09T03:54:01.370 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.369546376Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-10-09T03:54:01.658 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.425534701Z level=info msg="Executing migration" id="create provenance_type table" 2023-10-09T03:54:01.659 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.459951439Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-10-09T03:54:01.659 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.496328855Z level=info msg="Executing migration" id="create alert_image table" 2023-10-09T03:54:01.659 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.549342459Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-10-09T03:54:01.659 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.608281916Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-10-09T03:54:01.659 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.658979337Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-10-09T03:54:01.862 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":29,"stamp":"2023-10-09T03:54:01.110809+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385020,"kb_used_data":6716,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576196,"statfs":{"total":767960285184,"available":765518024704,"internally_reserved":0,"allocated":6877184,"data_stored":3898264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002335"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.362673+0000","last_change":"2023-10-09T03:53:20.111685+0000","last_active":"2023-10-09T03:53:21.362673+0000","last_peered":"2023-10-09T03:53:21.362673+0000","last_clean":"2023-10-09T03:53:21.362673+0000","last_became_active":"2023-10-09T03:53:20.111378+0000","last_became_peered":"2023-10-09T03:53:20.111378+0000","last_unstale":"2023-10-09T03:53:21.362673+0000","last_undegraded":"2023-10-09T03:53:21.362673+0000","last_fullsized":"2023-10-09T03:53:21.362673+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:41.663034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361892+0000","last_change":"2023-10-09T03:53:20.474740+0000","last_active":"2023-10-09T03:53:21.361892+0000","last_peered":"2023-10-09T03:53:21.361892+0000","last_clean":"2023-10-09T03:53:21.361892+0000","last_became_active":"2023-10-09T03:53:20.474458+0000","last_became_peered":"2023-10-09T03:53:20.474458+0000","last_unstale":"2023-10-09T03:53:21.361892+0000","last_undegraded":"2023-10-09T03:53:21.361892+0000","last_fullsized":"2023-10-09T03:53:21.361892+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:26.858718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.111514+0000","last_change":"2023-10-09T03:53:20.474716+0000","last_active":"2023-10-09T03:53:21.111514+0000","last_peered":"2023-10-09T03:53:21.111514+0000","last_clean":"2023-10-09T03:53:21.111514+0000","last_became_active":"2023-10-09T03:53:20.474362+0000","last_became_peered":"2023-10-09T03:53:20.474362+0000","last_unstale":"2023-10-09T03:53:21.111514+0000","last_undegraded":"2023-10-09T03:53:21.111514+0000","last_fullsized":"2023-10-09T03:53:21.111514+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:35.972389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.111449+0000","last_change":"2023-10-09T03:53:20.112489+0000","last_active":"2023-10-09T03:53:21.111449+0000","last_peered":"2023-10-09T03:53:21.111449+0000","last_clean":"2023-10-09T03:53:21.111449+0000","last_became_active":"2023-10-09T03:53:20.112217+0000","last_became_peered":"2023-10-09T03:53:20.112217+0000","last_unstale":"2023-10-09T03:53:21.111449+0000","last_undegraded":"2023-10-09T03:53:21.111449+0000","last_fullsized":"2023-10-09T03:53:21.111449+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:06.850905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.111452+0000","last_change":"2023-10-09T03:52:48.503079+0000","last_active":"2023-10-09T03:53:21.111452+0000","last_peered":"2023-10-09T03:53:21.111452+0000","last_clean":"2023-10-09T03:53:21.111452+0000","last_became_active":"2023-10-09T03:52:48.502616+0000","last_became_peered":"2023-10-09T03:52:48.502616+0000","last_unstale":"2023-10-09T03:53:21.111452+0000","last_undegraded":"2023-10-09T03:53:21.111452+0000","last_fullsized":"2023-10-09T03:53:21.111452+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T03:51:17.354073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361929+0000","last_change":"2023-10-09T03:53:20.111364+0000","last_active":"2023-10-09T03:53:21.361929+0000","last_peered":"2023-10-09T03:53:21.361929+0000","last_clean":"2023-10-09T03:53:21.361929+0000","last_became_active":"2023-10-09T03:53:20.111052+0000","last_became_peered":"2023-10-09T03:53:20.111052+0000","last_unstale":"2023-10-09T03:53:21.361929+0000","last_undegraded":"2023-10-09T03:53:21.361929+0000","last_fullsized":"2023-10-09T03:53:21.361929+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:41.535319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.110269+0000","last_change":"2023-10-09T03:53:20.475582+0000","last_active":"2023-10-09T03:53:21.110269+0000","last_peered":"2023-10-09T03:53:21.110269+0000","last_clean":"2023-10-09T03:53:21.110269+0000","last_became_active":"2023-10-09T03:53:20.475313+0000","last_became_peered":"2023-10-09T03:53:20.475313+0000","last_unstale":"2023-10-09T03:53:21.110269+0000","last_undegraded":"2023-10-09T03:53:21.110269+0000","last_fullsized":"2023-10-09T03:53:21.110269+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:25.695798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361919+0000","last_change":"2023-10-09T03:53:20.109068+0000","last_active":"2023-10-09T03:53:21.361919+0000","last_peered":"2023-10-09T03:53:21.361919+0000","last_clean":"2023-10-09T03:53:21.361919+0000","last_became_active":"2023-10-09T03:53:20.108811+0000","last_became_peered":"2023-10-09T03:53:20.108811+0000","last_unstale":"2023-10-09T03:53:21.361919+0000","last_undegraded":"2023-10-09T03:53:21.361919+0000","last_fullsized":"2023-10-09T03:53:21.361919+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.169451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361956+0000","last_change":"2023-10-09T03:53:20.110963+0000","last_active":"2023-10-09T03:53:21.361956+0000","last_peered":"2023-10-09T03:53:21.361956+0000","last_clean":"2023-10-09T03:53:21.361956+0000","last_became_active":"2023-10-09T03:53:20.110726+0000","last_became_peered":"2023-10-09T03:53:20.110726+0000","last_unstale":"2023-10-09T03:53:21.361956+0000","last_undegraded":"2023-10-09T03:53:21.361956+0000","last_fullsized":"2023-10-09T03:53:21.361956+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:53.134560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201296,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298492,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446660,"statfs":{"total":95995035648,"available":95689379840,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.50700000000000001}]},{"osd":1,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.629}]},{"osd":3,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.67000000000000004}]},{"osd":4,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.55000000000000004}]},{"osd":5,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.53900000000000003}]},{"osd":6,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.68600000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462933,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298488,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446664,"statfs":{"total":95995035648,"available":95689383936,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.80400000000000005}]},{"osd":1,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.66500000000000004}]},{"osd":2,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.73799999999999999}]},{"osd":4,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.71899999999999997}]},{"osd":5,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.77100000000000002}]},{"osd":7,"last update":"Mon Oct 9 03:53:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.82499999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574891,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:41 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34399999999999997,"15min":0.32600000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.54400000000000004,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.36099999999999999,"15min":0.33900000000000002},"min":{"1min":0.25600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 03:52:58 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.25900000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40300000000000002,"15min":0.39900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57099999999999995,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.27600000000000002}]},{"osd":3,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.433,"15min":0.438},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63600000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.372}]},{"osd":4,"last update":"Mon Oct 9 03:53:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.872,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.501},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.78000000000000003,"5min":0.875,"15min":0.875},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 03:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 03:53:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.54900000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298492,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446660,"statfs":{"total":95995035648,"available":95689379840,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 03:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.34899999999999998,"15min":0.34100000000000003},"min":{"1min":0.221,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.50600000000000001,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.36799999999999999,"15min":0.34999999999999998},"min":{"1min":0.27000000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.53800000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.32400000000000001}]},{"osd":2,"last update":"Mon Oct 9 03:52:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57099999999999995,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.437,"15min":0.435},"min":{"1min":0.311,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.625,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.46200000000000002}]},{"osd":3,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.47299999999999998,"15min":0.47699999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.51300000000000001,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63200000000000001,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55500000000000005}]},{"osd":4,"last update":"Mon Oct 9 03:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 03:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.59099999999999997}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.628},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54000000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297908,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447244,"statfs":{"total":95995035648,"available":95689977856,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.36399999999999999,"15min":0.35499999999999998},"min":{"1min":0.28799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.379,"15min":0.371},"min":{"1min":0.25600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.311}]},{"osd":1,"last update":"Mon Oct 9 03:53:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.38400000000000001,"15min":0.374},"min":{"1min":0.27500000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.28899999999999998}]},{"osd":3,"last update":"Mon Oct 9 03:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.44,"15min":0.434},"min":{"1min":0.32200000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.441,"15min":0.437},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.57399999999999995,"15min":0.58199999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.748,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.58899999999999997,"15min":0.59899999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.78500000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.79300000000000004}]},{"osd":5,"last update":"Mon Oct 9 03:53:06 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.753},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.59099999999999997}]},{"osd":6,"last update":"Mon Oct 9 03:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.81299999999999994}]},{"osd":7,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.82699999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.38800000000000001,"15min":0.39300000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.55100000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.29699999999999999}]},{"osd":1,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41999999999999998,"15min":0.42399999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55300000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.48399999999999999,"15min":0.47699999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.501,"15min":0.49299999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46400000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.35499999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.64500000000000002,"15min":0.64800000000000002},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Mon Oct 9 03:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.496},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46000000000000002}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Oct 9 03:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.48699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297908,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447244,"statfs":{"total":95995035648,"available":95689977856,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.622},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65700000000000003,"5min":0.753,"15min":0.753},"last":0.57699999999999996}]},{"osd":1,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55100000000000005,"15min":0.55200000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70999999999999996,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.55100000000000005,"15min":0.55900000000000005},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.315},"max":{"1min":0.69199999999999995,"5min":0.75,"15min":0.75},"last":0.63500000000000001}]},{"osd":3,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69099999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.72099999999999997}]},{"osd":5,"last update":"Mon Oct 9 03:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.55000000000000004}]},{"osd":7,"last update":"Mon Oct 9 03:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.079,"5min":1.079,"15min":1.079},"last":0.501}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297908,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447244,"statfs":{"total":95995035648,"available":95689977856,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.69799999999999995}]},{"osd":1,"last update":"Mon Oct 9 03:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.59899999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.71099999999999997}]},{"osd":3,"last update":"Mon Oct 9 03:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.75}]},{"osd":4,"last update":"Mon Oct 9 03:53:01 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.65200000000000002}]},{"osd":7,"last update":"Mon Oct 9 03:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.73599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T03:54:01.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T03:54:01.923 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.684529272Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-10-09T03:54:01.923 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.718168161Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T03:54:01.923 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.71832205Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T03:54:01.923 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.75189238Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-10-09T03:54:01.924 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.785566924Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-10-09T03:54:01.924 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.819218732Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-10-09T03:54:01.924 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.819427249Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-10-09T03:54:01.924 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.819593569Z level=info msg="alerts found to migrate" alerts=0 2023-10-09T03:54:01.924 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.85322843Z level=info msg="Executing migration" id="create library_element table v1" 2023-10-09T03:54:01.924 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.923608546Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-10-09T03:54:02.210 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:02.214 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:01 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:01.996080323Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-10-09T03:54:02.215 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.072236262Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T03:54:02.215 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.14780837Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-10-09T03:54:02.215 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.215316284Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-10-09T03:54:02.493 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:02 smithi181 ceph-mon[105375]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:02.493 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:02 smithi181 ceph-mon[105375]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:02.493 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:02 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:02.493 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.266091697Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-10-09T03:54:02.493 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.341902555Z level=info msg="Executing migration" id="create data_keys table" 2023-10-09T03:54:02.494 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.417399922Z level=info msg="Executing migration" id="create secrets table" 2023-10-09T03:54:02.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.493389362Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-10-09T03:54:02.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.569401093Z level=info msg="Executing migration" id="add name column into data_keys" 2023-10-09T03:54:02.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.636776498Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-10-09T03:54:02.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.720640729Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-10-09T03:54:02.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:02 smithi163 ceph-mon[118807]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:02.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:02 smithi163 ceph-mon[118807]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:02.771 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:02 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:02.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:02 smithi163 ceph-mon[101022]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:02.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:02 smithi163 ceph-mon[101022]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:02.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:02 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.771056703Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.813336725Z level=info msg="Executing migration" id="create kv_store table v1" 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.87214053Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.9226196Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:02.964435621Z level=info msg="Executing migration" id="create permission table" 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.006464051Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-10-09T03:54:03.048 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.048153018Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-10-09T03:54:03.049 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.895Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T03:54:03.049 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.895Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T03:54:03.049 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.895Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T03:54:03.049 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.895Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T03:54:03.049 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.895Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.895Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T03:54:03.050 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T03:54:03.051 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T03:54:03.052 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T03:54:03.053 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.896Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T03:54:03.054 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T03:54:03.055 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 conmon[153737]: ts=2023-10-09T03:54:02.897Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T03:54:03.055 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 bash[153652]: 02d966c4a8a947c08e2ef82588267be70b106eabfe9a249ba90a99fbbe17201f 2023-10-09T03:54:03.055 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:54:02 smithi181 systemd[1]: Started Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:54:03.300 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.12078933Z level=info msg="Executing migration" id="create role table" 2023-10-09T03:54:03.300 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.166310032Z level=info msg="Executing migration" id="add column display_name" 2023-10-09T03:54:03.300 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.191579473Z level=info msg="Executing migration" id="add column group_name" 2023-10-09T03:54:03.300 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.233463989Z level=info msg="Executing migration" id="add index role.org_id" 2023-10-09T03:54:03.301 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.275379138Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-10-09T03:54:03.301 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.300595388Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-10-09T03:54:03.552 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.351184599Z level=info msg="Executing migration" id="create team role table" 2023-10-09T03:54:03.553 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.393056451Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-10-09T03:54:03.553 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.418326107Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T03:54:03.553 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.443464102Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-10-09T03:54:03.553 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.485384839Z level=info msg="Executing migration" id="create user role table" 2023-10-09T03:54:03.553 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.50233846Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-10-09T03:54:03.553 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.527524592Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.552613111Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.578028166Z level=info msg="Executing migration" id="create builtin role table" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.603313561Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.62842148Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.678789656Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.761303548Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-10-09T03:54:03.804 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.804420404Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T03:54:03.878 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:03.881 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T03:54:04.056 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.056 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.056 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.056 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:04.057 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:03 smithi181 ceph-mon[105375]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:04.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.876268583Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-10-09T03:54:04.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.939148411Z level=info msg="Executing migration" id="add unique index role.uid" 2023-10-09T03:54:04.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:03 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:03.997622543Z level=info msg="Executing migration" id="create seed assignment table" 2023-10-09T03:54:04.057 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.056562529Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.106818487Z level=info msg="Executing migration" id="add column hidden to role table" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.14877814Z level=info msg="Executing migration" id="create query_history table v1" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.173940514Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.215890768Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.25776491Z level=info msg="Executing migration" id="teams permissions migration" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.282911794Z level=info msg="Executing migration" id="dashboard permissions" 2023-10-09T03:54:04.308 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.308217659Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[118807]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:03 smithi163 ceph-mon[101022]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:04.576 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.341836129Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-10-09T03:54:04.576 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.367022613Z level=info msg="Executing migration" id="alerting notification permissions" 2023-10-09T03:54:04.576 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.433999917Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-10-09T03:54:04.576 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.459231282Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-10-09T03:54:04.576 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.484370632Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-10-09T03:54:04.576 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.526272267Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-10-09T03:54:04.577 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.576591265Z level=info msg="Executing migration" id="create correlation table v1" 2023-10-09T03:54:04.689 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":30,"stamp":"2023-10-09T03:54:03.111096+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385020,"kb_used_data":6716,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576196,"statfs":{"total":767960285184,"available":765518024704,"internally_reserved":0,"allocated":6877184,"data_stored":3898264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002229"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.362673+0000","last_change":"2023-10-09T03:53:20.111685+0000","last_active":"2023-10-09T03:53:21.362673+0000","last_peered":"2023-10-09T03:53:21.362673+0000","last_clean":"2023-10-09T03:53:21.362673+0000","last_became_active":"2023-10-09T03:53:20.111378+0000","last_became_peered":"2023-10-09T03:53:20.111378+0000","last_unstale":"2023-10-09T03:53:21.362673+0000","last_undegraded":"2023-10-09T03:53:21.362673+0000","last_fullsized":"2023-10-09T03:53:21.362673+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:41.663034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361892+0000","last_change":"2023-10-09T03:53:20.474740+0000","last_active":"2023-10-09T03:53:21.361892+0000","last_peered":"2023-10-09T03:53:21.361892+0000","last_clean":"2023-10-09T03:53:21.361892+0000","last_became_active":"2023-10-09T03:53:20.474458+0000","last_became_peered":"2023-10-09T03:53:20.474458+0000","last_unstale":"2023-10-09T03:53:21.361892+0000","last_undegraded":"2023-10-09T03:53:21.361892+0000","last_fullsized":"2023-10-09T03:53:21.361892+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:26.858718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.111514+0000","last_change":"2023-10-09T03:53:20.474716+0000","last_active":"2023-10-09T03:53:21.111514+0000","last_peered":"2023-10-09T03:53:21.111514+0000","last_clean":"2023-10-09T03:53:21.111514+0000","last_became_active":"2023-10-09T03:53:20.474362+0000","last_became_peered":"2023-10-09T03:53:20.474362+0000","last_unstale":"2023-10-09T03:53:21.111514+0000","last_undegraded":"2023-10-09T03:53:21.111514+0000","last_fullsized":"2023-10-09T03:53:21.111514+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:35.972389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.111449+0000","last_change":"2023-10-09T03:53:20.112489+0000","last_active":"2023-10-09T03:53:21.111449+0000","last_peered":"2023-10-09T03:53:21.111449+0000","last_clean":"2023-10-09T03:53:21.111449+0000","last_became_active":"2023-10-09T03:53:20.112217+0000","last_became_peered":"2023-10-09T03:53:20.112217+0000","last_unstale":"2023-10-09T03:53:21.111449+0000","last_undegraded":"2023-10-09T03:53:21.111449+0000","last_fullsized":"2023-10-09T03:53:21.111449+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:06.850905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.111452+0000","last_change":"2023-10-09T03:52:48.503079+0000","last_active":"2023-10-09T03:53:21.111452+0000","last_peered":"2023-10-09T03:53:21.111452+0000","last_clean":"2023-10-09T03:53:21.111452+0000","last_became_active":"2023-10-09T03:52:48.502616+0000","last_became_peered":"2023-10-09T03:52:48.502616+0000","last_unstale":"2023-10-09T03:53:21.111452+0000","last_undegraded":"2023-10-09T03:53:21.111452+0000","last_fullsized":"2023-10-09T03:53:21.111452+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T03:51:17.354073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361929+0000","last_change":"2023-10-09T03:53:20.111364+0000","last_active":"2023-10-09T03:53:21.361929+0000","last_peered":"2023-10-09T03:53:21.361929+0000","last_clean":"2023-10-09T03:53:21.361929+0000","last_became_active":"2023-10-09T03:53:20.111052+0000","last_became_peered":"2023-10-09T03:53:20.111052+0000","last_unstale":"2023-10-09T03:53:21.361929+0000","last_undegraded":"2023-10-09T03:53:21.361929+0000","last_fullsized":"2023-10-09T03:53:21.361929+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:41.535319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.110269+0000","last_change":"2023-10-09T03:53:20.475582+0000","last_active":"2023-10-09T03:53:21.110269+0000","last_peered":"2023-10-09T03:53:21.110269+0000","last_clean":"2023-10-09T03:53:21.110269+0000","last_became_active":"2023-10-09T03:53:20.475313+0000","last_became_peered":"2023-10-09T03:53:20.475313+0000","last_unstale":"2023-10-09T03:53:21.110269+0000","last_undegraded":"2023-10-09T03:53:21.110269+0000","last_fullsized":"2023-10-09T03:53:21.110269+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:25.695798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361919+0000","last_change":"2023-10-09T03:53:20.109068+0000","last_active":"2023-10-09T03:53:21.361919+0000","last_peered":"2023-10-09T03:53:21.361919+0000","last_clean":"2023-10-09T03:53:21.361919+0000","last_became_active":"2023-10-09T03:53:20.108811+0000","last_became_peered":"2023-10-09T03:53:20.108811+0000","last_unstale":"2023-10-09T03:53:21.361919+0000","last_undegraded":"2023-10-09T03:53:21.361919+0000","last_fullsized":"2023-10-09T03:53:21.361919+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.169451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-10-09T03:53:21.361956+0000","last_change":"2023-10-09T03:53:20.110963+0000","last_active":"2023-10-09T03:53:21.361956+0000","last_peered":"2023-10-09T03:53:21.361956+0000","last_clean":"2023-10-09T03:53:21.361956+0000","last_became_active":"2023-10-09T03:53:20.110726+0000","last_became_peered":"2023-10-09T03:53:20.110726+0000","last_unstale":"2023-10-09T03:53:21.361956+0000","last_undegraded":"2023-10-09T03:53:21.361956+0000","last_fullsized":"2023-10-09T03:53:21.361956+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:53.134560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201297,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298492,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446660,"statfs":{"total":95995035648,"available":95689379840,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.54200000000000004}]},{"osd":1,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.66100000000000003}]},{"osd":3,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.59199999999999997}]},{"osd":4,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.45200000000000001}]},{"osd":5,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.5}]}]},{"osd":6,"up_from":47,"seq":201863462934,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298488,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446664,"statfs":{"total":95995035648,"available":95689383936,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.52300000000000002}]},{"osd":1,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56299999999999994}]},{"osd":3,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.501},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.66500000000000004}]},{"osd":4,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.41099999999999998}]},{"osd":5,"last update":"Mon Oct 9 03:53:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.436},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 03:53:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.68500000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:41 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34399999999999997,"15min":0.32600000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.54400000000000004,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.36099999999999999,"15min":0.33900000000000002},"min":{"1min":0.25600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.313}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39100000000000001,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.53800000000000003,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.371},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.52700000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.433,"15min":0.438},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.59199999999999997,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63600000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 03:53:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.872,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.78000000000000003,"5min":0.875,"15min":0.875},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 03:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 03:53:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.59199999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738416,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298492,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446660,"statfs":{"total":95995035648,"available":95689379840,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 03:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.34899999999999998,"15min":0.34100000000000003},"min":{"1min":0.221,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.50600000000000001,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.316},{"interface":"front","average":{"1min":0.37,"5min":0.36799999999999999,"15min":0.34999999999999998},"min":{"1min":0.27000000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.53800000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.27800000000000002}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51900000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.433,"15min":0.433},"min":{"1min":0.29599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.47299999999999998,"15min":0.47699999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.51300000000000001,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63200000000000001,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:53:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 03:53:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.48399999999999999}]},{"osd":7,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56399999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297908,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447244,"statfs":{"total":95995035648,"available":95689977856,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.36399999999999999,"15min":0.35499999999999998},"min":{"1min":0.28799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.379,"15min":0.371},"min":{"1min":0.25600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 03:53:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.38400000000000001,"15min":0.374},"min":{"1min":0.27500000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.309}]},{"osd":3,"last update":"Mon Oct 9 03:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.44,"15min":0.434},"min":{"1min":0.32200000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.441,"15min":0.437},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.39700000000000002}]},{"osd":4,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.57399999999999995,"15min":0.58199999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.748,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.58899999999999997,"15min":0.59899999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.78500000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.56499999999999995}]},{"osd":5,"last update":"Mon Oct 9 03:53:06 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.67600000000000005}]},{"osd":6,"last update":"Mon Oct 9 03:53:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 03:53:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.63800000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149731,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297912,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447240,"statfs":{"total":95995035648,"available":95689973760,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.38800000000000001,"15min":0.39300000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.317},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.55100000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.22500000000000001}]},{"osd":1,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.41999999999999998,"15min":0.42399999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55300000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.48399999999999999,"15min":0.47699999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Oct 9 03:53:20 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.501,"15min":0.49299999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46400000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.34999999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.626,"5min":0.64500000000000002,"15min":0.64800000000000002},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.54900000000000004}]},{"osd":5,"last update":"Mon Oct 9 03:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.54200000000000004}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 03:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.46700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986206,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297908,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447244,"statfs":{"total":95995035648,"available":95689977856,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.65700000000000003,"5min":0.753,"15min":0.753},"last":0.47799999999999998}]},{"osd":1,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55100000000000005,"15min":0.55200000000000005},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.139,"5min":1.139,"15min":1.139},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.59799999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70999999999999996,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.504,"5min":0.55100000000000005,"15min":0.55900000000000005},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.315},"max":{"1min":0.69199999999999995,"5min":0.75,"15min":0.75},"last":0.52200000000000002}]},{"osd":3,"last update":"Mon Oct 9 03:53:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69099999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.66200000000000003}]},{"osd":5,"last update":"Mon Oct 9 03:53:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.32700000000000001}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.41099999999999998}]},{"osd":7,"last update":"Mon Oct 9 03:53:46 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.079,"5min":1.079,"15min":1.079},"last":0.39700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724570,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297908,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447244,"statfs":{"total":95995035648,"available":95689977856,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.40600000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.82799999999999996,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.40600000000000003}]},{"osd":1,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.84899999999999998,"5min":1.002,"15min":1.002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.90200000000000002,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.499}]},{"osd":3,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83799999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.432},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.54100000000000004}]},{"osd":4,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.47299999999999998,"15min":0.46899999999999997},"min":{"1min":0.32700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.47999999999999998},"min":{"1min":0.34899999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.36199999999999999}]},{"osd":6,"last update":"Mon Oct 9 03:53:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 03:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.55400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T03:54:04.693 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T03:54:04.693 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T03:54:04.693 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T03:54:04.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T03:54:04.878 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.641150572Z level=info msg="Executing migration" id="add index correlations.uid" 2023-10-09T03:54:04.878 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.685683415Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-10-09T03:54:04.879 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.769850494Z level=info msg="Executing migration" id="add correlation config column" 2023-10-09T03:54:04.879 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.821384908Z level=info msg="Executing migration" id="create entity_events table" 2023-10-09T03:54:04.879 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.878856002Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-10-09T03:54:05.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:05 smithi163 ceph-mon[101022]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:05.078 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:05.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:05 smithi181 ceph-mon[105375]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:05.194 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.929160428Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T03:54:05.194 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.929340895Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T03:54:05.194 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.954448742Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T03:54:05.194 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.954608954Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T03:54:05.195 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:04 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:04.996333824Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-10-09T03:54:05.195 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.021502021Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-10-09T03:54:05.195 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.063494264Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-10-09T03:54:05.195 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.126066472Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T03:54:05.195 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.19457383Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-10-09T03:54:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:05 smithi163 ceph-mon[118807]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:05.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.239907898Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T03:54:05.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.29041541Z level=info msg="Executing migration" id="Drop public config table" 2023-10-09T03:54:05.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.349108593Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-10-09T03:54:05.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.374566925Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-10-09T03:54:05.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.434410497Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.522636588Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.558984156Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.584276034Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.626257225Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.651465014Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-10-09T03:54:05.844 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.67662679Z level=info msg="Executing migration" id="add share column" 2023-10-09T03:54:05.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.701887231Z level=info msg="Executing migration" id="create default alerting folders" 2023-10-09T03:54:05.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.760310278Z level=info msg="Executing migration" id="create file table" 2023-10-09T03:54:05.845 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.844009024Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-10-09T03:54:06.163 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:06 smithi181 ceph-mon[105375]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:06.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.903394942Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-10-09T03:54:06.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:05 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:05.953931815Z level=info msg="Executing migration" id="create file_meta table" 2023-10-09T03:54:06.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.020661514Z level=info msg="Executing migration" id="file table idx: path key" 2023-10-09T03:54:06.163 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.088432647Z level=info msg="Executing migration" id="set path collation in file table" 2023-10-09T03:54:06.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:06 smithi163 ceph-mon[118807]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:06.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:06 smithi163 ceph-mon[101022]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:06.414 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.163400359Z level=info msg="Executing migration" id="managed permissions migration" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.214056814Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.239032254Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.26421934Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.30631045Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.331553188Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.373314207Z level=info msg="Executing migration" id="update group index for alert rules" 2023-10-09T03:54:06.415 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.415148188Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-10-09T03:54:06.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.44881923Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-10-09T03:54:06.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.499094951Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-10-09T03:54:06.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.549587374Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-10-09T03:54:06.683 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.582900615Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-10-09T03:54:06.951 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.683261498Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-10-09T03:54:06.951 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.716969086Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-10-09T03:54:06.951 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.742096652Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-10-09T03:54:06.952 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.767325066Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-10-09T03:54:06.952 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.809283902Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-10-09T03:54:06.952 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.834425631Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-10-09T03:54:06.952 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.884840459Z level=info msg="Executing migration" id="create folder table" 2023-10-09T03:54:06.952 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.926699793Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-10-09T03:54:06.952 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.951766131Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-10-09T03:54:07.059 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:07.059 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T03:54:07.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:07 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1512385118' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T03:54:07.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:06 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:06.9769432Z level=info msg="Executing migration" id="Update folder title length" 2023-10-09T03:54:07.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:07.002170476Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-10-09T03:54:07.269 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=migrator t=2023-10-09T03:54:07.027316724Z level=info msg="migrations completed" performed=485 skipped=0 duration=17.750184282s 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=sqlstore t=2023-10-09T03:54:07.027835967Z level=info msg="Created default organization" 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=plugin.loader t=2023-10-09T03:54:07.106719532Z level=info msg="Plugin registered" pluginID=input 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=plugin.loader t=2023-10-09T03:54:07.133621123Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=plugin.loader t=2023-10-09T03:54:07.133645766Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=secrets t=2023-10-09T03:54:07.133691382Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=query_data t=2023-10-09T03:54:07.136647161Z level=info msg="Query Service initialization" 2023-10-09T03:54:07.270 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=live.push_http t=2023-10-09T03:54:07.139564373Z level=info msg="Live Push Gateway initialization" 2023-10-09T03:54:07.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:07.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1512385118' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:07.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:07 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1512385118' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T03:54:08.055 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T03:54:08.055 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T03:54:08.055 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T03:54:08.065 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T03:54:08.065 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T03:54:08.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=infra.usagestats.collector t=2023-10-09T03:54:07.86979581Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T03:54:08.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=provisioning.datasources t=2023-10-09T03:54:07.870193318Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T03:54:08.177 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=provisioning.datasources t=2023-10-09T03:54:07.902690733Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:07 smithi181 conmon[153221]: logger=provisioning.datasources t=2023-10-09T03:54:07.963654698Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=provisioning.alerting t=2023-10-09T03:54:08.003978915Z level=info msg="starting to provision alerting" 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=provisioning.alerting t=2023-10-09T03:54:08.00400453Z level=info msg="finished to provision alerting" 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=ngalert.state.manager t=2023-10-09T03:54:08.004121321Z level=info msg="Warming state cache for startup" 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=grafanaStorageLogger t=2023-10-09T03:54:08.004170614Z level=info msg="storage starting" 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=http.server t=2023-10-09T03:54:08.005994814Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=sqlstore.transactions t=2023-10-09T03:54:08.015533623Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=ngalert.state.manager t=2023-10-09T03:54:08.068376081Z level=info msg="State cache has been initialized" states=0 duration=64.25353ms 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=ticker t=2023-10-09T03:54:08.068474429Z level=info msg=starting first_tick=2023-10-09T03:54:10Z 2023-10-09T03:54:08.178 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:54:08 smithi181 conmon[153221]: logger=ngalert.multiorg.alertmanager t=2023-10-09T03:54:08.068465234Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T03:54:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[118807]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T03:54:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[118807]: Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T03:54:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[118807]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[101022]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T03:54:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[101022]: Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T03:54:08.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[101022]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:08.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:08 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:08 smithi181 ceph-mon[105375]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T03:54:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:08 smithi181 ceph-mon[105375]: Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T03:54:08.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:08 smithi181 ceph-mon[105375]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:08.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:08 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:08.650 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:10.082 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:09 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:54:10.386 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:10 smithi163 ceph-mon[118807]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:10.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:10 smithi163 ceph-mon[101022]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:10.387 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:10 smithi163 conmon[166098]: ts=2023-10-09T03:54:10.244Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T03:54:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:10 smithi181 ceph-mon[105375]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:11.059 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:10 smithi163 bash[176715]: ce053ea1fbdc7736d9a9ad353c8d944b93218afa50bff1c724633f447e493f7a 2023-10-09T03:54:11.331 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T03:54:11.332 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:54:11.332 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 systemd[1]: Starting Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:54:11.831 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.712Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.712Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.713Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.714Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 bash[177135]: 38b3c41a75cc16bd7c9a651726d43dedcbf4c09a24c6b3bfff327474e2add4ea 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 systemd[1]: Started Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.738Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.738Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.741Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T03:54:11.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:11 smithi163 conmon[177179]: ts=2023-10-09T03:54:11.741Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T03:54:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:12 smithi181 ceph-mon[105375]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:12 smithi181 ceph-mon[105375]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:54:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:12 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T03:54:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:12 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T03:54:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:12 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:12 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[118807]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[118807]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:54:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T03:54:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T03:54:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[101022]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[101022]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:54:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T03:54:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T03:54:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:12 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:13 smithi181 ceph-mon[105375]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:54:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:13 smithi181 ceph-mon[105375]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:54:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:13 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T03:54:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:13 smithi181 ceph-mon[105375]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T03:54:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:13 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:13 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:13.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[118807]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[118807]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[118807]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[101022]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:54:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[101022]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:54:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T03:54:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[101022]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T03:54:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:13 smithi163 conmon[100998]: 2023-10-09T03:54:13.221+0000 7fe548f5b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T03:54:13.963 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T03:54:13.992 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:13 smithi163 conmon[177179]: ts=2023-10-09T03:54:13.715Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000633632s 2023-10-09T03:54:14.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: pgmap v36: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:14.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:54:14.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T03:54:14.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T03:54:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T03:54:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T03:54:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T03:54:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T03:54:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: fsmap foo:0 2023-10-09T03:54:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: Saving service mds.foo spec with placement count:2 2023-10-09T03:54:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:14 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:14.320 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:14.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: pgmap v36: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T03:54:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T03:54:14.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: fsmap foo:0 2023-10-09T03:54:14.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: Saving service mds.foo spec with placement count:2 2023-10-09T03:54:14.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: pgmap v36: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T03:54:14.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T03:54:14.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T03:54:14.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: fsmap foo:0 2023-10-09T03:54:14.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: Saving service mds.foo spec with placement count:2 2023-10-09T03:54:14.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:14 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:14.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 systemd[1]: Stopping Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:54:14.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.708Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T03:54:14.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.708Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T03:54:14.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.708Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T03:54:14.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.708Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T03:54:14.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.708Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T03:54:14.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:14 smithi181 conmon[150019]: ts=2023-10-09T03:54:14.709Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T03:54:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:15 smithi181 ceph-mon[105375]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T03:54:15.518 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 bash[155276]: 4c5e87623c24c0c559a0f1053d9627fba476d7a3f3d3977f27502752b8d1c2bb 2023-10-09T03:54:15.519 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T03:54:15.519 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 systemd[1]: Stopped Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:54:15.519 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 systemd[1]: Starting Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:54:15.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:15 smithi163 ceph-mon[118807]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T03:54:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:15 smithi163 ceph-mon[101022]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T03:54:15.961 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:15 smithi163 conmon[101450]: [09/Oct/2023:03:54:15] ENGINE Bus STOPPING 2023-10-09T03:54:16.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 bash[155602]: 7e99604366c169f37ce1cd562ff3e65ea92b5eafcb1bcf9c5eed49910b981459 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 systemd[1]: Started Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.900Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.901Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.901Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi181 (none))" 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.901Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.901Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.903Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.904Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.906Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.906Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T03:54:16.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.907Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.907Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.632µs 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.907Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.907Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.908Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.908Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.974µs wal_replay_duration=481.809µs wbl_replay_duration=106ns total_replay_duration=525.148µs 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.908Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.908Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.908Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.924Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.868882ms db_storage=902ns remote_storage=1.385µs web_handler=433ns query_engine=604ns scrape=650.17µs scrape_sd=29.63µs notify=23.238µs notify_sd=11.837µs rules=14.923592ms tracing=7.132µs 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.924Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T03:54:16.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:54:15 smithi181 conmon[155643]: ts=2023-10-09T03:54:15.924Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: pgmap v40: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:54:16.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163:9093"}]: dispatch 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:15 smithi163 conmon[101450]: [09/Oct/2023:03:54:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:54:16.243 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:15 smithi163 conmon[101450]: [09/Oct/2023:03:54:15] ENGINE Bus STOPPED 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:15 smithi163 conmon[101450]: [09/Oct/2023:03:54:15] ENGINE Bus STARTING 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Serving on http://:::9283 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STARTED 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STOPPING 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STOPPED 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STARTING 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Serving on http://:::9283 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STARTED 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STOPPING 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STOPPED 2023-10-09T03:54:16.244 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STARTING 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: pgmap v40: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163:9093"}]: dispatch 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:54:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:54:16.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:54:16.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T03:54:16.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:16 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:16.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Serving on http://:::9283 2023-10-09T03:54:16.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:16 smithi163 conmon[101450]: [09/Oct/2023:03:54:16] ENGINE Bus STARTED 2023-10-09T03:54:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: pgmap v40: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:54:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T03:54:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163:9093"}]: dispatch 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:16 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:16.750 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T03:54:17.370 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:17.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163:9093"}]: dispatch 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:17.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:54:17.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163:9093"}]: dispatch 2023-10-09T03:54:17.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163:9093"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:54:17.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:54:17.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T03:54:17.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:17.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:18 smithi181 ceph-mon[105375]: pgmap v42: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:18 smithi163 ceph-mon[118807]: pgmap v42: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:18 smithi163 ceph-mon[101022]: pgmap v42: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[101022]: Deploying daemon mds.foo.smithi163.crqkdj on smithi163 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:19.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:19 smithi163 ceph-mon[118807]: Deploying daemon mds.foo.smithi163.crqkdj on smithi163 2023-10-09T03:54:19.865 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T03:54:19.875 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T03:54:19.875 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T03:54:19.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T03:54:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:19 smithi181 ceph-mon[105375]: Deploying daemon mds.foo.smithi163.crqkdj on smithi163 2023-10-09T03:54:20.938 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:22.082 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:54:21 smithi163 conmon[177179]: ts=2023-10-09T03:54:21.718Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003346961s 2023-10-09T03:54:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:22 smithi181 ceph-mon[105375]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:22.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:22 smithi163 ceph-mon[118807]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:22.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:22 smithi163 ceph-mon[101022]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:22.974 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: Deploying daemon mds.foo.smithi181.sboiwx on smithi181 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1193554686' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:boot 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: daemon mds.foo.smithi163.crqkdj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T03:54:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: Cluster is now healthy 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: fsmap foo:0 1 up:standby 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:creating} 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[118807]: daemon mds.foo.smithi163.crqkdj is now active in filesystem foo as rank 0 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:23.695 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: Deploying daemon mds.foo.smithi181.sboiwx on smithi181 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1193554686' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:boot 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: daemon mds.foo.smithi163.crqkdj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: Cluster is now healthy 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: fsmap foo:0 1 up:standby 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:creating} 2023-10-09T03:54:23.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:23 smithi163 ceph-mon[101022]: daemon mds.foo.smithi163.crqkdj is now active in filesystem foo as rank 0 2023-10-09T03:54:23.708 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":19,"active_gid":14457,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":55760003},{"type":"v1","addr":"172.21.15.163:6801","nonce":55760003}]},"active_addr":"172.21.15.163:6801/55760003","active_change":"2023-10-09T03:53:17.088335+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24364,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":2462309240}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1847962187}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3524034575}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":1361101496}]}]} 2023-10-09T03:54:23.713 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T03:54:23.713 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T03:54:23.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: Deploying daemon mds.foo.smithi181.sboiwx on smithi181 2023-10-09T03:54:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1193554686' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:boot 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: daemon mds.foo.smithi163.crqkdj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: Cluster is now healthy 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: fsmap foo:0 1 up:standby 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:creating} 2023-10-09T03:54:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:23 smithi181 ceph-mon[105375]: daemon mds.foo.smithi163.crqkdj is now active in filesystem foo as rank 0 2023-10-09T03:54:24.060 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:24 smithi163 ceph-mon[101022]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:24 smithi163 ceph-mon[101022]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:active 2023-10-09T03:54:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:24 smithi163 ceph-mon[101022]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 2023-10-09T03:54:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:24 smithi163 ceph-mon[118807]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:24 smithi163 ceph-mon[118807]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:active 2023-10-09T03:54:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:24 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 2023-10-09T03:54:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:24 smithi181 ceph-mon[105375]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:24 smithi181 ceph-mon[105375]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:active 2023-10-09T03:54:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:24 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 2023-10-09T03:54:25.034 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:24 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:24] "GET /metrics HTTP/1.1" 200 208012 "" "Prometheus/2.43.0" 2023-10-09T03:54:25.697 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:25.697 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":69,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T03:54:15.231935+0000","last_up_change":"2023-10-09T03:52:44.469504+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T03:54:11.510171+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T03:54:12.437589+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6803","nonce":3842166717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6805","nonce":3842166717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6809","nonce":3842166717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6807","nonce":3842166717}]},"public_addr":"172.21.15.163:6803/3842166717","cluster_addr":"172.21.15.163:6805/3842166717","heartbeat_back_addr":"172.21.15.163:6809/3842166717","heartbeat_front_addr":"172.21.15.163:6807/3842166717","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6811","nonce":147795175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6813","nonce":147795175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6817","nonce":147795175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6815","nonce":147795175}]},"public_addr":"172.21.15.163:6811/147795175","cluster_addr":"172.21.15.163:6813/147795175","heartbeat_back_addr":"172.21.15.163:6817/147795175","heartbeat_front_addr":"172.21.15.163:6815/147795175","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6819","nonce":2150896924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6821","nonce":2150896924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6825","nonce":2150896924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6823","nonce":2150896924}]},"public_addr":"172.21.15.163:6819/2150896924","cluster_addr":"172.21.15.163:6821/2150896924","heartbeat_back_addr":"172.21.15.163:6825/2150896924","heartbeat_front_addr":"172.21.15.163:6823/2150896924","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6827","nonce":1216700115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6829","nonce":1216700115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6833","nonce":1216700115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6831","nonce":1216700115}]},"public_addr":"172.21.15.163:6827/1216700115","cluster_addr":"172.21.15.163:6829/1216700115","heartbeat_back_addr":"172.21.15.163:6833/1216700115","heartbeat_front_addr":"172.21.15.163:6831/1216700115","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T03:54:26.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:54:25 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: mds.? [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] up:boot 2023-10-09T03:54:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:54:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:25 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: mds.? [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] up:boot 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: mds.? [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] up:boot 2023-10-09T03:54:26.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:26.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:54:26.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:25 smithi163 ceph-mon[101022]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:26.518 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T03:54:26.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T03:54:26.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:26 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/881835421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:54:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:26 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/881835421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:54:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:26 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/881835421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:54:27.162 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:28.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:28 smithi181 ceph-mon[105375]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:28 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:28 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:28 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[118807]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[101022]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:28 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:28.996 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:28.996 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":69,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T03:54:15.231935+0000","last_up_change":"2023-10-09T03:52:44.469504+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T03:54:11.510171+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T03:54:12.437589+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6803","nonce":3842166717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6805","nonce":3842166717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6809","nonce":3842166717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":3842166717},{"type":"v1","addr":"172.21.15.163:6807","nonce":3842166717}]},"public_addr":"172.21.15.163:6803/3842166717","cluster_addr":"172.21.15.163:6805/3842166717","heartbeat_back_addr":"172.21.15.163:6809/3842166717","heartbeat_front_addr":"172.21.15.163:6807/3842166717","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6811","nonce":147795175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6813","nonce":147795175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6817","nonce":147795175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":147795175},{"type":"v1","addr":"172.21.15.163:6815","nonce":147795175}]},"public_addr":"172.21.15.163:6811/147795175","cluster_addr":"172.21.15.163:6813/147795175","heartbeat_back_addr":"172.21.15.163:6817/147795175","heartbeat_front_addr":"172.21.15.163:6815/147795175","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6819","nonce":2150896924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6821","nonce":2150896924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6825","nonce":2150896924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":2150896924},{"type":"v1","addr":"172.21.15.163:6823","nonce":2150896924}]},"public_addr":"172.21.15.163:6819/2150896924","cluster_addr":"172.21.15.163:6821/2150896924","heartbeat_back_addr":"172.21.15.163:6825/2150896924","heartbeat_front_addr":"172.21.15.163:6823/2150896924","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6827","nonce":1216700115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6829","nonce":1216700115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6833","nonce":1216700115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":1216700115},{"type":"v1","addr":"172.21.15.163:6831","nonce":1216700115}]},"public_addr":"172.21.15.163:6827/1216700115","cluster_addr":"172.21.15.163:6829/1216700115","heartbeat_back_addr":"172.21.15.163:6833/1216700115","heartbeat_front_addr":"172.21.15.163:6831/1216700115","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T03:54:29.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:active 2023-10-09T03:54:29.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:29.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:29.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:29.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3153930144' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:54:29.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:active 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3153930144' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:54:29.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:29 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:29.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: mds.? [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] up:active 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3153930144' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T03:54:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:29 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:54:29.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T03:54:29.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T03:54:29.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T03:54:29.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T03:54:29.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T03:54:29.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T03:54:29.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T03:54:29.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T03:54:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:30 smithi181 ceph-mon[105375]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:30 smithi181 ceph-mon[105375]: mds.? [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] up:standby 2023-10-09T03:54:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:30 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:30 smithi163 ceph-mon[101022]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:30 smithi163 ceph-mon[101022]: mds.? [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] up:standby 2023-10-09T03:54:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:30 smithi163 ceph-mon[101022]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:30 smithi163 ceph-mon[118807]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:30 smithi163 ceph-mon[118807]: mds.? [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] up:standby 2023-10-09T03:54:30.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:30 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:54:30.603 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:30.654 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:30.763 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:32.241 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:32.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:32 smithi181 ceph-mon[105375]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:32 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:32.533 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[118807]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:32.533 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.533 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.533 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.533 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:32.534 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[101022]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:32.534 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.534 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.534 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:32.534 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:32 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:32.566 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:33.429 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:33.631 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:34.178 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:34.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:34 smithi163 ceph-mon[118807]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:34.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:34 smithi163 ceph-mon[101022]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:34.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:34 smithi181 ceph-mon[105375]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:34.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:34 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:34] "GET /metrics HTTP/1.1" 200 238874 "" "Prometheus/2.43.0" 2023-10-09T03:54:36.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:54:35 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:54:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:36 smithi163 ceph-mon[118807]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:36 smithi163 ceph-mon[101022]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:36 smithi181 ceph-mon[105375]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:36.930 INFO:teuthology.orchestra.run.smithi163.stdout:55834574899 2023-10-09T03:54:36.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T03:54:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:37 smithi163 ceph-mon[118807]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:37.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:37 smithi163 ceph-mon[101022]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:38.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:37 smithi181 ceph-mon[105375]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:38.416 INFO:teuthology.orchestra.run.smithi163.stdout:167503724577 2023-10-09T03:54:38.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T03:54:39.458 INFO:teuthology.orchestra.run.smithi163.stdout:201863462941 2023-10-09T03:54:39.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T03:54:40.194 INFO:teuthology.orchestra.run.smithi163.stdout:111669149739 2023-10-09T03:54:40.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T03:54:40.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:40 smithi163 ceph-mon[118807]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:40.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:40 smithi163 ceph-mon[101022]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:40.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:40 smithi181 ceph-mon[105375]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T03:54:40.770 INFO:teuthology.orchestra.run.smithi163.stdout:77309411376 2023-10-09T03:54:40.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T03:54:41.352 INFO:teuthology.orchestra.run.smithi163.stdout:34359738423 2023-10-09T03:54:41.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T03:54:42.123 INFO:teuthology.orchestra.run.smithi163.stdout:133143986214 2023-10-09T03:54:42.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T03:54:42.282 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:42.306 INFO:teuthology.orchestra.run.smithi163.stdout:236223201305 2023-10-09T03:54:42.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T03:54:42.357 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:42.422 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:42.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:42 smithi181 ceph-mon[105375]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:42.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:42 smithi163 ceph-mon[118807]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:42.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:42 smithi163 ceph-mon[101022]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:42.858 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:43.497 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:44.455 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:44 smithi163 ceph-mon[101022]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:44.456 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:44 smithi163 ceph-mon[118807]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:44 smithi181 ceph-mon[105375]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:44.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:44 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:44] "GET /metrics HTTP/1.1" 200 238874 "" "Prometheus/2.43.0" 2023-10-09T03:54:46.021 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:54:45 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:54:46.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:46 smithi163 ceph-mon[118807]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:46.444 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:46 smithi163 ceph-mon[101022]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:46 smithi181 ceph-mon[105375]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:46.735 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:47.026 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:47.109 INFO:teuthology.orchestra.run.smithi163.stdout:55834574901 2023-10-09T03:54:47.114 INFO:teuthology.orchestra.run.smithi163.stdout:201863462943 2023-10-09T03:54:47.166 INFO:teuthology.orchestra.run.smithi163.stdout:167503724579 2023-10-09T03:54:47.236 INFO:teuthology.orchestra.run.smithi163.stdout:77309411377 2023-10-09T03:54:47.531 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2201944960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T03:54:47.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/114574801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T03:54:47.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[101022]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:47.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3859354646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T03:54:47.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:47.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2217193051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T03:54:47.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2201944960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T03:54:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/114574801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T03:54:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[118807]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3859354646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T03:54:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:47 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2217193051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T03:54:48.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:47 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2201944960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T03:54:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:47 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/114574801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T03:54:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:47 smithi181 ceph-mon[105375]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:47 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3859354646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T03:54:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:47 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:54:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:47 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2217193051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T03:54:48.730 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724577 got 167503724579 for osd.5 2023-10-09T03:54:48.730 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:48.776 INFO:teuthology.orchestra.run.smithi163.stdout:111669149740 2023-10-09T03:54:48.861 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574899 got 55834574901 for osd.1 2023-10-09T03:54:48.861 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:48.997 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:48 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1020909604' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T03:54:48.998 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:48 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1020909604' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T03:54:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:48 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1020909604' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T03:54:50.066 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:50.089 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411376 got 77309411377 for osd.2 2023-10-09T03:54:50.090 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:50.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:49 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:49 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:49 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:49 smithi181 ceph-mon[105375]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:50.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[118807]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:54:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:54:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:54:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:49 smithi163 ceph-mon[101022]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:50.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149739 got 111669149740 for osd.3 2023-10-09T03:54:50.461 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:50.614 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462941 got 201863462943 for osd.6 2023-10-09T03:54:50.614 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:51.246 INFO:teuthology.orchestra.run.smithi163.stdout:34359738426 2023-10-09T03:54:52.107 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738423 got 34359738426 for osd.0 2023-10-09T03:54:52.107 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:52.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:52 smithi163 ceph-mon[118807]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:52.402 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:52 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4276655419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T03:54:52.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:52 smithi163 ceph-mon[101022]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:52.402 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:52 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4276655419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T03:54:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:52 smithi181 ceph-mon[105375]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:52 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4276655419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T03:54:52.997 INFO:teuthology.orchestra.run.smithi163.stdout:133143986216 2023-10-09T03:54:53.217 INFO:teuthology.orchestra.run.smithi163.stdout:236223201308 2023-10-09T03:54:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:53 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3218548277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T03:54:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:53 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3218548277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T03:54:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:53 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3218548277' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T03:54:54.079 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986214 got 133143986216 for osd.4 2023-10-09T03:54:54.079 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:54.252 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201305 got 236223201308 for osd.7 2023-10-09T03:54:54.253 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:54.253 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T03:54:54.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T03:54:54.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:54 smithi163 ceph-mon[101022]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:54.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:54 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/355485850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T03:54:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:54 smithi163 ceph-mon[118807]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:54 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/355485850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T03:54:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:54 smithi181 ceph-mon[105375]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:54 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/355485850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T03:54:54.612 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:54.863 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:54:54 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:54] "GET /metrics HTTP/1.1" 200 238828 "" "Prometheus/2.43.0" 2023-10-09T03:54:56.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:54:55 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:54:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:54:56.323 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:56.325 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T03:54:56.506 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:56 smithi163 ceph-mon[118807]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:56.507 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:56 smithi163 ceph-mon[101022]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:56 smithi181 ceph-mon[105375]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:56.993 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":61,"stamp":"2023-10-09T03:54:55.122410+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385596,"kb_used_data":7292,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575620,"statfs":{"total":767960285184,"available":765517434880,"internally_reserved":0,"allocated":7467008,"data_stored":4195854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002457"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.898939+0000","last_change":"2023-10-09T03:54:13.386628+0000","last_active":"2023-10-09T03:54:14.898939+0000","last_peered":"2023-10-09T03:54:14.898939+0000","last_clean":"2023-10-09T03:54:14.898939+0000","last_became_active":"2023-10-09T03:54:13.385451+0000","last_became_peered":"2023-10-09T03:54:13.385451+0000","last_unstale":"2023-10-09T03:54:14.898939+0000","last_undegraded":"2023-10-09T03:54:14.898939+0000","last_fullsized":"2023-10-09T03:54:14.898939+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:31.987326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.182562+0000","last_change":"2023-10-09T03:54:13.220219+0000","last_active":"2023-10-09T03:54:23.182562+0000","last_peered":"2023-10-09T03:54:23.182562+0000","last_clean":"2023-10-09T03:54:23.182562+0000","last_became_active":"2023-10-09T03:54:13.218725+0000","last_became_peered":"2023-10-09T03:54:13.218725+0000","last_unstale":"2023-10-09T03:54:23.182562+0000","last_undegraded":"2023-10-09T03:54:23.182562+0000","last_fullsized":"2023-10-09T03:54:23.182562+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:23.774862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234877+0000","last_change":"2023-10-09T03:54:13.242838+0000","last_active":"2023-10-09T03:54:14.234877+0000","last_peered":"2023-10-09T03:54:14.234877+0000","last_clean":"2023-10-09T03:54:14.234877+0000","last_became_active":"2023-10-09T03:54:13.242548+0000","last_became_peered":"2023-10-09T03:54:13.242548+0000","last_unstale":"2023-10-09T03:54:14.234877+0000","last_undegraded":"2023-10-09T03:54:14.234877+0000","last_fullsized":"2023-10-09T03:54:14.234877+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.406483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.240193+0000","last_change":"2023-10-09T03:54:13.228008+0000","last_active":"2023-10-09T03:54:15.240193+0000","last_peered":"2023-10-09T03:54:15.240193+0000","last_clean":"2023-10-09T03:54:15.240193+0000","last_became_active":"2023-10-09T03:54:13.227440+0000","last_became_peered":"2023-10-09T03:54:13.227440+0000","last_unstale":"2023-10-09T03:54:15.240193+0000","last_undegraded":"2023-10-09T03:54:15.240193+0000","last_fullsized":"2023-10-09T03:54:15.240193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:48.443487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899114+0000","last_change":"2023-10-09T03:54:13.242320+0000","last_active":"2023-10-09T03:54:14.899114+0000","last_peered":"2023-10-09T03:54:14.899114+0000","last_clean":"2023-10-09T03:54:14.899114+0000","last_became_active":"2023-10-09T03:54:13.241937+0000","last_became_peered":"2023-10-09T03:54:13.241937+0000","last_unstale":"2023-10-09T03:54:14.899114+0000","last_undegraded":"2023-10-09T03:54:14.899114+0000","last_fullsized":"2023-10-09T03:54:14.899114+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:03:44.435784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.185528+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T03:54:23.185528+0000","last_peered":"2023-10-09T03:54:23.185528+0000","last_clean":"2023-10-09T03:54:23.185528+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T03:54:23.185528+0000","last_undegraded":"2023-10-09T03:54:23.185528+0000","last_fullsized":"2023-10-09T03:54:23.185528+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899214+0000","last_change":"2023-10-09T03:54:13.387084+0000","last_active":"2023-10-09T03:54:14.899214+0000","last_peered":"2023-10-09T03:54:14.899214+0000","last_clean":"2023-10-09T03:54:14.899214+0000","last_became_active":"2023-10-09T03:54:13.386739+0000","last_became_peered":"2023-10-09T03:54:13.386739+0000","last_unstale":"2023-10-09T03:54:14.899214+0000","last_undegraded":"2023-10-09T03:54:14.899214+0000","last_fullsized":"2023-10-09T03:54:14.899214+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:18.787228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.184647+0000","last_change":"2023-10-09T03:54:13.226919+0000","last_active":"2023-10-09T03:54:23.184647+0000","last_peered":"2023-10-09T03:54:23.184647+0000","last_clean":"2023-10-09T03:54:23.184647+0000","last_became_active":"2023-10-09T03:54:13.226604+0000","last_became_peered":"2023-10-09T03:54:13.226604+0000","last_unstale":"2023-10-09T03:54:23.184647+0000","last_undegraded":"2023-10-09T03:54:23.184647+0000","last_fullsized":"2023-10-09T03:54:23.184647+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:59.895137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.172527+0000","last_change":"2023-10-09T03:54:13.386113+0000","last_active":"2023-10-09T03:54:15.172527+0000","last_peered":"2023-10-09T03:54:15.172527+0000","last_clean":"2023-10-09T03:54:15.172527+0000","last_became_active":"2023-10-09T03:54:13.385774+0000","last_became_peered":"2023-10-09T03:54:13.385774+0000","last_unstale":"2023-10-09T03:54:15.172527+0000","last_undegraded":"2023-10-09T03:54:15.172527+0000","last_fullsized":"2023-10-09T03:54:15.172527+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:13.981606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:16.431952+0000","last_change":"2023-10-09T03:54:13.231818+0000","last_active":"2023-10-09T03:54:16.431952+0000","last_peered":"2023-10-09T03:54:16.431952+0000","last_clean":"2023-10-09T03:54:16.431952+0000","last_became_active":"2023-10-09T03:54:13.231557+0000","last_became_peered":"2023-10-09T03:54:13.231557+0000","last_unstale":"2023-10-09T03:54:16.431952+0000","last_undegraded":"2023-10-09T03:54:16.431952+0000","last_fullsized":"2023-10-09T03:54:16.431952+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:07.072823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899161+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T03:54:14.899161+0000","last_peered":"2023-10-09T03:54:14.899161+0000","last_clean":"2023-10-09T03:54:14.899161+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T03:54:14.899161+0000","last_undegraded":"2023-10-09T03:54:14.899161+0000","last_fullsized":"2023-10-09T03:54:14.899161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:16.432040+0000","last_change":"2023-10-09T03:54:13.226657+0000","last_active":"2023-10-09T03:54:16.432040+0000","last_peered":"2023-10-09T03:54:16.432040+0000","last_clean":"2023-10-09T03:54:16.432040+0000","last_became_active":"2023-10-09T03:54:13.226298+0000","last_became_peered":"2023-10-09T03:54:13.226298+0000","last_unstale":"2023-10-09T03:54:16.432040+0000","last_undegraded":"2023-10-09T03:54:16.432040+0000","last_fullsized":"2023-10-09T03:54:16.432040+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:29.543896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425618+0000","last_change":"2023-10-09T03:53:20.111685+0000","last_active":"2023-10-09T03:54:14.425618+0000","last_peered":"2023-10-09T03:54:14.425618+0000","last_clean":"2023-10-09T03:54:14.425618+0000","last_became_active":"2023-10-09T03:53:20.111378+0000","last_became_peered":"2023-10-09T03:53:20.111378+0000","last_unstale":"2023-10-09T03:54:14.425618+0000","last_undegraded":"2023-10-09T03:54:14.425618+0000","last_fullsized":"2023-10-09T03:54:14.425618+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:41.663034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235270+0000","last_change":"2023-10-09T03:54:13.390094+0000","last_active":"2023-10-09T03:54:14.235270+0000","last_peered":"2023-10-09T03:54:14.235270+0000","last_clean":"2023-10-09T03:54:14.235270+0000","last_became_active":"2023-10-09T03:54:13.389687+0000","last_became_peered":"2023-10-09T03:54:13.389687+0000","last_unstale":"2023-10-09T03:54:14.235270+0000","last_undegraded":"2023-10-09T03:54:14.235270+0000","last_fullsized":"2023-10-09T03:54:14.235270+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:29.050861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.233898+0000","last_change":"2023-10-09T03:53:20.474740+0000","last_active":"2023-10-09T03:54:14.233898+0000","last_peered":"2023-10-09T03:54:14.233898+0000","last_clean":"2023-10-09T03:54:14.233898+0000","last_became_active":"2023-10-09T03:53:20.474458+0000","last_became_peered":"2023-10-09T03:53:20.474458+0000","last_unstale":"2023-10-09T03:54:14.233898+0000","last_undegraded":"2023-10-09T03:54:14.233898+0000","last_fullsized":"2023-10-09T03:54:14.233898+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:26.858718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.181642+0000","last_change":"2023-10-09T03:54:13.226015+0000","last_active":"2023-10-09T03:54:23.181642+0000","last_peered":"2023-10-09T03:54:23.181642+0000","last_clean":"2023-10-09T03:54:23.181642+0000","last_became_active":"2023-10-09T03:54:13.225742+0000","last_became_peered":"2023-10-09T03:54:13.225742+0000","last_unstale":"2023-10-09T03:54:23.181642+0000","last_undegraded":"2023-10-09T03:54:23.181642+0000","last_fullsized":"2023-10-09T03:54:23.181642+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:23.722906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901999+0000","last_change":"2023-10-09T03:54:13.387150+0000","last_active":"2023-10-09T03:54:14.901999+0000","last_peered":"2023-10-09T03:54:14.901999+0000","last_clean":"2023-10-09T03:54:14.901999+0000","last_became_active":"2023-10-09T03:54:13.386847+0000","last_became_peered":"2023-10-09T03:54:13.386847+0000","last_unstale":"2023-10-09T03:54:14.901999+0000","last_undegraded":"2023-10-09T03:54:14.901999+0000","last_fullsized":"2023-10-09T03:54:14.901999+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:49.764730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235371+0000","last_change":"2023-10-09T03:54:13.231307+0000","last_active":"2023-10-09T03:54:14.235371+0000","last_peered":"2023-10-09T03:54:14.235371+0000","last_clean":"2023-10-09T03:54:14.235371+0000","last_became_active":"2023-10-09T03:54:13.230984+0000","last_became_peered":"2023-10-09T03:54:13.230984+0000","last_unstale":"2023-10-09T03:54:14.235371+0000","last_undegraded":"2023-10-09T03:54:14.235371+0000","last_fullsized":"2023-10-09T03:54:14.235371+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:39.291581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.632047+0000","last_change":"2023-10-09T03:53:20.474716+0000","last_active":"2023-10-09T03:54:15.632047+0000","last_peered":"2023-10-09T03:54:15.632047+0000","last_clean":"2023-10-09T03:54:15.632047+0000","last_became_active":"2023-10-09T03:53:20.474362+0000","last_became_peered":"2023-10-09T03:53:20.474362+0000","last_unstale":"2023-10-09T03:54:15.632047+0000","last_undegraded":"2023-10-09T03:54:15.632047+0000","last_fullsized":"2023-10-09T03:54:15.632047+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:35.972389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902127+0000","last_change":"2023-10-09T03:54:13.390577+0000","last_active":"2023-10-09T03:54:14.902127+0000","last_peered":"2023-10-09T03:54:14.902127+0000","last_clean":"2023-10-09T03:54:14.902127+0000","last_became_active":"2023-10-09T03:54:13.390285+0000","last_became_peered":"2023-10-09T03:54:13.390285+0000","last_unstale":"2023-10-09T03:54:14.902127+0000","last_undegraded":"2023-10-09T03:54:14.902127+0000","last_fullsized":"2023-10-09T03:54:14.902127+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:42.866073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901832+0000","last_change":"2023-10-09T03:54:13.224078+0000","last_active":"2023-10-09T03:54:14.901832+0000","last_peered":"2023-10-09T03:54:14.901832+0000","last_clean":"2023-10-09T03:54:14.901832+0000","last_became_active":"2023-10-09T03:54:13.223768+0000","last_became_peered":"2023-10-09T03:54:13.223768+0000","last_unstale":"2023-10-09T03:54:14.901832+0000","last_undegraded":"2023-10-09T03:54:14.901832+0000","last_fullsized":"2023-10-09T03:54:14.901832+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:24.072600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631554+0000","last_change":"2023-10-09T03:53:20.112489+0000","last_active":"2023-10-09T03:54:15.631554+0000","last_peered":"2023-10-09T03:54:15.631554+0000","last_clean":"2023-10-09T03:54:15.631554+0000","last_became_active":"2023-10-09T03:53:20.112217+0000","last_became_peered":"2023-10-09T03:53:20.112217+0000","last_unstale":"2023-10-09T03:54:15.631554+0000","last_undegraded":"2023-10-09T03:54:15.631554+0000","last_fullsized":"2023-10-09T03:54:15.631554+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:06.850905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425546+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T03:54:14.425546+0000","last_peered":"2023-10-09T03:54:14.425546+0000","last_clean":"2023-10-09T03:54:14.425546+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T03:54:14.425546+0000","last_undegraded":"2023-10-09T03:54:14.425546+0000","last_fullsized":"2023-10-09T03:54:14.425546+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234468+0000","last_change":"2023-10-09T03:54:13.228292+0000","last_active":"2023-10-09T03:54:14.234468+0000","last_peered":"2023-10-09T03:54:14.234468+0000","last_clean":"2023-10-09T03:54:14.234468+0000","last_became_active":"2023-10-09T03:54:13.227985+0000","last_became_peered":"2023-10-09T03:54:13.227985+0000","last_unstale":"2023-10-09T03:54:14.234468+0000","last_undegraded":"2023-10-09T03:54:14.234468+0000","last_fullsized":"2023-10-09T03:54:14.234468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:31.580822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":276,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631466+0000","last_change":"2023-10-09T03:52:48.503079+0000","last_active":"2023-10-09T03:54:15.631466+0000","last_peered":"2023-10-09T03:54:15.631466+0000","last_clean":"2023-10-09T03:54:15.631466+0000","last_became_active":"2023-10-09T03:52:48.502616+0000","last_became_peered":"2023-10-09T03:52:48.502616+0000","last_unstale":"2023-10-09T03:54:15.631466+0000","last_undegraded":"2023-10-09T03:54:15.631466+0000","last_fullsized":"2023-10-09T03:54:15.631466+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T03:51:17.354073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238147+0000","last_change":"2023-10-09T03:53:20.111364+0000","last_active":"2023-10-09T03:54:14.238147+0000","last_peered":"2023-10-09T03:54:14.238147+0000","last_clean":"2023-10-09T03:54:14.238147+0000","last_became_active":"2023-10-09T03:53:20.111052+0000","last_became_peered":"2023-10-09T03:53:20.111052+0000","last_unstale":"2023-10-09T03:54:14.238147+0000","last_undegraded":"2023-10-09T03:54:14.238147+0000","last_fullsized":"2023-10-09T03:54:14.238147+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:41.535319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902923+0000","last_change":"2023-10-09T03:54:14.902726+0000","last_active":"2023-10-09T03:54:14.902923+0000","last_peered":"2023-10-09T03:54:14.902923+0000","last_clean":"2023-10-09T03:54:14.902923+0000","last_became_active":"2023-10-09T03:54:14.902459+0000","last_became_peered":"2023-10-09T03:54:14.902459+0000","last_unstale":"2023-10-09T03:54:14.902923+0000","last_undegraded":"2023-10-09T03:54:14.902923+0000","last_fullsized":"2023-10-09T03:54:14.902923+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:16.261746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234458+0000","last_change":"2023-10-09T03:54:13.231206+0000","last_active":"2023-10-09T03:54:14.234458+0000","last_peered":"2023-10-09T03:54:14.234458+0000","last_clean":"2023-10-09T03:54:14.234458+0000","last_became_active":"2023-10-09T03:54:13.230934+0000","last_became_peered":"2023-10-09T03:54:13.230934+0000","last_unstale":"2023-10-09T03:54:14.234458+0000","last_undegraded":"2023-10-09T03:54:14.234458+0000","last_fullsized":"2023-10-09T03:54:14.234458+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:57.235449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.172114+0000","last_change":"2023-10-09T03:53:20.475582+0000","last_active":"2023-10-09T03:54:15.172114+0000","last_peered":"2023-10-09T03:54:15.172114+0000","last_clean":"2023-10-09T03:54:15.172114+0000","last_became_active":"2023-10-09T03:53:20.475313+0000","last_became_peered":"2023-10-09T03:53:20.475313+0000","last_unstale":"2023-10-09T03:54:15.172114+0000","last_undegraded":"2023-10-09T03:54:15.172114+0000","last_fullsized":"2023-10-09T03:54:15.172114+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:25.695798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.174161+0000","last_change":"2023-10-09T03:54:15.173633+0000","last_active":"2023-10-09T03:54:15.174161+0000","last_peered":"2023-10-09T03:54:15.174161+0000","last_clean":"2023-10-09T03:54:15.174161+0000","last_became_active":"2023-10-09T03:54:15.173293+0000","last_became_peered":"2023-10-09T03:54:15.173293+0000","last_unstale":"2023-10-09T03:54:15.174161+0000","last_undegraded":"2023-10-09T03:54:15.174161+0000","last_fullsized":"2023-10-09T03:54:15.174161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:57.842218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899224+0000","last_change":"2023-10-09T03:54:13.223301+0000","last_active":"2023-10-09T03:54:14.899224+0000","last_peered":"2023-10-09T03:54:14.899224+0000","last_clean":"2023-10-09T03:54:14.899224+0000","last_became_active":"2023-10-09T03:54:13.223024+0000","last_became_peered":"2023-10-09T03:54:13.223024+0000","last_unstale":"2023-10-09T03:54:14.899224+0000","last_undegraded":"2023-10-09T03:54:14.899224+0000","last_fullsized":"2023-10-09T03:54:14.899224+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:58.715626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238206+0000","last_change":"2023-10-09T03:53:20.109068+0000","last_active":"2023-10-09T03:54:14.238206+0000","last_peered":"2023-10-09T03:54:14.238206+0000","last_clean":"2023-10-09T03:54:14.238206+0000","last_became_active":"2023-10-09T03:53:20.108811+0000","last_became_peered":"2023-10-09T03:53:20.108811+0000","last_unstale":"2023-10-09T03:54:14.238206+0000","last_undegraded":"2023-10-09T03:54:14.238206+0000","last_fullsized":"2023-10-09T03:54:14.238206+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.169451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.900637+0000","last_change":"2023-10-09T03:54:14.900151+0000","last_active":"2023-10-09T03:54:14.900637+0000","last_peered":"2023-10-09T03:54:14.900637+0000","last_clean":"2023-10-09T03:54:14.900637+0000","last_became_active":"2023-10-09T03:54:14.899807+0000","last_became_peered":"2023-10-09T03:54:14.899807+0000","last_unstale":"2023-10-09T03:54:14.900637+0000","last_undegraded":"2023-10-09T03:54:14.900637+0000","last_fullsized":"2023-10-09T03:54:14.900637+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:42.682264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238418+0000","last_change":"2023-10-09T03:54:13.226097+0000","last_active":"2023-10-09T03:54:14.238418+0000","last_peered":"2023-10-09T03:54:14.238418+0000","last_clean":"2023-10-09T03:54:14.238418+0000","last_became_active":"2023-10-09T03:54:13.225797+0000","last_became_peered":"2023-10-09T03:54:13.225797+0000","last_unstale":"2023-10-09T03:54:14.238418+0000","last_undegraded":"2023-10-09T03:54:14.238418+0000","last_fullsized":"2023-10-09T03:54:14.238418+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:24:34.378883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235384+0000","last_change":"2023-10-09T03:53:20.110963+0000","last_active":"2023-10-09T03:54:14.235384+0000","last_peered":"2023-10-09T03:54:14.235384+0000","last_clean":"2023-10-09T03:54:14.235384+0000","last_became_active":"2023-10-09T03:53:20.110726+0000","last_became_peered":"2023-10-09T03:53:20.110726+0000","last_unstale":"2023-10-09T03:54:14.235384+0000","last_undegraded":"2023-10-09T03:54:14.235384+0000","last_fullsized":"2023-10-09T03:54:14.235384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:53.134560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.900634+0000","last_change":"2023-10-09T03:54:14.900142+0000","last_active":"2023-10-09T03:54:14.900634+0000","last_peered":"2023-10-09T03:54:14.900634+0000","last_clean":"2023-10-09T03:54:14.900634+0000","last_became_active":"2023-10-09T03:54:14.899825+0000","last_became_peered":"2023-10-09T03:54:14.899825+0000","last_unstale":"2023-10-09T03:54:14.900634+0000","last_undegraded":"2023-10-09T03:54:14.900634+0000","last_fullsized":"2023-10-09T03:54:14.900634+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:58.614149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235362+0000","last_change":"2023-10-09T03:54:13.225693+0000","last_active":"2023-10-09T03:54:14.235362+0000","last_peered":"2023-10-09T03:54:14.235362+0000","last_clean":"2023-10-09T03:54:14.235362+0000","last_became_active":"2023-10-09T03:54:13.225412+0000","last_became_peered":"2023-10-09T03:54:13.225412+0000","last_unstale":"2023-10-09T03:54:14.235362+0000","last_undegraded":"2023-10-09T03:54:14.235362+0000","last_fullsized":"2023-10-09T03:54:14.235362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:50.987303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899376+0000","last_change":"2023-10-09T03:54:13.392302+0000","last_active":"2023-10-09T03:54:14.899376+0000","last_peered":"2023-10-09T03:54:14.899376+0000","last_clean":"2023-10-09T03:54:14.899376+0000","last_became_active":"2023-10-09T03:54:13.391911+0000","last_became_peered":"2023-10-09T03:54:13.391911+0000","last_unstale":"2023-10-09T03:54:14.899376+0000","last_undegraded":"2023-10-09T03:54:14.899376+0000","last_fullsized":"2023-10-09T03:54:14.899376+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:55.954773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899258+0000","last_change":"2023-10-09T03:54:13.222214+0000","last_active":"2023-10-09T03:54:14.899258+0000","last_peered":"2023-10-09T03:54:14.899258+0000","last_clean":"2023-10-09T03:54:14.899258+0000","last_became_active":"2023-10-09T03:54:13.221328+0000","last_became_peered":"2023-10-09T03:54:13.221328+0000","last_unstale":"2023-10-09T03:54:14.899258+0000","last_undegraded":"2023-10-09T03:54:14.899258+0000","last_fullsized":"2023-10-09T03:54:14.899258+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:56.353256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899476+0000","last_change":"2023-10-09T03:54:13.386688+0000","last_active":"2023-10-09T03:54:14.899476+0000","last_peered":"2023-10-09T03:54:14.899476+0000","last_clean":"2023-10-09T03:54:14.899476+0000","last_became_active":"2023-10-09T03:54:13.386053+0000","last_became_peered":"2023-10-09T03:54:13.386053+0000","last_unstale":"2023-10-09T03:54:14.899476+0000","last_undegraded":"2023-10-09T03:54:14.899476+0000","last_fullsized":"2023-10-09T03:54:14.899476+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:24.011861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425506+0000","last_change":"2023-10-09T03:54:13.230286+0000","last_active":"2023-10-09T03:54:14.425506+0000","last_peered":"2023-10-09T03:54:14.425506+0000","last_clean":"2023-10-09T03:54:14.425506+0000","last_became_active":"2023-10-09T03:54:13.229933+0000","last_became_peered":"2023-10-09T03:54:13.229933+0000","last_unstale":"2023-10-09T03:54:14.425506+0000","last_undegraded":"2023-10-09T03:54:14.425506+0000","last_fullsized":"2023-10-09T03:54:14.425506+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:34.348293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899226+0000","last_change":"2023-10-09T03:54:13.242037+0000","last_active":"2023-10-09T03:54:14.899226+0000","last_peered":"2023-10-09T03:54:14.899226+0000","last_clean":"2023-10-09T03:54:14.899226+0000","last_became_active":"2023-10-09T03:54:13.241753+0000","last_became_peered":"2023-10-09T03:54:13.241753+0000","last_unstale":"2023-10-09T03:54:14.899226+0000","last_undegraded":"2023-10-09T03:54:14.899226+0000","last_fullsized":"2023-10-09T03:54:14.899226+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.328661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183195+0000","last_change":"2023-10-09T03:54:13.226705+0000","last_active":"2023-10-09T03:54:23.183195+0000","last_peered":"2023-10-09T03:54:23.183195+0000","last_clean":"2023-10-09T03:54:23.183195+0000","last_became_active":"2023-10-09T03:54:13.226335+0000","last_became_peered":"2023-10-09T03:54:13.226335+0000","last_unstale":"2023-10-09T03:54:23.183195+0000","last_undegraded":"2023-10-09T03:54:23.183195+0000","last_fullsized":"2023-10-09T03:54:23.183195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:35.033771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901910+0000","last_change":"2023-10-09T03:54:13.387252+0000","last_active":"2023-10-09T03:54:14.901910+0000","last_peered":"2023-10-09T03:54:14.901910+0000","last_clean":"2023-10-09T03:54:14.901910+0000","last_became_active":"2023-10-09T03:54:13.386963+0000","last_became_peered":"2023-10-09T03:54:13.386963+0000","last_unstale":"2023-10-09T03:54:14.901910+0000","last_undegraded":"2023-10-09T03:54:14.901910+0000","last_fullsized":"2023-10-09T03:54:14.901910+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:19:00.954422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238265+0000","last_change":"2023-10-09T03:54:13.222789+0000","last_active":"2023-10-09T03:54:14.238265+0000","last_peered":"2023-10-09T03:54:14.238265+0000","last_clean":"2023-10-09T03:54:14.238265+0000","last_became_active":"2023-10-09T03:54:13.222433+0000","last_became_peered":"2023-10-09T03:54:13.222433+0000","last_unstale":"2023-10-09T03:54:14.238265+0000","last_undegraded":"2023-10-09T03:54:14.238265+0000","last_fullsized":"2023-10-09T03:54:14.238265+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:53.372287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425340+0000","last_change":"2023-10-09T03:54:13.387297+0000","last_active":"2023-10-09T03:54:14.425340+0000","last_peered":"2023-10-09T03:54:14.425340+0000","last_clean":"2023-10-09T03:54:14.425340+0000","last_became_active":"2023-10-09T03:54:13.386982+0000","last_became_peered":"2023-10-09T03:54:13.386982+0000","last_unstale":"2023-10-09T03:54:14.425340+0000","last_undegraded":"2023-10-09T03:54:14.425340+0000","last_fullsized":"2023-10-09T03:54:14.425340+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:00.680974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:17.134615+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T03:54:17.134615+0000","last_peered":"2023-10-09T03:54:17.134615+0000","last_clean":"2023-10-09T03:54:17.134615+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T03:54:17.134615+0000","last_undegraded":"2023-10-09T03:54:17.134615+0000","last_fullsized":"2023-10-09T03:54:17.134615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899061+0000","last_change":"2023-10-09T03:54:13.385895+0000","last_active":"2023-10-09T03:54:14.899061+0000","last_peered":"2023-10-09T03:54:14.899061+0000","last_clean":"2023-10-09T03:54:14.899061+0000","last_became_active":"2023-10-09T03:54:13.385558+0000","last_became_peered":"2023-10-09T03:54:13.385558+0000","last_unstale":"2023-10-09T03:54:14.899061+0000","last_undegraded":"2023-10-09T03:54:14.899061+0000","last_fullsized":"2023-10-09T03:54:14.899061+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:10.759913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183590+0000","last_change":"2023-10-09T03:54:13.226606+0000","last_active":"2023-10-09T03:54:23.183590+0000","last_peered":"2023-10-09T03:54:23.183590+0000","last_clean":"2023-10-09T03:54:23.183590+0000","last_became_active":"2023-10-09T03:54:13.226283+0000","last_became_peered":"2023-10-09T03:54:13.226283+0000","last_unstale":"2023-10-09T03:54:23.183590+0000","last_undegraded":"2023-10-09T03:54:23.183590+0000","last_fullsized":"2023-10-09T03:54:23.183590+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:46.736265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238147+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T03:54:14.238147+0000","last_peered":"2023-10-09T03:54:14.238147+0000","last_clean":"2023-10-09T03:54:14.238147+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T03:54:14.238147+0000","last_undegraded":"2023-10-09T03:54:14.238147+0000","last_fullsized":"2023-10-09T03:54:14.238147+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:17.134741+0000","last_change":"2023-10-09T03:54:13.231923+0000","last_active":"2023-10-09T03:54:17.134741+0000","last_peered":"2023-10-09T03:54:17.134741+0000","last_clean":"2023-10-09T03:54:17.134741+0000","last_became_active":"2023-10-09T03:54:13.231559+0000","last_became_peered":"2023-10-09T03:54:13.231559+0000","last_unstale":"2023-10-09T03:54:17.134741+0000","last_undegraded":"2023-10-09T03:54:17.134741+0000","last_fullsized":"2023-10-09T03:54:17.134741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:49.479131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.233729+0000","last_change":"2023-10-09T03:54:13.388403+0000","last_active":"2023-10-09T03:54:14.233729+0000","last_peered":"2023-10-09T03:54:14.233729+0000","last_clean":"2023-10-09T03:54:14.233729+0000","last_became_active":"2023-10-09T03:54:13.388091+0000","last_became_peered":"2023-10-09T03:54:13.388091+0000","last_unstale":"2023-10-09T03:54:14.233729+0000","last_undegraded":"2023-10-09T03:54:14.233729+0000","last_fullsized":"2023-10-09T03:54:14.233729+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:21.971508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.185595+0000","last_change":"2023-10-09T03:54:13.223658+0000","last_active":"2023-10-09T03:54:23.185595+0000","last_peered":"2023-10-09T03:54:23.185595+0000","last_clean":"2023-10-09T03:54:23.185595+0000","last_became_active":"2023-10-09T03:54:13.223344+0000","last_became_peered":"2023-10-09T03:54:13.223344+0000","last_unstale":"2023-10-09T03:54:23.185595+0000","last_undegraded":"2023-10-09T03:54:23.185595+0000","last_fullsized":"2023-10-09T03:54:23.185595+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:43.813381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901959+0000","last_change":"2023-10-09T03:54:13.389725+0000","last_active":"2023-10-09T03:54:14.901959+0000","last_peered":"2023-10-09T03:54:14.901959+0000","last_clean":"2023-10-09T03:54:14.901959+0000","last_became_active":"2023-10-09T03:54:13.389228+0000","last_became_peered":"2023-10-09T03:54:13.389228+0000","last_unstale":"2023-10-09T03:54:14.901959+0000","last_undegraded":"2023-10-09T03:54:14.901959+0000","last_fullsized":"2023-10-09T03:54:14.901959+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:07.477701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631412+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T03:54:15.631412+0000","last_peered":"2023-10-09T03:54:15.631412+0000","last_clean":"2023-10-09T03:54:15.631412+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T03:54:15.631412+0000","last_undegraded":"2023-10-09T03:54:15.631412+0000","last_fullsized":"2023-10-09T03:54:15.631412+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238326+0000","last_change":"2023-10-09T03:54:13.390929+0000","last_active":"2023-10-09T03:54:14.238326+0000","last_peered":"2023-10-09T03:54:14.238326+0000","last_clean":"2023-10-09T03:54:14.238326+0000","last_became_active":"2023-10-09T03:54:13.389911+0000","last_became_peered":"2023-10-09T03:54:13.389911+0000","last_unstale":"2023-10-09T03:54:14.238326+0000","last_undegraded":"2023-10-09T03:54:14.238326+0000","last_fullsized":"2023-10-09T03:54:14.238326+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:16.431009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901843+0000","last_change":"2023-10-09T03:54:13.230386+0000","last_active":"2023-10-09T03:54:14.901843+0000","last_peered":"2023-10-09T03:54:14.901843+0000","last_clean":"2023-10-09T03:54:14.901843+0000","last_became_active":"2023-10-09T03:54:13.230038+0000","last_became_peered":"2023-10-09T03:54:13.230038+0000","last_unstale":"2023-10-09T03:54:14.901843+0000","last_undegraded":"2023-10-09T03:54:14.901843+0000","last_fullsized":"2023-10-09T03:54:14.901843+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:25.790371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234308+0000","last_change":"2023-10-09T03:54:13.388361+0000","last_active":"2023-10-09T03:54:14.234308+0000","last_peered":"2023-10-09T03:54:14.234308+0000","last_clean":"2023-10-09T03:54:14.234308+0000","last_became_active":"2023-10-09T03:54:13.388054+0000","last_became_peered":"2023-10-09T03:54:13.388054+0000","last_unstale":"2023-10-09T03:54:14.234308+0000","last_undegraded":"2023-10-09T03:54:14.234308+0000","last_fullsized":"2023-10-09T03:54:14.234308+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:12.673234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183209+0000","last_change":"2023-10-09T03:54:13.228564+0000","last_active":"2023-10-09T03:54:23.183209+0000","last_peered":"2023-10-09T03:54:23.183209+0000","last_clean":"2023-10-09T03:54:23.183209+0000","last_became_active":"2023-10-09T03:54:13.228240+0000","last_became_peered":"2023-10-09T03:54:13.228240+0000","last_unstale":"2023-10-09T03:54:23.183209+0000","last_undegraded":"2023-10-09T03:54:23.183209+0000","last_fullsized":"2023-10-09T03:54:23.183209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:13.153308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899130+0000","last_change":"2023-10-09T03:54:13.386245+0000","last_active":"2023-10-09T03:54:14.899130+0000","last_peered":"2023-10-09T03:54:14.899130+0000","last_clean":"2023-10-09T03:54:14.899130+0000","last_became_active":"2023-10-09T03:54:13.385685+0000","last_became_peered":"2023-10-09T03:54:13.385685+0000","last_unstale":"2023-10-09T03:54:14.899130+0000","last_undegraded":"2023-10-09T03:54:14.899130+0000","last_fullsized":"2023-10-09T03:54:14.899130+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.339050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.185891+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T03:54:23.185891+0000","last_peered":"2023-10-09T03:54:23.185891+0000","last_clean":"2023-10-09T03:54:23.185891+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T03:54:23.185891+0000","last_undegraded":"2023-10-09T03:54:23.185891+0000","last_fullsized":"2023-10-09T03:54:23.185891+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.174291+0000","last_change":"2023-10-09T03:54:15.173687+0000","last_active":"2023-10-09T03:54:15.174291+0000","last_peered":"2023-10-09T03:54:15.174291+0000","last_clean":"2023-10-09T03:54:15.174291+0000","last_became_active":"2023-10-09T03:54:15.173367+0000","last_became_peered":"2023-10-09T03:54:15.173367+0000","last_unstale":"2023-10-09T03:54:15.174291+0000","last_undegraded":"2023-10-09T03:54:15.174291+0000","last_fullsized":"2023-10-09T03:54:15.174291+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:15:59.955464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183290+0000","last_change":"2023-10-09T03:54:13.226162+0000","last_active":"2023-10-09T03:54:23.183290+0000","last_peered":"2023-10-09T03:54:23.183290+0000","last_clean":"2023-10-09T03:54:23.183290+0000","last_became_active":"2023-10-09T03:54:13.225774+0000","last_became_peered":"2023-10-09T03:54:13.225774+0000","last_unstale":"2023-10-09T03:54:23.183290+0000","last_undegraded":"2023-10-09T03:54:23.183290+0000","last_fullsized":"2023-10-09T03:54:23.183290+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:03.496497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234997+0000","last_change":"2023-10-09T03:54:13.387399+0000","last_active":"2023-10-09T03:54:14.234997+0000","last_peered":"2023-10-09T03:54:14.234997+0000","last_clean":"2023-10-09T03:54:14.234997+0000","last_became_active":"2023-10-09T03:54:13.387085+0000","last_became_peered":"2023-10-09T03:54:13.387085+0000","last_unstale":"2023-10-09T03:54:14.234997+0000","last_undegraded":"2023-10-09T03:54:14.234997+0000","last_fullsized":"2023-10-09T03:54:14.234997+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:58.092470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238384+0000","last_change":"2023-10-09T03:54:13.226204+0000","last_active":"2023-10-09T03:54:14.238384+0000","last_peered":"2023-10-09T03:54:14.238384+0000","last_clean":"2023-10-09T03:54:14.238384+0000","last_became_active":"2023-10-09T03:54:13.225925+0000","last_became_peered":"2023-10-09T03:54:13.225925+0000","last_unstale":"2023-10-09T03:54:14.238384+0000","last_undegraded":"2023-10-09T03:54:14.238384+0000","last_fullsized":"2023-10-09T03:54:14.238384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:02.027420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235712+0000","last_change":"2023-10-09T03:54:13.388994+0000","last_active":"2023-10-09T03:54:14.235712+0000","last_peered":"2023-10-09T03:54:14.235712+0000","last_clean":"2023-10-09T03:54:14.235712+0000","last_became_active":"2023-10-09T03:54:13.388715+0000","last_became_peered":"2023-10-09T03:54:13.388715+0000","last_unstale":"2023-10-09T03:54:14.235712+0000","last_undegraded":"2023-10-09T03:54:14.235712+0000","last_fullsized":"2023-10-09T03:54:14.235712+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:26.161561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902151+0000","last_change":"2023-10-09T03:54:13.223099+0000","last_active":"2023-10-09T03:54:14.902151+0000","last_peered":"2023-10-09T03:54:14.902151+0000","last_clean":"2023-10-09T03:54:14.902151+0000","last_became_active":"2023-10-09T03:54:13.222771+0000","last_became_peered":"2023-10-09T03:54:13.222771+0000","last_unstale":"2023-10-09T03:54:14.902151+0000","last_undegraded":"2023-10-09T03:54:14.902151+0000","last_fullsized":"2023-10-09T03:54:14.902151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:02.054533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425595+0000","last_change":"2023-10-09T03:54:13.389291+0000","last_active":"2023-10-09T03:54:14.425595+0000","last_peered":"2023-10-09T03:54:14.425595+0000","last_clean":"2023-10-09T03:54:14.425595+0000","last_became_active":"2023-10-09T03:54:13.388934+0000","last_became_peered":"2023-10-09T03:54:13.388934+0000","last_unstale":"2023-10-09T03:54:14.425595+0000","last_undegraded":"2023-10-09T03:54:14.425595+0000","last_fullsized":"2023-10-09T03:54:14.425595+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:48:35.112863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235740+0000","last_change":"2023-10-09T03:54:13.225218+0000","last_active":"2023-10-09T03:54:14.235740+0000","last_peered":"2023-10-09T03:54:14.235740+0000","last_clean":"2023-10-09T03:54:14.235740+0000","last_became_active":"2023-10-09T03:54:13.224870+0000","last_became_peered":"2023-10-09T03:54:13.224870+0000","last_unstale":"2023-10-09T03:54:14.235740+0000","last_undegraded":"2023-10-09T03:54:14.235740+0000","last_fullsized":"2023-10-09T03:54:14.235740+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:26.196818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902017+0000","last_change":"2023-10-09T03:54:13.389519+0000","last_active":"2023-10-09T03:54:14.902017+0000","last_peered":"2023-10-09T03:54:14.902017+0000","last_clean":"2023-10-09T03:54:14.902017+0000","last_became_active":"2023-10-09T03:54:13.388817+0000","last_became_peered":"2023-10-09T03:54:13.388817+0000","last_unstale":"2023-10-09T03:54:14.902017+0000","last_undegraded":"2023-10-09T03:54:14.902017+0000","last_fullsized":"2023-10-09T03:54:14.902017+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:40.701614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.233572+0000","last_change":"2023-10-09T03:54:13.230822+0000","last_active":"2023-10-09T03:54:14.233572+0000","last_peered":"2023-10-09T03:54:14.233572+0000","last_clean":"2023-10-09T03:54:14.233572+0000","last_became_active":"2023-10-09T03:54:13.230496+0000","last_became_peered":"2023-10-09T03:54:13.230496+0000","last_unstale":"2023-10-09T03:54:14.233572+0000","last_undegraded":"2023-10-09T03:54:14.233572+0000","last_fullsized":"2023-10-09T03:54:14.233572+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:37.250589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234098+0000","last_change":"2023-10-09T03:54:13.385615+0000","last_active":"2023-10-09T03:54:14.234098+0000","last_peered":"2023-10-09T03:54:14.234098+0000","last_clean":"2023-10-09T03:54:14.234098+0000","last_became_active":"2023-10-09T03:54:13.385341+0000","last_became_peered":"2023-10-09T03:54:13.385341+0000","last_unstale":"2023-10-09T03:54:14.234098+0000","last_undegraded":"2023-10-09T03:54:14.234098+0000","last_fullsized":"2023-10-09T03:54:14.234098+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:21.643886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631407+0000","last_change":"2023-10-09T03:54:13.222567+0000","last_active":"2023-10-09T03:54:15.631407+0000","last_peered":"2023-10-09T03:54:15.631407+0000","last_clean":"2023-10-09T03:54:15.631407+0000","last_became_active":"2023-10-09T03:54:13.222226+0000","last_became_peered":"2023-10-09T03:54:13.222226+0000","last_unstale":"2023-10-09T03:54:15.631407+0000","last_undegraded":"2023-10-09T03:54:15.631407+0000","last_fullsized":"2023-10-09T03:54:15.631407+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:10.480578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201308,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298580,"kb_used_data":1292,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446572,"statfs":{"total":95995035648,"available":95689289728,"internally_reserved":0,"allocated":1323008,"data_stored":894496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.54200000000000004,"15min":0.52200000000000002},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.944,"5min":2.944,"15min":2.944},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.57299999999999995,"15min":0.52900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.3879999999999999,"5min":5.3879999999999999,"15min":5.3879999999999999},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.56100000000000005,"15min":0.53900000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.8969999999999998,"5min":2.8969999999999998,"15min":2.8969999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.621,"15min":0.56000000000000005},"min":{"1min":0.41199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":7.8390000000000004},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.51900000000000002,"15min":0.50900000000000001},"min":{"1min":0.39200000000000002,"5min":0.37,"15min":0.37},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.624},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.61699999999999999,"15min":0.58899999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.2530000000000001,"5min":4.2530000000000001,"15min":4.2530000000000001},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.60099999999999998,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.7709999999999999,"5min":3.7709999999999999,"15min":3.7709999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.61499999999999999,"15min":0.59099999999999997},"min":{"1min":0.39500000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.6190000000000002,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.59699999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.56999999999999995,"15min":0.52300000000000002},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.3129999999999997,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.54000000000000004,"15min":0.50800000000000001},"min":{"1min":0.316,"5min":0.308,"15min":0.308},"max":{"1min":3.8769999999999998,"5min":3.8769999999999998,"15min":3.8769999999999998},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55900000000000005,"15min":0.55500000000000005},"min":{"1min":0.36099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":0.495},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.1840000000000002,"5min":3.1840000000000002,"15min":3.1840000000000002},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.54100000000000004,"15min":0.52600000000000002},"min":{"1min":0.40699999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.57299999999999995,"15min":0.55900000000000005},"min":{"1min":0.439,"5min":0.373,"15min":0.373},"max":{"1min":1.9350000000000001,"5min":1.9350000000000001,"15min":1.9350000000000001},"last":0.72499999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462945,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298564,"kb_used_data":1276,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446588,"statfs":{"total":95995035648,"available":95689306112,"internally_reserved":0,"allocated":1306624,"data_stored":894027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.49199999999999999,"15min":0.48999999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.52600000000000002,"15min":0.51200000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0990000000000002,"5min":2.0990000000000002,"15min":2.0990000000000002},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.30599999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.486,"5min":1.486,"15min":1.486},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.55400000000000005}]},{"osd":2,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.48299999999999998,"15min":0.47299999999999998},"min":{"1min":0.35099999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.505,"15min":0.49399999999999999},"min":{"1min":0.33500000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.86199999999999999,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.52800000000000002,"15min":0.51900000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3839999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.61799999999999999}]},{"osd":4,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57099999999999995,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.71899999999999997,"5min":0.995,"15min":0.995},"last":0.67600000000000005}]},{"osd":5,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.47499999999999998,"15min":0.46899999999999997},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69999999999999996,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.51900000000000002,"15min":0.51500000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.77100000000000002,"5min":1.03,"15min":1.03},"last":0.65600000000000003}]},{"osd":7,"last update":"Mon Oct 9 03:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.57199999999999995,"15min":0.56499999999999995},"min":{"1min":0.41799999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.93000000000000005,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.58199999999999996,"15min":0.57299999999999995},"min":{"1min":0.45800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":1.0089999999999999,"5min":1.077,"15min":1.077},"last":0.63400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574903,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":302376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.32800000000000001},"min":{"1min":0.249,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.46200000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.36199999999999999,"15min":0.34000000000000002},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.5,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39100000000000001,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.53800000000000003,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.378},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.52700000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.26000000000000001}]},{"osd":3,"last update":"Mon Oct 9 03:54:21 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.42099999999999999,"15min":0.434},"min":{"1min":0.28899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55600000000000005,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45900000000000002,"15min":0.46800000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58299999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Oct 9 03:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68600000000000005,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71799999999999997,"5min":0.875,"15min":0.875},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 03:54:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.74299999999999999,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.54500000000000004}]},{"osd":6,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61799999999999999,"15min":0.625},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.374,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73699999999999999,"5min":0.874,"15min":0.874},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 03:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.622,"15min":0.629},"min":{"1min":0.432,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76500000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59899999999999998,"15min":0.60199999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.70599999999999996,"5min":0.877,"15min":0.877},"last":0.64300000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738427,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1272,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1302528,"data_stored":893320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 03:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32500000000000001,"5min":0.34699999999999998,"15min":0.34100000000000003},"min":{"1min":0.245,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.51300000000000001,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.315,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.44500000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51900000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.433,"15min":0.433},"min":{"1min":0.29599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.45400000000000001,"15min":0.47099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.48899999999999999,"5min":0.623,"15min":0.623},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.48699999999999999,"15min":0.51300000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55500000000000005,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Oct 9 03:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.58099999999999996,"15min":0.59699999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.70099999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64800000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46000000000000002}]},{"osd":5,"last update":"Mon Oct 9 03:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.61199999999999999,"15min":0.62},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.76500000000000001,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60999999999999999,"15min":0.622},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.72799999999999998,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.48899999999999999}]},{"osd":7,"last update":"Mon Oct 9 03:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.70599999999999996,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.60199999999999998,"15min":0.61199999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70799999999999996,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.58099999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":704512,"data_stored":302699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":0.35699999999999998,"15min":0.35299999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45000000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.376,"15min":0.37},"min":{"1min":0.26600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.54100000000000004,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.69899999999999995}]},{"osd":1,"last update":"Mon Oct 9 03:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52800000000000002,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.308},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.27000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 03:54:28 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42799999999999999,"15min":0.42999999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.495,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.434,"15min":0.435},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56599999999999995,"15min":0.57899999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73199999999999998,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.627},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58399999999999996,"15min":0.59699999999999998},"min":{"1min":0.46000000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79300000000000004,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 03:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.60399999999999998,"15min":0.61799999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.753,"5min":0.876,"15min":0.876},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 03:54:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.628,"15min":0.64500000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73499999999999999,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.65300000000000002,"15min":0.66300000000000003},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.81299999999999994,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.53400000000000003}]},{"osd":7,"last update":"Mon Oct 9 03:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60399999999999998,"15min":0.60099999999999998},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":0.77300000000000002,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.82699999999999996,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.72999999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149742,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":704512,"data_stored":302466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.31,"5min":0.37,"15min":0.38700000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.40899999999999997,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.39900000000000002,"15min":0.40899999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55700000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.44}]},{"osd":1,"last update":"Mon Oct 9 03:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.40200000000000002,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.46700000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.57499999999999996,"5min":1.518,"15min":1.518},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.47799999999999998,"15min":0.48599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.54800000000000004,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.441,"15min":0.45700000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56399999999999995,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.53500000000000003}]},{"osd":4,"last update":"Mon Oct 9 03:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.59199999999999997,"15min":0.59999999999999998},"min":{"1min":0.375,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85399999999999998,"5min":1.611,"15min":1.611},"last":0.623},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63300000000000001,"15min":0.64400000000000002},"min":{"1min":0.43099999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.82399999999999995,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.71099999999999997}]},{"osd":5,"last update":"Mon Oct 9 03:54:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57599999999999996,"15min":0.57899999999999996},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.502,"5min":0.57299999999999995,"15min":0.58499999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76900000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.56999999999999995,"15min":0.58099999999999996},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72099999999999997,"5min":0.753,"15min":0.753},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57799999999999996,"15min":0.58099999999999996},"min":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.749}]},{"osd":7,"last update":"Mon Oct 9 03:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45200000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.60999999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986217,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297992,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447160,"statfs":{"total":95995035648,"available":95689891840,"internally_reserved":0,"allocated":720896,"data_stored":303793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.54000000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.64100000000000001,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.621},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.55500000000000005},"min":{"1min":0.40000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.59099999999999997,"5min":0.753,"15min":0.753},"last":0.67700000000000005}]},{"osd":1,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.54200000000000004,"15min":0.54900000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68200000000000005,"5min":1.139,"15min":1.139},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.53000000000000003,"15min":0.54200000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.60799999999999998,"5min":1.05,"15min":1.05},"last":0.69599999999999995}]},{"osd":2,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.54500000000000004},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.625,"5min":1.625,"15min":1.625},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.40100000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.64700000000000002,"5min":0.75,"15min":0.75},"last":0.58399999999999996}]},{"osd":3,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.38300000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66700000000000004,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.629},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35499999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72099999999999997,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.64600000000000002}]},{"osd":5,"last update":"Mon Oct 9 03:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.378,"15min":0.36899999999999999},"min":{"1min":0.29799999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.40999999999999998,"15min":0.39500000000000002},"min":{"1min":0.314,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.47699999999999998,"15min":0.46700000000000003},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.73399999999999999,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.48399999999999999,"15min":0.47899999999999998},"min":{"1min":0.39400000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64600000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 03:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.50800000000000001,"15min":0.5},"min":{"1min":0.38400000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.59499999999999997,"15min":0.59799999999999998},"min":{"1min":0.39700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77100000000000002,"5min":1.079,"15min":1.079},"last":0.70599999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724581,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":704512,"data_stored":302677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.40600000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.82799999999999996,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.84899999999999998,"5min":1.002,"15min":1.002},"last":0.501},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.35599999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.90200000000000002,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83799999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.47299999999999998,"15min":0.46899999999999997},"min":{"1min":0.32700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.47999999999999998},"min":{"1min":0.34899999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 03:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.504,"15min":0.51100000000000001},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.68400000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.496,"5min":0.53100000000000003,"15min":0.53700000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.77000000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 03:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.59999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T03:54:56.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T03:54:57.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:57 smithi163 ceph-mon[118807]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:57.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:57 smithi163 ceph-mon[101022]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:57.348 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:54:57.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:57 smithi181 ceph-mon[105375]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:54:58.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:54:58 smithi181 ceph-mon[105375]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:58.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:54:58 smithi163 ceph-mon[118807]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:58.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:54:58 smithi163 ceph-mon[101022]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:54:59.049 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:54:59.051 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T03:54:59.734 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":62,"stamp":"2023-10-09T03:54:57.122734+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385596,"kb_used_data":7292,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575620,"statfs":{"total":767960285184,"available":765517434880,"internally_reserved":0,"allocated":7467008,"data_stored":4195854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002344"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.898939+0000","last_change":"2023-10-09T03:54:13.386628+0000","last_active":"2023-10-09T03:54:14.898939+0000","last_peered":"2023-10-09T03:54:14.898939+0000","last_clean":"2023-10-09T03:54:14.898939+0000","last_became_active":"2023-10-09T03:54:13.385451+0000","last_became_peered":"2023-10-09T03:54:13.385451+0000","last_unstale":"2023-10-09T03:54:14.898939+0000","last_undegraded":"2023-10-09T03:54:14.898939+0000","last_fullsized":"2023-10-09T03:54:14.898939+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:31.987326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.182562+0000","last_change":"2023-10-09T03:54:13.220219+0000","last_active":"2023-10-09T03:54:23.182562+0000","last_peered":"2023-10-09T03:54:23.182562+0000","last_clean":"2023-10-09T03:54:23.182562+0000","last_became_active":"2023-10-09T03:54:13.218725+0000","last_became_peered":"2023-10-09T03:54:13.218725+0000","last_unstale":"2023-10-09T03:54:23.182562+0000","last_undegraded":"2023-10-09T03:54:23.182562+0000","last_fullsized":"2023-10-09T03:54:23.182562+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:23.774862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234877+0000","last_change":"2023-10-09T03:54:13.242838+0000","last_active":"2023-10-09T03:54:14.234877+0000","last_peered":"2023-10-09T03:54:14.234877+0000","last_clean":"2023-10-09T03:54:14.234877+0000","last_became_active":"2023-10-09T03:54:13.242548+0000","last_became_peered":"2023-10-09T03:54:13.242548+0000","last_unstale":"2023-10-09T03:54:14.234877+0000","last_undegraded":"2023-10-09T03:54:14.234877+0000","last_fullsized":"2023-10-09T03:54:14.234877+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:02.406483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.240193+0000","last_change":"2023-10-09T03:54:13.228008+0000","last_active":"2023-10-09T03:54:15.240193+0000","last_peered":"2023-10-09T03:54:15.240193+0000","last_clean":"2023-10-09T03:54:15.240193+0000","last_became_active":"2023-10-09T03:54:13.227440+0000","last_became_peered":"2023-10-09T03:54:13.227440+0000","last_unstale":"2023-10-09T03:54:15.240193+0000","last_undegraded":"2023-10-09T03:54:15.240193+0000","last_fullsized":"2023-10-09T03:54:15.240193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:48.443487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899114+0000","last_change":"2023-10-09T03:54:13.242320+0000","last_active":"2023-10-09T03:54:14.899114+0000","last_peered":"2023-10-09T03:54:14.899114+0000","last_clean":"2023-10-09T03:54:14.899114+0000","last_became_active":"2023-10-09T03:54:13.241937+0000","last_became_peered":"2023-10-09T03:54:13.241937+0000","last_unstale":"2023-10-09T03:54:14.899114+0000","last_undegraded":"2023-10-09T03:54:14.899114+0000","last_fullsized":"2023-10-09T03:54:14.899114+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:03:44.435784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.185528+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T03:54:23.185528+0000","last_peered":"2023-10-09T03:54:23.185528+0000","last_clean":"2023-10-09T03:54:23.185528+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T03:54:23.185528+0000","last_undegraded":"2023-10-09T03:54:23.185528+0000","last_fullsized":"2023-10-09T03:54:23.185528+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899214+0000","last_change":"2023-10-09T03:54:13.387084+0000","last_active":"2023-10-09T03:54:14.899214+0000","last_peered":"2023-10-09T03:54:14.899214+0000","last_clean":"2023-10-09T03:54:14.899214+0000","last_became_active":"2023-10-09T03:54:13.386739+0000","last_became_peered":"2023-10-09T03:54:13.386739+0000","last_unstale":"2023-10-09T03:54:14.899214+0000","last_undegraded":"2023-10-09T03:54:14.899214+0000","last_fullsized":"2023-10-09T03:54:14.899214+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:18.787228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.184647+0000","last_change":"2023-10-09T03:54:13.226919+0000","last_active":"2023-10-09T03:54:23.184647+0000","last_peered":"2023-10-09T03:54:23.184647+0000","last_clean":"2023-10-09T03:54:23.184647+0000","last_became_active":"2023-10-09T03:54:13.226604+0000","last_became_peered":"2023-10-09T03:54:13.226604+0000","last_unstale":"2023-10-09T03:54:23.184647+0000","last_undegraded":"2023-10-09T03:54:23.184647+0000","last_fullsized":"2023-10-09T03:54:23.184647+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:59.895137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.172527+0000","last_change":"2023-10-09T03:54:13.386113+0000","last_active":"2023-10-09T03:54:15.172527+0000","last_peered":"2023-10-09T03:54:15.172527+0000","last_clean":"2023-10-09T03:54:15.172527+0000","last_became_active":"2023-10-09T03:54:13.385774+0000","last_became_peered":"2023-10-09T03:54:13.385774+0000","last_unstale":"2023-10-09T03:54:15.172527+0000","last_undegraded":"2023-10-09T03:54:15.172527+0000","last_fullsized":"2023-10-09T03:54:15.172527+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:13.981606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:16.431952+0000","last_change":"2023-10-09T03:54:13.231818+0000","last_active":"2023-10-09T03:54:16.431952+0000","last_peered":"2023-10-09T03:54:16.431952+0000","last_clean":"2023-10-09T03:54:16.431952+0000","last_became_active":"2023-10-09T03:54:13.231557+0000","last_became_peered":"2023-10-09T03:54:13.231557+0000","last_unstale":"2023-10-09T03:54:16.431952+0000","last_undegraded":"2023-10-09T03:54:16.431952+0000","last_fullsized":"2023-10-09T03:54:16.431952+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:07.072823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899161+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T03:54:14.899161+0000","last_peered":"2023-10-09T03:54:14.899161+0000","last_clean":"2023-10-09T03:54:14.899161+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T03:54:14.899161+0000","last_undegraded":"2023-10-09T03:54:14.899161+0000","last_fullsized":"2023-10-09T03:54:14.899161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:16.432040+0000","last_change":"2023-10-09T03:54:13.226657+0000","last_active":"2023-10-09T03:54:16.432040+0000","last_peered":"2023-10-09T03:54:16.432040+0000","last_clean":"2023-10-09T03:54:16.432040+0000","last_became_active":"2023-10-09T03:54:13.226298+0000","last_became_peered":"2023-10-09T03:54:13.226298+0000","last_unstale":"2023-10-09T03:54:16.432040+0000","last_undegraded":"2023-10-09T03:54:16.432040+0000","last_fullsized":"2023-10-09T03:54:16.432040+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:29.543896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425618+0000","last_change":"2023-10-09T03:53:20.111685+0000","last_active":"2023-10-09T03:54:14.425618+0000","last_peered":"2023-10-09T03:54:14.425618+0000","last_clean":"2023-10-09T03:54:14.425618+0000","last_became_active":"2023-10-09T03:53:20.111378+0000","last_became_peered":"2023-10-09T03:53:20.111378+0000","last_unstale":"2023-10-09T03:54:14.425618+0000","last_undegraded":"2023-10-09T03:54:14.425618+0000","last_fullsized":"2023-10-09T03:54:14.425618+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:41.663034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235270+0000","last_change":"2023-10-09T03:54:13.390094+0000","last_active":"2023-10-09T03:54:14.235270+0000","last_peered":"2023-10-09T03:54:14.235270+0000","last_clean":"2023-10-09T03:54:14.235270+0000","last_became_active":"2023-10-09T03:54:13.389687+0000","last_became_peered":"2023-10-09T03:54:13.389687+0000","last_unstale":"2023-10-09T03:54:14.235270+0000","last_undegraded":"2023-10-09T03:54:14.235270+0000","last_fullsized":"2023-10-09T03:54:14.235270+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:29.050861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.233898+0000","last_change":"2023-10-09T03:53:20.474740+0000","last_active":"2023-10-09T03:54:14.233898+0000","last_peered":"2023-10-09T03:54:14.233898+0000","last_clean":"2023-10-09T03:54:14.233898+0000","last_became_active":"2023-10-09T03:53:20.474458+0000","last_became_peered":"2023-10-09T03:53:20.474458+0000","last_unstale":"2023-10-09T03:54:14.233898+0000","last_undegraded":"2023-10-09T03:54:14.233898+0000","last_fullsized":"2023-10-09T03:54:14.233898+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:26.858718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.181642+0000","last_change":"2023-10-09T03:54:13.226015+0000","last_active":"2023-10-09T03:54:23.181642+0000","last_peered":"2023-10-09T03:54:23.181642+0000","last_clean":"2023-10-09T03:54:23.181642+0000","last_became_active":"2023-10-09T03:54:13.225742+0000","last_became_peered":"2023-10-09T03:54:13.225742+0000","last_unstale":"2023-10-09T03:54:23.181642+0000","last_undegraded":"2023-10-09T03:54:23.181642+0000","last_fullsized":"2023-10-09T03:54:23.181642+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:23.722906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901999+0000","last_change":"2023-10-09T03:54:13.387150+0000","last_active":"2023-10-09T03:54:14.901999+0000","last_peered":"2023-10-09T03:54:14.901999+0000","last_clean":"2023-10-09T03:54:14.901999+0000","last_became_active":"2023-10-09T03:54:13.386847+0000","last_became_peered":"2023-10-09T03:54:13.386847+0000","last_unstale":"2023-10-09T03:54:14.901999+0000","last_undegraded":"2023-10-09T03:54:14.901999+0000","last_fullsized":"2023-10-09T03:54:14.901999+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:49.764730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235371+0000","last_change":"2023-10-09T03:54:13.231307+0000","last_active":"2023-10-09T03:54:14.235371+0000","last_peered":"2023-10-09T03:54:14.235371+0000","last_clean":"2023-10-09T03:54:14.235371+0000","last_became_active":"2023-10-09T03:54:13.230984+0000","last_became_peered":"2023-10-09T03:54:13.230984+0000","last_unstale":"2023-10-09T03:54:14.235371+0000","last_undegraded":"2023-10-09T03:54:14.235371+0000","last_fullsized":"2023-10-09T03:54:14.235371+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:39.291581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.632047+0000","last_change":"2023-10-09T03:53:20.474716+0000","last_active":"2023-10-09T03:54:15.632047+0000","last_peered":"2023-10-09T03:54:15.632047+0000","last_clean":"2023-10-09T03:54:15.632047+0000","last_became_active":"2023-10-09T03:53:20.474362+0000","last_became_peered":"2023-10-09T03:53:20.474362+0000","last_unstale":"2023-10-09T03:54:15.632047+0000","last_undegraded":"2023-10-09T03:54:15.632047+0000","last_fullsized":"2023-10-09T03:54:15.632047+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:35.972389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902127+0000","last_change":"2023-10-09T03:54:13.390577+0000","last_active":"2023-10-09T03:54:14.902127+0000","last_peered":"2023-10-09T03:54:14.902127+0000","last_clean":"2023-10-09T03:54:14.902127+0000","last_became_active":"2023-10-09T03:54:13.390285+0000","last_became_peered":"2023-10-09T03:54:13.390285+0000","last_unstale":"2023-10-09T03:54:14.902127+0000","last_undegraded":"2023-10-09T03:54:14.902127+0000","last_fullsized":"2023-10-09T03:54:14.902127+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:42.866073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901832+0000","last_change":"2023-10-09T03:54:13.224078+0000","last_active":"2023-10-09T03:54:14.901832+0000","last_peered":"2023-10-09T03:54:14.901832+0000","last_clean":"2023-10-09T03:54:14.901832+0000","last_became_active":"2023-10-09T03:54:13.223768+0000","last_became_peered":"2023-10-09T03:54:13.223768+0000","last_unstale":"2023-10-09T03:54:14.901832+0000","last_undegraded":"2023-10-09T03:54:14.901832+0000","last_fullsized":"2023-10-09T03:54:14.901832+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:24.072600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631554+0000","last_change":"2023-10-09T03:53:20.112489+0000","last_active":"2023-10-09T03:54:15.631554+0000","last_peered":"2023-10-09T03:54:15.631554+0000","last_clean":"2023-10-09T03:54:15.631554+0000","last_became_active":"2023-10-09T03:53:20.112217+0000","last_became_peered":"2023-10-09T03:53:20.112217+0000","last_unstale":"2023-10-09T03:54:15.631554+0000","last_undegraded":"2023-10-09T03:54:15.631554+0000","last_fullsized":"2023-10-09T03:54:15.631554+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:06.850905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425546+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T03:54:14.425546+0000","last_peered":"2023-10-09T03:54:14.425546+0000","last_clean":"2023-10-09T03:54:14.425546+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T03:54:14.425546+0000","last_undegraded":"2023-10-09T03:54:14.425546+0000","last_fullsized":"2023-10-09T03:54:14.425546+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234468+0000","last_change":"2023-10-09T03:54:13.228292+0000","last_active":"2023-10-09T03:54:14.234468+0000","last_peered":"2023-10-09T03:54:14.234468+0000","last_clean":"2023-10-09T03:54:14.234468+0000","last_became_active":"2023-10-09T03:54:13.227985+0000","last_became_peered":"2023-10-09T03:54:13.227985+0000","last_unstale":"2023-10-09T03:54:14.234468+0000","last_undegraded":"2023-10-09T03:54:14.234468+0000","last_fullsized":"2023-10-09T03:54:14.234468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:31.580822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":276,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631466+0000","last_change":"2023-10-09T03:52:48.503079+0000","last_active":"2023-10-09T03:54:15.631466+0000","last_peered":"2023-10-09T03:54:15.631466+0000","last_clean":"2023-10-09T03:54:15.631466+0000","last_became_active":"2023-10-09T03:52:48.502616+0000","last_became_peered":"2023-10-09T03:52:48.502616+0000","last_unstale":"2023-10-09T03:54:15.631466+0000","last_undegraded":"2023-10-09T03:54:15.631466+0000","last_fullsized":"2023-10-09T03:54:15.631466+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T03:51:17.354073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238147+0000","last_change":"2023-10-09T03:53:20.111364+0000","last_active":"2023-10-09T03:54:14.238147+0000","last_peered":"2023-10-09T03:54:14.238147+0000","last_clean":"2023-10-09T03:54:14.238147+0000","last_became_active":"2023-10-09T03:53:20.111052+0000","last_became_peered":"2023-10-09T03:53:20.111052+0000","last_unstale":"2023-10-09T03:54:14.238147+0000","last_undegraded":"2023-10-09T03:54:14.238147+0000","last_fullsized":"2023-10-09T03:54:14.238147+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:41.535319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902923+0000","last_change":"2023-10-09T03:54:14.902726+0000","last_active":"2023-10-09T03:54:14.902923+0000","last_peered":"2023-10-09T03:54:14.902923+0000","last_clean":"2023-10-09T03:54:14.902923+0000","last_became_active":"2023-10-09T03:54:14.902459+0000","last_became_peered":"2023-10-09T03:54:14.902459+0000","last_unstale":"2023-10-09T03:54:14.902923+0000","last_undegraded":"2023-10-09T03:54:14.902923+0000","last_fullsized":"2023-10-09T03:54:14.902923+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:16.261746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234458+0000","last_change":"2023-10-09T03:54:13.231206+0000","last_active":"2023-10-09T03:54:14.234458+0000","last_peered":"2023-10-09T03:54:14.234458+0000","last_clean":"2023-10-09T03:54:14.234458+0000","last_became_active":"2023-10-09T03:54:13.230934+0000","last_became_peered":"2023-10-09T03:54:13.230934+0000","last_unstale":"2023-10-09T03:54:14.234458+0000","last_undegraded":"2023-10-09T03:54:14.234458+0000","last_fullsized":"2023-10-09T03:54:14.234458+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:57.235449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.172114+0000","last_change":"2023-10-09T03:53:20.475582+0000","last_active":"2023-10-09T03:54:15.172114+0000","last_peered":"2023-10-09T03:54:15.172114+0000","last_clean":"2023-10-09T03:54:15.172114+0000","last_became_active":"2023-10-09T03:53:20.475313+0000","last_became_peered":"2023-10-09T03:53:20.475313+0000","last_unstale":"2023-10-09T03:54:15.172114+0000","last_undegraded":"2023-10-09T03:54:15.172114+0000","last_fullsized":"2023-10-09T03:54:15.172114+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:25.695798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.174161+0000","last_change":"2023-10-09T03:54:15.173633+0000","last_active":"2023-10-09T03:54:15.174161+0000","last_peered":"2023-10-09T03:54:15.174161+0000","last_clean":"2023-10-09T03:54:15.174161+0000","last_became_active":"2023-10-09T03:54:15.173293+0000","last_became_peered":"2023-10-09T03:54:15.173293+0000","last_unstale":"2023-10-09T03:54:15.174161+0000","last_undegraded":"2023-10-09T03:54:15.174161+0000","last_fullsized":"2023-10-09T03:54:15.174161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:57.842218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899224+0000","last_change":"2023-10-09T03:54:13.223301+0000","last_active":"2023-10-09T03:54:14.899224+0000","last_peered":"2023-10-09T03:54:14.899224+0000","last_clean":"2023-10-09T03:54:14.899224+0000","last_became_active":"2023-10-09T03:54:13.223024+0000","last_became_peered":"2023-10-09T03:54:13.223024+0000","last_unstale":"2023-10-09T03:54:14.899224+0000","last_undegraded":"2023-10-09T03:54:14.899224+0000","last_fullsized":"2023-10-09T03:54:14.899224+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:58.715626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238206+0000","last_change":"2023-10-09T03:53:20.109068+0000","last_active":"2023-10-09T03:54:14.238206+0000","last_peered":"2023-10-09T03:54:14.238206+0000","last_clean":"2023-10-09T03:54:14.238206+0000","last_became_active":"2023-10-09T03:53:20.108811+0000","last_became_peered":"2023-10-09T03:53:20.108811+0000","last_unstale":"2023-10-09T03:54:14.238206+0000","last_undegraded":"2023-10-09T03:54:14.238206+0000","last_fullsized":"2023-10-09T03:54:14.238206+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.169451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.900637+0000","last_change":"2023-10-09T03:54:14.900151+0000","last_active":"2023-10-09T03:54:14.900637+0000","last_peered":"2023-10-09T03:54:14.900637+0000","last_clean":"2023-10-09T03:54:14.900637+0000","last_became_active":"2023-10-09T03:54:14.899807+0000","last_became_peered":"2023-10-09T03:54:14.899807+0000","last_unstale":"2023-10-09T03:54:14.900637+0000","last_undegraded":"2023-10-09T03:54:14.900637+0000","last_fullsized":"2023-10-09T03:54:14.900637+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:42.682264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238418+0000","last_change":"2023-10-09T03:54:13.226097+0000","last_active":"2023-10-09T03:54:14.238418+0000","last_peered":"2023-10-09T03:54:14.238418+0000","last_clean":"2023-10-09T03:54:14.238418+0000","last_became_active":"2023-10-09T03:54:13.225797+0000","last_became_peered":"2023-10-09T03:54:13.225797+0000","last_unstale":"2023-10-09T03:54:14.238418+0000","last_undegraded":"2023-10-09T03:54:14.238418+0000","last_fullsized":"2023-10-09T03:54:14.238418+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:24:34.378883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235384+0000","last_change":"2023-10-09T03:53:20.110963+0000","last_active":"2023-10-09T03:54:14.235384+0000","last_peered":"2023-10-09T03:54:14.235384+0000","last_clean":"2023-10-09T03:54:14.235384+0000","last_became_active":"2023-10-09T03:53:20.110726+0000","last_became_peered":"2023-10-09T03:53:20.110726+0000","last_unstale":"2023-10-09T03:54:14.235384+0000","last_undegraded":"2023-10-09T03:54:14.235384+0000","last_fullsized":"2023-10-09T03:54:14.235384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:53.134560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.900634+0000","last_change":"2023-10-09T03:54:14.900142+0000","last_active":"2023-10-09T03:54:14.900634+0000","last_peered":"2023-10-09T03:54:14.900634+0000","last_clean":"2023-10-09T03:54:14.900634+0000","last_became_active":"2023-10-09T03:54:14.899825+0000","last_became_peered":"2023-10-09T03:54:14.899825+0000","last_unstale":"2023-10-09T03:54:14.900634+0000","last_undegraded":"2023-10-09T03:54:14.900634+0000","last_fullsized":"2023-10-09T03:54:14.900634+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:58.614149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235362+0000","last_change":"2023-10-09T03:54:13.225693+0000","last_active":"2023-10-09T03:54:14.235362+0000","last_peered":"2023-10-09T03:54:14.235362+0000","last_clean":"2023-10-09T03:54:14.235362+0000","last_became_active":"2023-10-09T03:54:13.225412+0000","last_became_peered":"2023-10-09T03:54:13.225412+0000","last_unstale":"2023-10-09T03:54:14.235362+0000","last_undegraded":"2023-10-09T03:54:14.235362+0000","last_fullsized":"2023-10-09T03:54:14.235362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:50.987303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899376+0000","last_change":"2023-10-09T03:54:13.392302+0000","last_active":"2023-10-09T03:54:14.899376+0000","last_peered":"2023-10-09T03:54:14.899376+0000","last_clean":"2023-10-09T03:54:14.899376+0000","last_became_active":"2023-10-09T03:54:13.391911+0000","last_became_peered":"2023-10-09T03:54:13.391911+0000","last_unstale":"2023-10-09T03:54:14.899376+0000","last_undegraded":"2023-10-09T03:54:14.899376+0000","last_fullsized":"2023-10-09T03:54:14.899376+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:55.954773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899258+0000","last_change":"2023-10-09T03:54:13.222214+0000","last_active":"2023-10-09T03:54:14.899258+0000","last_peered":"2023-10-09T03:54:14.899258+0000","last_clean":"2023-10-09T03:54:14.899258+0000","last_became_active":"2023-10-09T03:54:13.221328+0000","last_became_peered":"2023-10-09T03:54:13.221328+0000","last_unstale":"2023-10-09T03:54:14.899258+0000","last_undegraded":"2023-10-09T03:54:14.899258+0000","last_fullsized":"2023-10-09T03:54:14.899258+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:56.353256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899476+0000","last_change":"2023-10-09T03:54:13.386688+0000","last_active":"2023-10-09T03:54:14.899476+0000","last_peered":"2023-10-09T03:54:14.899476+0000","last_clean":"2023-10-09T03:54:14.899476+0000","last_became_active":"2023-10-09T03:54:13.386053+0000","last_became_peered":"2023-10-09T03:54:13.386053+0000","last_unstale":"2023-10-09T03:54:14.899476+0000","last_undegraded":"2023-10-09T03:54:14.899476+0000","last_fullsized":"2023-10-09T03:54:14.899476+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:24.011861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425506+0000","last_change":"2023-10-09T03:54:13.230286+0000","last_active":"2023-10-09T03:54:14.425506+0000","last_peered":"2023-10-09T03:54:14.425506+0000","last_clean":"2023-10-09T03:54:14.425506+0000","last_became_active":"2023-10-09T03:54:13.229933+0000","last_became_peered":"2023-10-09T03:54:13.229933+0000","last_unstale":"2023-10-09T03:54:14.425506+0000","last_undegraded":"2023-10-09T03:54:14.425506+0000","last_fullsized":"2023-10-09T03:54:14.425506+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:34.348293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899226+0000","last_change":"2023-10-09T03:54:13.242037+0000","last_active":"2023-10-09T03:54:14.899226+0000","last_peered":"2023-10-09T03:54:14.899226+0000","last_clean":"2023-10-09T03:54:14.899226+0000","last_became_active":"2023-10-09T03:54:13.241753+0000","last_became_peered":"2023-10-09T03:54:13.241753+0000","last_unstale":"2023-10-09T03:54:14.899226+0000","last_undegraded":"2023-10-09T03:54:14.899226+0000","last_fullsized":"2023-10-09T03:54:14.899226+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:49.328661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183195+0000","last_change":"2023-10-09T03:54:13.226705+0000","last_active":"2023-10-09T03:54:23.183195+0000","last_peered":"2023-10-09T03:54:23.183195+0000","last_clean":"2023-10-09T03:54:23.183195+0000","last_became_active":"2023-10-09T03:54:13.226335+0000","last_became_peered":"2023-10-09T03:54:13.226335+0000","last_unstale":"2023-10-09T03:54:23.183195+0000","last_undegraded":"2023-10-09T03:54:23.183195+0000","last_fullsized":"2023-10-09T03:54:23.183195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:35.033771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901910+0000","last_change":"2023-10-09T03:54:13.387252+0000","last_active":"2023-10-09T03:54:14.901910+0000","last_peered":"2023-10-09T03:54:14.901910+0000","last_clean":"2023-10-09T03:54:14.901910+0000","last_became_active":"2023-10-09T03:54:13.386963+0000","last_became_peered":"2023-10-09T03:54:13.386963+0000","last_unstale":"2023-10-09T03:54:14.901910+0000","last_undegraded":"2023-10-09T03:54:14.901910+0000","last_fullsized":"2023-10-09T03:54:14.901910+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:19:00.954422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238265+0000","last_change":"2023-10-09T03:54:13.222789+0000","last_active":"2023-10-09T03:54:14.238265+0000","last_peered":"2023-10-09T03:54:14.238265+0000","last_clean":"2023-10-09T03:54:14.238265+0000","last_became_active":"2023-10-09T03:54:13.222433+0000","last_became_peered":"2023-10-09T03:54:13.222433+0000","last_unstale":"2023-10-09T03:54:14.238265+0000","last_undegraded":"2023-10-09T03:54:14.238265+0000","last_fullsized":"2023-10-09T03:54:14.238265+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:53.372287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425340+0000","last_change":"2023-10-09T03:54:13.387297+0000","last_active":"2023-10-09T03:54:14.425340+0000","last_peered":"2023-10-09T03:54:14.425340+0000","last_clean":"2023-10-09T03:54:14.425340+0000","last_became_active":"2023-10-09T03:54:13.386982+0000","last_became_peered":"2023-10-09T03:54:13.386982+0000","last_unstale":"2023-10-09T03:54:14.425340+0000","last_undegraded":"2023-10-09T03:54:14.425340+0000","last_fullsized":"2023-10-09T03:54:14.425340+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:00.680974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:17.134615+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T03:54:17.134615+0000","last_peered":"2023-10-09T03:54:17.134615+0000","last_clean":"2023-10-09T03:54:17.134615+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T03:54:17.134615+0000","last_undegraded":"2023-10-09T03:54:17.134615+0000","last_fullsized":"2023-10-09T03:54:17.134615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899061+0000","last_change":"2023-10-09T03:54:13.385895+0000","last_active":"2023-10-09T03:54:14.899061+0000","last_peered":"2023-10-09T03:54:14.899061+0000","last_clean":"2023-10-09T03:54:14.899061+0000","last_became_active":"2023-10-09T03:54:13.385558+0000","last_became_peered":"2023-10-09T03:54:13.385558+0000","last_unstale":"2023-10-09T03:54:14.899061+0000","last_undegraded":"2023-10-09T03:54:14.899061+0000","last_fullsized":"2023-10-09T03:54:14.899061+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:10.759913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183590+0000","last_change":"2023-10-09T03:54:13.226606+0000","last_active":"2023-10-09T03:54:23.183590+0000","last_peered":"2023-10-09T03:54:23.183590+0000","last_clean":"2023-10-09T03:54:23.183590+0000","last_became_active":"2023-10-09T03:54:13.226283+0000","last_became_peered":"2023-10-09T03:54:13.226283+0000","last_unstale":"2023-10-09T03:54:23.183590+0000","last_undegraded":"2023-10-09T03:54:23.183590+0000","last_fullsized":"2023-10-09T03:54:23.183590+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:46.736265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238147+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T03:54:14.238147+0000","last_peered":"2023-10-09T03:54:14.238147+0000","last_clean":"2023-10-09T03:54:14.238147+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T03:54:14.238147+0000","last_undegraded":"2023-10-09T03:54:14.238147+0000","last_fullsized":"2023-10-09T03:54:14.238147+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:17.134741+0000","last_change":"2023-10-09T03:54:13.231923+0000","last_active":"2023-10-09T03:54:17.134741+0000","last_peered":"2023-10-09T03:54:17.134741+0000","last_clean":"2023-10-09T03:54:17.134741+0000","last_became_active":"2023-10-09T03:54:13.231559+0000","last_became_peered":"2023-10-09T03:54:13.231559+0000","last_unstale":"2023-10-09T03:54:17.134741+0000","last_undegraded":"2023-10-09T03:54:17.134741+0000","last_fullsized":"2023-10-09T03:54:17.134741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:49.479131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.233729+0000","last_change":"2023-10-09T03:54:13.388403+0000","last_active":"2023-10-09T03:54:14.233729+0000","last_peered":"2023-10-09T03:54:14.233729+0000","last_clean":"2023-10-09T03:54:14.233729+0000","last_became_active":"2023-10-09T03:54:13.388091+0000","last_became_peered":"2023-10-09T03:54:13.388091+0000","last_unstale":"2023-10-09T03:54:14.233729+0000","last_undegraded":"2023-10-09T03:54:14.233729+0000","last_fullsized":"2023-10-09T03:54:14.233729+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:21.971508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.185595+0000","last_change":"2023-10-09T03:54:13.223658+0000","last_active":"2023-10-09T03:54:23.185595+0000","last_peered":"2023-10-09T03:54:23.185595+0000","last_clean":"2023-10-09T03:54:23.185595+0000","last_became_active":"2023-10-09T03:54:13.223344+0000","last_became_peered":"2023-10-09T03:54:13.223344+0000","last_unstale":"2023-10-09T03:54:23.185595+0000","last_undegraded":"2023-10-09T03:54:23.185595+0000","last_fullsized":"2023-10-09T03:54:23.185595+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:43.813381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901959+0000","last_change":"2023-10-09T03:54:13.389725+0000","last_active":"2023-10-09T03:54:14.901959+0000","last_peered":"2023-10-09T03:54:14.901959+0000","last_clean":"2023-10-09T03:54:14.901959+0000","last_became_active":"2023-10-09T03:54:13.389228+0000","last_became_peered":"2023-10-09T03:54:13.389228+0000","last_unstale":"2023-10-09T03:54:14.901959+0000","last_undegraded":"2023-10-09T03:54:14.901959+0000","last_fullsized":"2023-10-09T03:54:14.901959+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:07.477701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631412+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T03:54:15.631412+0000","last_peered":"2023-10-09T03:54:15.631412+0000","last_clean":"2023-10-09T03:54:15.631412+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T03:54:15.631412+0000","last_undegraded":"2023-10-09T03:54:15.631412+0000","last_fullsized":"2023-10-09T03:54:15.631412+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238326+0000","last_change":"2023-10-09T03:54:13.390929+0000","last_active":"2023-10-09T03:54:14.238326+0000","last_peered":"2023-10-09T03:54:14.238326+0000","last_clean":"2023-10-09T03:54:14.238326+0000","last_became_active":"2023-10-09T03:54:13.389911+0000","last_became_peered":"2023-10-09T03:54:13.389911+0000","last_unstale":"2023-10-09T03:54:14.238326+0000","last_undegraded":"2023-10-09T03:54:14.238326+0000","last_fullsized":"2023-10-09T03:54:14.238326+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:16.431009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.901843+0000","last_change":"2023-10-09T03:54:13.230386+0000","last_active":"2023-10-09T03:54:14.901843+0000","last_peered":"2023-10-09T03:54:14.901843+0000","last_clean":"2023-10-09T03:54:14.901843+0000","last_became_active":"2023-10-09T03:54:13.230038+0000","last_became_peered":"2023-10-09T03:54:13.230038+0000","last_unstale":"2023-10-09T03:54:14.901843+0000","last_undegraded":"2023-10-09T03:54:14.901843+0000","last_fullsized":"2023-10-09T03:54:14.901843+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:25.790371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234308+0000","last_change":"2023-10-09T03:54:13.388361+0000","last_active":"2023-10-09T03:54:14.234308+0000","last_peered":"2023-10-09T03:54:14.234308+0000","last_clean":"2023-10-09T03:54:14.234308+0000","last_became_active":"2023-10-09T03:54:13.388054+0000","last_became_peered":"2023-10-09T03:54:13.388054+0000","last_unstale":"2023-10-09T03:54:14.234308+0000","last_undegraded":"2023-10-09T03:54:14.234308+0000","last_fullsized":"2023-10-09T03:54:14.234308+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:12.673234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183209+0000","last_change":"2023-10-09T03:54:13.228564+0000","last_active":"2023-10-09T03:54:23.183209+0000","last_peered":"2023-10-09T03:54:23.183209+0000","last_clean":"2023-10-09T03:54:23.183209+0000","last_became_active":"2023-10-09T03:54:13.228240+0000","last_became_peered":"2023-10-09T03:54:13.228240+0000","last_unstale":"2023-10-09T03:54:23.183209+0000","last_undegraded":"2023-10-09T03:54:23.183209+0000","last_fullsized":"2023-10-09T03:54:23.183209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:13.153308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.899130+0000","last_change":"2023-10-09T03:54:13.386245+0000","last_active":"2023-10-09T03:54:14.899130+0000","last_peered":"2023-10-09T03:54:14.899130+0000","last_clean":"2023-10-09T03:54:14.899130+0000","last_became_active":"2023-10-09T03:54:13.385685+0000","last_became_peered":"2023-10-09T03:54:13.385685+0000","last_unstale":"2023-10-09T03:54:14.899130+0000","last_undegraded":"2023-10-09T03:54:14.899130+0000","last_fullsized":"2023-10-09T03:54:14.899130+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.339050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.185891+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T03:54:23.185891+0000","last_peered":"2023-10-09T03:54:23.185891+0000","last_clean":"2023-10-09T03:54:23.185891+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T03:54:23.185891+0000","last_undegraded":"2023-10-09T03:54:23.185891+0000","last_fullsized":"2023-10-09T03:54:23.185891+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.174291+0000","last_change":"2023-10-09T03:54:15.173687+0000","last_active":"2023-10-09T03:54:15.174291+0000","last_peered":"2023-10-09T03:54:15.174291+0000","last_clean":"2023-10-09T03:54:15.174291+0000","last_became_active":"2023-10-09T03:54:15.173367+0000","last_became_peered":"2023-10-09T03:54:15.173367+0000","last_unstale":"2023-10-09T03:54:15.174291+0000","last_undegraded":"2023-10-09T03:54:15.174291+0000","last_fullsized":"2023-10-09T03:54:15.174291+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:15:59.955464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:23.183290+0000","last_change":"2023-10-09T03:54:13.226162+0000","last_active":"2023-10-09T03:54:23.183290+0000","last_peered":"2023-10-09T03:54:23.183290+0000","last_clean":"2023-10-09T03:54:23.183290+0000","last_became_active":"2023-10-09T03:54:13.225774+0000","last_became_peered":"2023-10-09T03:54:13.225774+0000","last_unstale":"2023-10-09T03:54:23.183290+0000","last_undegraded":"2023-10-09T03:54:23.183290+0000","last_fullsized":"2023-10-09T03:54:23.183290+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:03.496497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234997+0000","last_change":"2023-10-09T03:54:13.387399+0000","last_active":"2023-10-09T03:54:14.234997+0000","last_peered":"2023-10-09T03:54:14.234997+0000","last_clean":"2023-10-09T03:54:14.234997+0000","last_became_active":"2023-10-09T03:54:13.387085+0000","last_became_peered":"2023-10-09T03:54:13.387085+0000","last_unstale":"2023-10-09T03:54:14.234997+0000","last_undegraded":"2023-10-09T03:54:14.234997+0000","last_fullsized":"2023-10-09T03:54:14.234997+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:58.092470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.238384+0000","last_change":"2023-10-09T03:54:13.226204+0000","last_active":"2023-10-09T03:54:14.238384+0000","last_peered":"2023-10-09T03:54:14.238384+0000","last_clean":"2023-10-09T03:54:14.238384+0000","last_became_active":"2023-10-09T03:54:13.225925+0000","last_became_peered":"2023-10-09T03:54:13.225925+0000","last_unstale":"2023-10-09T03:54:14.238384+0000","last_undegraded":"2023-10-09T03:54:14.238384+0000","last_fullsized":"2023-10-09T03:54:14.238384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:02.027420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235712+0000","last_change":"2023-10-09T03:54:13.388994+0000","last_active":"2023-10-09T03:54:14.235712+0000","last_peered":"2023-10-09T03:54:14.235712+0000","last_clean":"2023-10-09T03:54:14.235712+0000","last_became_active":"2023-10-09T03:54:13.388715+0000","last_became_peered":"2023-10-09T03:54:13.388715+0000","last_unstale":"2023-10-09T03:54:14.235712+0000","last_undegraded":"2023-10-09T03:54:14.235712+0000","last_fullsized":"2023-10-09T03:54:14.235712+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:26.161561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902151+0000","last_change":"2023-10-09T03:54:13.223099+0000","last_active":"2023-10-09T03:54:14.902151+0000","last_peered":"2023-10-09T03:54:14.902151+0000","last_clean":"2023-10-09T03:54:14.902151+0000","last_became_active":"2023-10-09T03:54:13.222771+0000","last_became_peered":"2023-10-09T03:54:13.222771+0000","last_unstale":"2023-10-09T03:54:14.902151+0000","last_undegraded":"2023-10-09T03:54:14.902151+0000","last_fullsized":"2023-10-09T03:54:14.902151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:02.054533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.425595+0000","last_change":"2023-10-09T03:54:13.389291+0000","last_active":"2023-10-09T03:54:14.425595+0000","last_peered":"2023-10-09T03:54:14.425595+0000","last_clean":"2023-10-09T03:54:14.425595+0000","last_became_active":"2023-10-09T03:54:13.388934+0000","last_became_peered":"2023-10-09T03:54:13.388934+0000","last_unstale":"2023-10-09T03:54:14.425595+0000","last_undegraded":"2023-10-09T03:54:14.425595+0000","last_fullsized":"2023-10-09T03:54:14.425595+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:48:35.112863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.235740+0000","last_change":"2023-10-09T03:54:13.225218+0000","last_active":"2023-10-09T03:54:14.235740+0000","last_peered":"2023-10-09T03:54:14.235740+0000","last_clean":"2023-10-09T03:54:14.235740+0000","last_became_active":"2023-10-09T03:54:13.224870+0000","last_became_peered":"2023-10-09T03:54:13.224870+0000","last_unstale":"2023-10-09T03:54:14.235740+0000","last_undegraded":"2023-10-09T03:54:14.235740+0000","last_fullsized":"2023-10-09T03:54:14.235740+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:26.196818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.902017+0000","last_change":"2023-10-09T03:54:13.389519+0000","last_active":"2023-10-09T03:54:14.902017+0000","last_peered":"2023-10-09T03:54:14.902017+0000","last_clean":"2023-10-09T03:54:14.902017+0000","last_became_active":"2023-10-09T03:54:13.388817+0000","last_became_peered":"2023-10-09T03:54:13.388817+0000","last_unstale":"2023-10-09T03:54:14.902017+0000","last_undegraded":"2023-10-09T03:54:14.902017+0000","last_fullsized":"2023-10-09T03:54:14.902017+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:40.701614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.233572+0000","last_change":"2023-10-09T03:54:13.230822+0000","last_active":"2023-10-09T03:54:14.233572+0000","last_peered":"2023-10-09T03:54:14.233572+0000","last_clean":"2023-10-09T03:54:14.233572+0000","last_became_active":"2023-10-09T03:54:13.230496+0000","last_became_peered":"2023-10-09T03:54:13.230496+0000","last_unstale":"2023-10-09T03:54:14.233572+0000","last_undegraded":"2023-10-09T03:54:14.233572+0000","last_fullsized":"2023-10-09T03:54:14.233572+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:37.250589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T03:54:14.234098+0000","last_change":"2023-10-09T03:54:13.385615+0000","last_active":"2023-10-09T03:54:14.234098+0000","last_peered":"2023-10-09T03:54:14.234098+0000","last_clean":"2023-10-09T03:54:14.234098+0000","last_became_active":"2023-10-09T03:54:13.385341+0000","last_became_peered":"2023-10-09T03:54:13.385341+0000","last_unstale":"2023-10-09T03:54:14.234098+0000","last_undegraded":"2023-10-09T03:54:14.234098+0000","last_fullsized":"2023-10-09T03:54:14.234098+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:21.643886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T03:54:15.631407+0000","last_change":"2023-10-09T03:54:13.222567+0000","last_active":"2023-10-09T03:54:15.631407+0000","last_peered":"2023-10-09T03:54:15.631407+0000","last_clean":"2023-10-09T03:54:15.631407+0000","last_became_active":"2023-10-09T03:54:13.222226+0000","last_became_peered":"2023-10-09T03:54:13.222226+0000","last_unstale":"2023-10-09T03:54:15.631407+0000","last_undegraded":"2023-10-09T03:54:15.631407+0000","last_fullsized":"2023-10-09T03:54:15.631407+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:10.480578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201308,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298580,"kb_used_data":1292,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446572,"statfs":{"total":95995035648,"available":95689289728,"internally_reserved":0,"allocated":1323008,"data_stored":894496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.54200000000000004,"15min":0.52200000000000002},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.944,"5min":2.944,"15min":2.944},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.57299999999999995,"15min":0.52900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.3879999999999999,"5min":5.3879999999999999,"15min":5.3879999999999999},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.56100000000000005,"15min":0.53900000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.8969999999999998,"5min":2.8969999999999998,"15min":2.8969999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.621,"15min":0.56000000000000005},"min":{"1min":0.41199999999999998,"5min":0.37,"15min":0.37},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":7.8390000000000004},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.51900000000000002,"15min":0.50900000000000001},"min":{"1min":0.39200000000000002,"5min":0.37,"15min":0.37},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.624},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.61699999999999999,"15min":0.58899999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.2530000000000001,"5min":4.2530000000000001,"15min":4.2530000000000001},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.60099999999999998,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.7709999999999999,"5min":3.7709999999999999,"15min":3.7709999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.61499999999999999,"15min":0.59099999999999997},"min":{"1min":0.39500000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.6190000000000002,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.59699999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.56999999999999995,"15min":0.52300000000000002},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.318},"max":{"1min":6.3129999999999997,"5min":6.3129999999999997,"15min":6.3129999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.54000000000000004,"15min":0.50800000000000001},"min":{"1min":0.316,"5min":0.308,"15min":0.308},"max":{"1min":3.8769999999999998,"5min":3.8769999999999998,"15min":3.8769999999999998},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55900000000000005,"15min":0.55500000000000005},"min":{"1min":0.36099999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":0.495},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.1840000000000002,"5min":3.1840000000000002,"15min":3.1840000000000002},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 03:54:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.54100000000000004,"15min":0.52600000000000002},"min":{"1min":0.40699999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.57299999999999995,"15min":0.55900000000000005},"min":{"1min":0.439,"5min":0.373,"15min":0.373},"max":{"1min":1.9350000000000001,"5min":1.9350000000000001,"15min":1.9350000000000001},"last":0.72499999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462945,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298564,"kb_used_data":1276,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446588,"statfs":{"total":95995035648,"available":95689306112,"internally_reserved":0,"allocated":1306624,"data_stored":894027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.49199999999999999,"15min":0.48999999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.52600000000000002,"15min":0.51200000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.0990000000000002,"5min":2.0990000000000002,"15min":2.0990000000000002},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.30599999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.486,"5min":1.486,"15min":1.486},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.55400000000000005}]},{"osd":2,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.48299999999999998,"15min":0.47299999999999998},"min":{"1min":0.35099999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.505,"15min":0.49399999999999999},"min":{"1min":0.33500000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.86199999999999999,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.52800000000000002,"15min":0.51900000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3839999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.61799999999999999}]},{"osd":4,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.38700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57099999999999995,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.71899999999999997,"5min":0.995,"15min":0.995},"last":0.67600000000000005}]},{"osd":5,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.47499999999999998,"15min":0.46899999999999997},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69999999999999996,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.51900000000000002,"15min":0.51500000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.77100000000000002,"5min":1.03,"15min":1.03},"last":0.65600000000000003}]},{"osd":7,"last update":"Mon Oct 9 03:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.57199999999999995,"15min":0.56499999999999995},"min":{"1min":0.41799999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.93000000000000005,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.58199999999999996,"15min":0.57299999999999995},"min":{"1min":0.45800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":1.0089999999999999,"5min":1.077,"15min":1.077},"last":0.63400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574903,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":302376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.32800000000000001},"min":{"1min":0.249,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.46200000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.33500000000000002,"5min":0.36199999999999999,"15min":0.34000000000000002},"min":{"1min":0.23899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.5,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39100000000000001,"15min":0.38},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.53800000000000003,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.378},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.52700000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.26000000000000001}]},{"osd":3,"last update":"Mon Oct 9 03:54:21 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.42099999999999999,"15min":0.434},"min":{"1min":0.28899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55600000000000005,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45900000000000002,"15min":0.46800000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58299999999999996,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Oct 9 03:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68600000000000005,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71799999999999997,"5min":0.875,"15min":0.875},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 03:54:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.74299999999999999,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.54500000000000004}]},{"osd":6,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61799999999999999,"15min":0.625},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.374,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73699999999999999,"5min":0.874,"15min":0.874},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 03:54:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.622,"15min":0.629},"min":{"1min":0.432,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76500000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59899999999999998,"15min":0.60199999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.70599999999999996,"5min":0.877,"15min":0.877},"last":0.64300000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738427,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1272,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1302528,"data_stored":893320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 03:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32500000000000001,"5min":0.34699999999999998,"15min":0.34100000000000003},"min":{"1min":0.245,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.51300000000000001,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.315,"5min":0.36299999999999999,"15min":0.34799999999999998},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.44500000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26800000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51900000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.433,"15min":0.433},"min":{"1min":0.29599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.45400000000000001,"15min":0.47099999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.48899999999999999,"5min":0.623,"15min":0.623},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.48699999999999999,"15min":0.51300000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55500000000000005,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Oct 9 03:54:44 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.58099999999999996,"15min":0.59699999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.70099999999999996,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.56399999999999995,"15min":0.57399999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64800000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46000000000000002}]},{"osd":5,"last update":"Mon Oct 9 03:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.61199999999999999,"15min":0.62},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.76500000000000001,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60999999999999999,"15min":0.622},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.72799999999999998,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.48899999999999999}]},{"osd":7,"last update":"Mon Oct 9 03:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.70599999999999996,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.60199999999999998,"15min":0.61199999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70799999999999996,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.58099999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":704512,"data_stored":302699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":0.35699999999999998,"15min":0.35299999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45000000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.376,"15min":0.37},"min":{"1min":0.26600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.54100000000000004,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.69899999999999995}]},{"osd":1,"last update":"Mon Oct 9 03:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52800000000000002,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.308},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.27000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 03:54:28 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42799999999999999,"15min":0.42999999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.495,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.434,"15min":0.435},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 03:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56599999999999995,"15min":0.57899999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.73199999999999998,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.627},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58399999999999996,"15min":0.59699999999999998},"min":{"1min":0.46000000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79300000000000004,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 03:54:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.60399999999999998,"15min":0.61799999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.753,"5min":0.876,"15min":0.876},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 03:54:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.628,"15min":0.64500000000000002},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73499999999999999,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.65300000000000002,"15min":0.66300000000000003},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.81299999999999994,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.53400000000000003}]},{"osd":7,"last update":"Mon Oct 9 03:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60399999999999998,"15min":0.60099999999999998},"min":{"1min":0.44800000000000001,"5min":0.433,"15min":0.433},"max":{"1min":0.77300000000000002,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60299999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.82699999999999996,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.72999999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149742,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":704512,"data_stored":302466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.31,"5min":0.37,"15min":0.38700000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.40899999999999997,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.39900000000000002,"15min":0.40899999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55700000000000005,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.44}]},{"osd":1,"last update":"Mon Oct 9 03:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.40200000000000002,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.46700000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.57499999999999996,"5min":1.518,"15min":1.518},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:54:22 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.47799999999999998,"15min":0.48599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.54800000000000004,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.441,"15min":0.45700000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56399999999999995,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.53500000000000003}]},{"osd":4,"last update":"Mon Oct 9 03:54:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.59199999999999997,"15min":0.59999999999999998},"min":{"1min":0.375,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85399999999999998,"5min":1.611,"15min":1.611},"last":0.623},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63300000000000001,"15min":0.64400000000000002},"min":{"1min":0.43099999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.82399999999999995,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.71099999999999997}]},{"osd":5,"last update":"Mon Oct 9 03:54:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57599999999999996,"15min":0.57899999999999996},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.502,"5min":0.57299999999999995,"15min":0.58499999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76900000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 03:54:24 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.56999999999999995,"15min":0.58099999999999996},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72099999999999997,"5min":0.753,"15min":0.753},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57799999999999996,"15min":0.58099999999999996},"min":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.749}]},{"osd":7,"last update":"Mon Oct 9 03:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45200000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.60999999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986217,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297992,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447160,"statfs":{"total":95995035648,"available":95689891840,"internally_reserved":0,"allocated":720896,"data_stored":303793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.54000000000000004,"15min":0.54600000000000004},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.64100000000000001,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.621},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.55500000000000005},"min":{"1min":0.40000000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.59099999999999997,"5min":0.753,"15min":0.753},"last":0.67700000000000005}]},{"osd":1,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.54200000000000004,"15min":0.54900000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68200000000000005,"5min":1.139,"15min":1.139},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.53000000000000003,"15min":0.54200000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.60799999999999998,"5min":1.05,"15min":1.05},"last":0.69599999999999995}]},{"osd":2,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.54500000000000004},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.625,"5min":1.625,"15min":1.625},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.40100000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.64700000000000002,"5min":0.75,"15min":0.75},"last":0.58399999999999996}]},{"osd":3,"last update":"Mon Oct 9 03:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.38300000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66700000000000004,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.629},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35499999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72099999999999997,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.64600000000000002}]},{"osd":5,"last update":"Mon Oct 9 03:54:04 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.378,"15min":0.36899999999999999},"min":{"1min":0.29799999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.40999999999999998,"15min":0.39500000000000002},"min":{"1min":0.314,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Oct 9 03:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.47699999999999998,"15min":0.46700000000000003},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.73399999999999999,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.48399999999999999,"15min":0.47899999999999998},"min":{"1min":0.39400000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64600000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 03:54:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.50800000000000001,"15min":0.5},"min":{"1min":0.38400000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.59499999999999997,"15min":0.59799999999999998},"min":{"1min":0.39700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77100000000000002,"5min":1.079,"15min":1.079},"last":0.70599999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724581,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":704512,"data_stored":302677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.54200000000000004,"15min":0.53600000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.40600000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.82799999999999996,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.84899999999999998,"5min":1.002,"15min":1.002},"last":0.501},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.35599999999999998}]},{"osd":2,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.90200000000000002,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83799999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.43099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Oct 9 03:54:02 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.47299999999999998,"15min":0.46899999999999997},"min":{"1min":0.32700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.47999999999999998},"min":{"1min":0.34899999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 03:54:25 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.504,"15min":0.51100000000000001},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.68400000000000005,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.496,"5min":0.53100000000000003,"15min":0.53700000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.77000000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 03:53:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.59999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T03:54:59.741 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T03:54:59.741 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T03:54:59.741 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T03:54:59.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T03:55:00.089 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:00.482 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:00 smithi163 ceph-mon[118807]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:55:00.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:00 smithi163 ceph-mon[118807]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:00.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:00 smithi163 ceph-mon[101022]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:55:00.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:00 smithi163 ceph-mon[101022]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:00 smithi181 ceph-mon[105375]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T03:55:00.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:00 smithi181 ceph-mon[105375]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:01.794 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T03:55:01.794 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T03:55:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:02 smithi181 ceph-mon[105375]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:02 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3957398636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T03:55:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:02 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:02.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:02 smithi163 ceph-mon[101022]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:02.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:02 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3957398636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T03:55:02.529 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:02 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:02.529 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:02 smithi163 ceph-mon[118807]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:02.529 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:02 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3957398636' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T03:55:02.529 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:02 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:02.543 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T03:55:02.543 INFO:teuthology.run_tasks:Running task print... 2023-10-09T03:55:02.552 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T03:55:02.552 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T03:55:02.565 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T03:55:02.566 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T03:55:02.566 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T03:55:02.570 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T03:55:02.570 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T03:55:02.570 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T03:55:02.571 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T03:55:02.571 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T03:55:02.572 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T03:55:02.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T03:55:02.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:55:02.590 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T03:55:02.590 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T03:55:02.647 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T03:55:02.647 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T03:55:02.703 INFO:tasks.workunit:timeout=3h 2023-10-09T03:55:02.703 INFO:tasks.workunit:cleanup=True 2023-10-09T03:55:02.704 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T03:55:02.761 INFO:tasks.workunit.client.0.smithi163.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T03:55:02.928 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:04.236 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:04 smithi163 ceph-mon[118807]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:04.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:04 smithi163 ceph-mon[101022]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:04.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:04 smithi181 ceph-mon[105375]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:05.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:04] "GET /metrics HTTP/1.1" 200 238801 "" "Prometheus/2.43.0" 2023-10-09T03:55:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: [09/Oct/2023:03:55:04] ENGINE Bus STOPPING 2023-10-09T03:55:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: [09/Oct/2023:03:55:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:55:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: [09/Oct/2023:03:55:04] ENGINE Bus STOPPED 2023-10-09T03:55:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: [09/Oct/2023:03:55:04] ENGINE Bus STARTING 2023-10-09T03:55:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: [09/Oct/2023:03:55:04] ENGINE Serving on http://:::9283 2023-10-09T03:55:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:04 smithi163 conmon[101450]: [09/Oct/2023:03:55:04] ENGINE Bus STARTED 2023-10-09T03:55:05.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T03:55:05.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2977561596' entity='client.admin' 2023-10-09T03:55:05.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:05.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:55:05.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:55:05.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:05.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[101022]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:05.806 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:06.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:55:05 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:55:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:05 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2977561596' entity='client.admin' 2023-10-09T03:55:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:05 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:05 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:55:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:05 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:55:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:05 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:05 smithi181 ceph-mon[105375]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:06.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2977561596' entity='client.admin' 2023-10-09T03:55:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:55:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:55:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:05 smithi163 ceph-mon[118807]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:07.085 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T03:55:07.662 INFO:tasks.workunit.client.0.smithi163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T03:55:07.665 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T03:55:07.665 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T03:55:07.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:07 smithi163 ceph-mon[118807]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:07 smithi163 ceph-mon[101022]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:07 smithi181 ceph-mon[105375]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:09.562 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T03:55:09.619 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T03:55:09.655 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T03:55:09.657 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T03:55:09.657 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T03:55:09.694 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T03:55:09.697 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T03:55:09.698 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T03:55:09.714 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T03:55:09.715 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T03:55:09.716 DEBUG:teuthology.orchestra.run.smithi163:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T03:55:09.785 INFO:tasks.workunit.client.0.smithi163.stdout:Running main() from gmock_main.cc 2023-10-09T03:55:09.785 INFO:tasks.workunit.client.0.smithi163.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T03:55:09.785 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] Global test environment set-up. 2023-10-09T03:55:09.785 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T03:55:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:10 smithi163 ceph-mon[118807]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:10 smithi163 ceph-mon[101022]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:10.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:10 smithi181 ceph-mon[105375]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:11.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T03:55:11.197 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T03:55:11.211 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_all_features (14 ms) 2023-10-09T03:55:11.211 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T03:55:11.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:11 smithi163 ceph-mon[118807]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T03:55:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:11 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2963602193' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-191647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:55:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:11 smithi163 ceph-mon[101022]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T03:55:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:11 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2963602193' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-191647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:55:11.415 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.copyup (204 ms) 2023-10-09T03:55:11.415 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T03:55:11.432 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.sparse_copyup (17 ms) 2023-10-09T03:55:11.432 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T03:55:11.440 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_and_set_id (9 ms) 2023-10-09T03:55:11.440 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T03:55:11.452 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2023-10-09T03:55:11.453 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T03:55:11.473 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:11.480 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.directory_methods (27 ms) 2023-10-09T03:55:11.480 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.create 2023-10-09T03:55:11.493 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.create (14 ms) 2023-10-09T03:55:11.494 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T03:55:11.495 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-10-09T03:55:11.495 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T03:55:11.497 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2023-10-09T03:55:11.497 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T03:55:11.498 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T03:55:11.499 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T03:55:11.500 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T03:55:11.500 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T03:55:11.501 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T03:55:11.501 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T03:55:11.505 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-10-09T03:55:11.505 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T03:55:11.509 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-09T03:55:11.509 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T03:55:11.513 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T03:55:11.513 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T03:55:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:11 smithi181 ceph-mon[105375]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T03:55:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:11 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2963602193' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-191647-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T03:55:11.533 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-10-09T03:55:11.533 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T03:55:11.541 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-10-09T03:55:11.541 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T03:55:11.579 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.parents_v1 (39 ms) 2023-10-09T03:55:11.579 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T03:55:11.607 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2023-10-09T03:55:11.607 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T03:55:11.627 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2023-10-09T03:55:11.628 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T03:55:11.633 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-09T03:55:11.633 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T03:55:11.641 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-10-09T03:55:11.641 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T03:55:11.642 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-10-09T03:55:11.642 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T03:55:11.650 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_resize (7 ms) 2023-10-09T03:55:11.650 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T03:55:11.659 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_update (9 ms) 2023-10-09T03:55:11.659 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T03:55:11.659 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-10-09T03:55:11.660 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T03:55:11.666 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-10-09T03:55:11.666 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T03:55:11.670 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2023-10-09T03:55:11.670 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T03:55:11.676 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T03:55:11.676 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T03:55:11.700 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.metadata (24 ms) 2023-10-09T03:55:11.700 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T03:55:11.706 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2023-10-09T03:55:11.706 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T03:55:11.738 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror (33 ms) 2023-10-09T03:55:11.738 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T03:55:11.753 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-10-09T03:55:11.753 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T03:55:12.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[118807]: pgmap v70: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[118807]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2963602193' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-191647-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[118807]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[101022]: pgmap v70: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[101022]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2963602193' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-191647-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T03:55:12.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:12 smithi163 ceph-mon[101022]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T03:55:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:12 smithi181 ceph-mon[105375]: pgmap v70: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:12 smithi181 ceph-mon[105375]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T03:55:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:12 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2963602193' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-191647-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T03:55:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:12 smithi181 ceph-mon[105375]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T03:55:13.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:13 smithi163 ceph-mon[118807]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T03:55:13.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:13 smithi163 ceph-mon[101022]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T03:55:13.406 INFO:teuthology.orchestra.run.smithi163.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:13 smithi181 ceph-mon[105375]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T03:55:14.209 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image_status (2455 ms) 2023-10-09T03:55:14.209 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T03:55:14.287 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image_map (78 ms) 2023-10-09T03:55:14.287 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T03:55:14.294 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-10-09T03:55:14.294 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T03:55:14.305 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2023-10-09T03:55:14.305 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T03:55:14.307 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T03:55:14.307 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T03:55:14.309 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T03:55:14.309 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T03:55:14.312 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T03:55:14.312 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T03:55:14.318 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T03:55:14.318 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T03:55:14.321 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T03:55:14.321 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T03:55:14.326 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T03:55:14.326 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T03:55:14.327 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-10-09T03:55:14.328 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T03:55:14.330 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T03:55:14.330 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T03:55:14.333 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T03:55:14.333 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T03:55:14.337 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_clean (3 ms) 2023-10-09T03:55:14.337 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T03:55:14.339 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-10-09T03:55:14.339 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T03:55:14.342 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T03:55:14.342 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T03:55:14.344 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T03:55:14.345 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T03:55:14.346 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T03:55:14.346 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T03:55:14.348 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (1 ms) 2023-10-09T03:55:14.348 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T03:55:14.350 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T03:55:14.350 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T03:55:14.353 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T03:55:14.353 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T03:55:14.355 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T03:55:14.355 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T03:55:14.358 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-10-09T03:55:14.358 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T03:55:14.373 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (15 ms) 2023-10-09T03:55:14.373 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T03:55:14.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: pgmap v73: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1013 KiB/s wr, 887 op/s 2023-10-09T03:55:14.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:55:14.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: Cluster is now healthy 2023-10-09T03:55:14.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:14.466 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: pgmap v73: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1013 KiB/s wr, 887 op/s 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: Cluster is now healthy 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:55:14.467 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:14 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:14.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T03:55:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: pgmap v73: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1013 KiB/s wr, 887 op/s 2023-10-09T03:55:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: Cluster is now healthy 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:55:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:14 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:14.563 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (190 ms) 2023-10-09T03:55:14.563 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T03:55:14.566 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T03:55:14.566 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T03:55:14.568 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T03:55:14.568 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T03:55:14.576 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-10-09T03:55:14.576 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T03:55:14.582 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-10-09T03:55:14.582 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T03:55:14.602 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-10-09T03:55:14.602 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T03:55:14.611 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T03:55:14.612 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T03:55:14.623 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-10-09T03:55:14.623 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T03:55:14.631 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-10-09T03:55:14.631 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T03:55:14.641 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-10-09T03:55:14.641 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T03:55:14.654 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.migration_v1 (13 ms) 2023-10-09T03:55:14.654 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T03:55:14.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:14 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:14] "GET /metrics HTTP/1.1" 200 238801 "" "Prometheus/2.43.0" 2023-10-09T03:55:14.846 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:15.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:15 smithi163 ceph-mon[118807]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:15.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:15 smithi163 ceph-mon[101022]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:15 smithi181 ceph-mon[105375]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:55:16.019 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:55:15 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:55:16.217 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1563 ms) 2023-10-09T03:55:16.218 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T03:55:16.259 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.sparsify (41 ms) 2023-10-09T03:55:16.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:16 smithi163 ceph-mon[118807]: pgmap v74: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1013 KiB/s wr, 887 op/s 2023-10-09T03:55:16.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:16 smithi163 ceph-mon[118807]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T03:55:16.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:16 smithi163 ceph-mon[101022]: pgmap v74: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1013 KiB/s wr, 887 op/s 2023-10-09T03:55:16.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:16 smithi163 ceph-mon[101022]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T03:55:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:16 smithi181 ceph-mon[105375]: pgmap v74: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1013 KiB/s wr, 887 op/s 2023-10-09T03:55:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:16 smithi181 ceph-mon[105375]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T03:55:16.854 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (65s) 27s ago 106s 16.0M - 0.25.0 c8568f914cd2 38b3c41a75cc 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (88s) 45s ago 87s 88.4M - 9.4.7 2c41d148cca3 be20d4ebd53f 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.crqkdj smithi163 running (55s) 27s ago 54s 15.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 8037aa8a1d7d 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.sboiwx smithi181 running (52s) 45s ago 52s 13.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 2c2246f2ef4b 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443 running (5m) 45s ago 5m 441M - 17.2.6-1303-gb2903cb0 5517396c50d4 1bddf6f7cdf3 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:9283 running (7m) 27s ago 7m 495M - 17.2.6-1303-gb2903cb0 5517396c50d4 ec02a34c21a5 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (7m) 27s ago 7m 82.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 51bd91cc44f8 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (5m) 45s ago 5m 55.9M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 4a4d2d630200 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (5m) 27s ago 5m 59.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 72ef10caabbb 2023-10-09T03:55:16.855 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (79s) 27s ago 78s 11.4M - 1.5.0 0da6a335fe13 8d858ae3dede 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (74s) 45s ago 73s 10.0M - 1.5.0 0da6a335fe13 02d966c4a8a9 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (5m) 27s ago 5m 57.0M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 1d12e7087516 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (4m) 27s ago 4m 58.3M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 1b27d5ad7adb 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (4m) 27s ago 4m 50.3M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 e2ce5dc7d22b 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (4m) 27s ago 4m 52.1M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 0e8301f836bc 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (3m) 45s ago 3m 50.7M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 b4d4d04d2a3e 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (3m) 45s ago 3m 51.2M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 4ed298059fdf 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (3m) 45s ago 2m 49.7M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 4d54b928e978 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (2m) 45s ago 2m 50.4M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 6ca0d6954129 2023-10-09T03:55:16.856 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (61s) 45s ago 2m 30.3M - 2.43.0 a07b618ecd1d 7e99604366c1 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.215 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.216 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:17.233 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] 70 tests from TestClsRbd (5061 ms total) 2023-10-09T03:55:17.233 INFO:tasks.workunit.client.0.smithi163.stdout: 2023-10-09T03:55:17.233 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] Global test environment tear-down 2023-10-09T03:55:17.233 INFO:tasks.workunit.client.0.smithi163.stdout:[==========] 70 tests from 1 test suite ran. (7449 ms total) 2023-10-09T03:55:17.234 INFO:tasks.workunit.client.0.smithi163.stdout:[ PASSED ] 70 tests. 2023-10-09T03:55:17.236 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T03:55:17.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T03:55:17.276 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T03:55:17.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T03:55:17.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T03:55:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:17.620 DEBUG:teuthology.parallel:result is None 2023-10-09T03:55:17.620 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T03:55:17.647 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T03:55:17.647 DEBUG:teuthology.orchestra.run.smithi163:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T03:55:17.819 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T03:55:17.819 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T03:55:17.819 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T03:55:17.820 DEBUG:teuthology.parallel:result is None 2023-10-09T03:55:17.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T03:55:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[118807]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:55:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[118807]: pgmap v77: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 897 op/s 2023-10-09T03:55:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-10-09T03:55:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T03:55:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T03:55:18.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[118807]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T03:55:18.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[101022]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:55:18.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[101022]: pgmap v77: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 897 op/s 2023-10-09T03:55:18.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-10-09T03:55:18.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T03:55:18.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T03:55:18.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:18 smithi163 ceph-mon[101022]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T03:55:18.367 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:18 smithi181 ceph-mon[105375]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:55:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:18 smithi181 ceph-mon[105375]: pgmap v77: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 897 op/s 2023-10-09T03:55:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:18 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-10-09T03:55:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:18 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T03:55:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:18 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T03:55:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:18 smithi181 ceph-mon[105375]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T03:55:19.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:19 smithi181 ceph-mon[105375]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T03:55:19.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:19 smithi163 ceph-mon[118807]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T03:55:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:19 smithi163 ceph-mon[101022]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T03:55:20.324 INFO:teuthology.orchestra.run.smithi163.stdout:wait for minority of mons to upgrade 2023-10-09T03:55:20.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:20 smithi163 ceph-mon[118807]: pgmap v80: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:20.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:20 smithi163 ceph-mon[101022]: pgmap v80: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:20 smithi181 ceph-mon[105375]: pgmap v80: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:21.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T03:55:21.559 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:55:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:22 smithi181 ceph-mon[105375]: pgmap v81: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:22.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:22 smithi163 ceph-mon[118807]: pgmap v81: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:22 smithi163 ceph-mon[101022]: pgmap v81: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:24.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:24 smithi181 ceph-mon[105375]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:24.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:24 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/399599306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:24 smithi163 ceph-mon[118807]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:24 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/399599306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:24 smithi163 ceph-mon[101022]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:24 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/399599306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:25.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:24 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:24] "GET /metrics HTTP/1.1" 200 241881 "" "Prometheus/2.43.0" 2023-10-09T03:55:26.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:55:25 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:55:26.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:26 smithi181 ceph-mon[105375]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:26.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:26 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4203581267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:26.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:26 smithi163 ceph-mon[118807]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:26 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4203581267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:26 smithi163 ceph-mon[101022]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:26 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4203581267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:27.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:27 smithi163 ceph-mon[118807]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:27.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:27 smithi163 ceph-mon[101022]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:28.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:27 smithi181 ceph-mon[105375]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:28.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:28 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4036448374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:28 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4036448374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:29.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:28 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4036448374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:29.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:29 smithi163 ceph-mon[118807]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:29.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:29 smithi163 ceph-mon[101022]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:30.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:29 smithi181 ceph-mon[105375]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:31.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:30 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1886887858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:31.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:30 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1886887858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:30 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1886887858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:32.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:31 smithi181 ceph-mon[105375]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:32.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:31 smithi163 ceph-mon[118807]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:32.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:31 smithi163 ceph-mon[101022]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:33.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:32 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:33.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:32 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:33.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:32 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:34.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:34 smithi181 ceph-mon[105375]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:34.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:34 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/916583184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:34.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:34 smithi163 ceph-mon[118807]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:34 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/916583184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:34 smithi163 ceph-mon[101022]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:34 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/916583184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:35.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:34 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:34] "GET /metrics HTTP/1.1" 200 239751 "" "Prometheus/2.43.0" 2023-10-09T03:55:36.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:55:35 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:55:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:36 smithi181 ceph-mon[105375]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:36 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/689603552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:36 smithi163 ceph-mon[118807]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:36 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/689603552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:36.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:36 smithi163 ceph-mon[101022]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:36.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:36 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/689603552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:37.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:37 smithi163 ceph-mon[118807]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:37 smithi163 ceph-mon[101022]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:38.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:37 smithi181 ceph-mon[105375]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:38.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:38 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1411471042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:38 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1411471042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:39.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:38 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1411471042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:39.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:39 smithi163 ceph-mon[101022]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:39 smithi163 ceph-mon[118807]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:40.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:39 smithi181 ceph-mon[105375]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:40 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1815471795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:41.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:40 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1815471795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:41.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:40 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1815471795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:42.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:41 smithi181 ceph-mon[105375]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:42.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:41 smithi163 ceph-mon[101022]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:42.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:41 smithi163 ceph-mon[118807]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:43 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1612800800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:43.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:43 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1612800800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:43 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1612800800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:44 smithi181 ceph-mon[105375]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:44 smithi163 ceph-mon[101022]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:44.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:44 smithi163 ceph-mon[118807]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:45.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:44 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:44] "GET /metrics HTTP/1.1" 200 239751 "" "Prometheus/2.43.0" 2023-10-09T03:55:46.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:55:45 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:55:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:46 smithi181 ceph-mon[105375]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:46 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3744496759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:46 smithi163 ceph-mon[118807]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:46 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3744496759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:46 smithi163 ceph-mon[101022]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:46 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3744496759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:47.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:47 smithi163 ceph-mon[118807]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:47 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:47.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:47 smithi163 ceph-mon[101022]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:47.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:47 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:48.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:47 smithi181 ceph-mon[105375]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:48.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:47 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:55:48.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:48 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1935040911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:48.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:48 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1935040911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:49.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:48 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1935040911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:50.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[101022]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:50.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T03:55:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T03:55:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T03:55:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:50 smithi163 ceph-mon[118807]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T03:55:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T03:55:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:55:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:55:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T03:55:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:50 smithi181 ceph-mon[105375]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:51 smithi163 ceph-mon[118807]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi181 2023-10-09T03:55:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:51 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2972339096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:51 smithi163 ceph-mon[101022]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi181 2023-10-09T03:55:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:51 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2972339096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:51 smithi181 ceph-mon[105375]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi181 2023-10-09T03:55:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:51 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2972339096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:52 smithi181 ceph-mon[105375]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:52.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:52 smithi163 ceph-mon[118807]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:52 smithi163 ceph-mon[101022]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:53 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4151750509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:53 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4151750509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:53 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4151750509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:54 smithi181 ceph-mon[105375]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:54 smithi163 ceph-mon[118807]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:54 smithi163 ceph-mon[101022]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:55.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:55:54 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:54] "GET /metrics HTTP/1.1" 200 239744 "" "Prometheus/2.43.0" 2023-10-09T03:55:55.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:55 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2713859776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:55 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2713859776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:55 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2713859776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:56.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:55:55 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:55:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:55:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:56 smithi181 ceph-mon[105375]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:56 smithi163 ceph-mon[118807]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:56 smithi163 ceph-mon[101022]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:57 smithi163 ceph-mon[118807]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:55:57 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4027550962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:57 smithi163 ceph-mon[101022]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:55:57 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4027550962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:55:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:57 smithi181 ceph-mon[105375]: pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:55:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:55:57 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4027550962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:00 smithi181 ceph-mon[105375]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:00 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1163963826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:00.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:00 smithi163 ceph-mon[101022]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:00.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:00 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1163963826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:00 smithi163 ceph-mon[118807]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:00 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1163963826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:02.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:02 smithi181 ceph-mon[105375]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:02 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/549767570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:02 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:02.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:02 smithi163 ceph-mon[118807]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:02 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/549767570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:02 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:02 smithi163 ceph-mon[101022]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:02 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/549767570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:02 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:04.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:04 smithi181 ceph-mon[105375]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:04 smithi163 ceph-mon[118807]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:04 smithi163 ceph-mon[101022]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:05.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:04 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:04] "GET /metrics HTTP/1.1" 200 239715 "" "Prometheus/2.43.0" 2023-10-09T03:56:05.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:05 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1302564623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:05.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:05 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1302564623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:05 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1302564623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:06.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:05 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:56:06.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:06 smithi181 ceph-mon[105375]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:06.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:06 smithi163 ceph-mon[101022]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:06.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:06 smithi163 ceph-mon[118807]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:07.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:07 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1636762419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:07 smithi163 ceph-mon[118807]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:07 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1636762419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:07 smithi163 ceph-mon[101022]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:07 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1636762419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:07 smithi181 ceph-mon[105375]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:10 smithi181 ceph-mon[105375]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:10 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/201864829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:10.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:10 smithi163 ceph-mon[101022]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:10 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/201864829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:10 smithi163 ceph-mon[118807]: pgmap v105: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:10 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/201864829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:12 smithi181 ceph-mon[105375]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:12 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2247709736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:12.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:12 smithi163 ceph-mon[101022]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:12.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:12 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2247709736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:12 smithi163 ceph-mon[118807]: pgmap v106: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:12 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2247709736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:14 smithi181 ceph-mon[105375]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:14 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1002473120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:14 smithi163 ceph-mon[101022]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:14 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1002473120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:14 smithi163 ceph-mon[118807]: pgmap v107: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:14 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1002473120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:15.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:14 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:14] "GET /metrics HTTP/1.1" 200 239715 "" "Prometheus/2.43.0" 2023-10-09T03:56:16.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:15 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:56:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:16 smithi181 ceph-mon[105375]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:16 smithi163 ceph-mon[118807]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:16 smithi163 ceph-mon[101022]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:17.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:17 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3289890405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:17 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:17 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3289890405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:17 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:17 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3289890405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:17 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:18 smithi181 ceph-mon[105375]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:18.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:18 smithi163 ceph-mon[118807]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:18 smithi163 ceph-mon[101022]: pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:19.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:19 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4270281534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:19 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/4270281534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:19 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/4270281534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:20 smithi181 ceph-mon[105375]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:20 smithi163 ceph-mon[118807]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:20 smithi163 ceph-mon[101022]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:22 smithi181 ceph-mon[105375]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:22 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2172988178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:22 smithi163 ceph-mon[118807]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:22 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2172988178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:22 smithi163 ceph-mon[101022]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:22 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2172988178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:24.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:24 smithi181 ceph-mon[105375]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:24.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:24 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2010250487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:24 smithi163 ceph-mon[118807]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:24 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2010250487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:24 smithi163 ceph-mon[101022]: pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:24 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2010250487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:25.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:24 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:24] "GET /metrics HTTP/1.1" 200 239721 "" "Prometheus/2.43.0" 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: Upgrade: Updating mgr.x 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:25.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: Deploying daemon mgr.x on smithi181 2023-10-09T03:56:25.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:25 smithi181 ceph-mon[105375]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:25.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: Upgrade: Updating mgr.x 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: Deploying daemon mgr.x on smithi181 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[118807]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: Upgrade: Updating mgr.x 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: Deploying daemon mgr.x on smithi181 2023-10-09T03:56:25.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:25 smithi163 ceph-mon[101022]: pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:26.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:25 smithi181 conmon[108147]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:56:26.381 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:26 smithi181 systemd[1]: Stopping Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:26 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/756562804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:26.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:26 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/756562804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:26 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/756562804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:27.100 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:26 smithi181 bash[160449]: 1bddf6f7cdf3b54006a0ea88cfa23d88535b289a07652957d62c9f818b2a3434 2023-10-09T03:56:27.100 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:26 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T03:56:27.100 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:26 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T03:56:27.100 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:26 smithi181 systemd[1]: Stopped Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:27.100 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 systemd[1]: Starting Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:27.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:27 smithi163 ceph-mon[118807]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:27.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:27 smithi163 ceph-mon[101022]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:27.886 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:27 smithi181 ceph-mon[105375]: pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:27.887 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 bash[160800]: e813fb0d7da872895fc254561e94542ea68d323b2c66f2e6b61841e2b8f034ed 2023-10-09T03:56:27.887 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 systemd[1]: Started Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:27.887 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 ceph-mgr[160865]: -- 172.21.15.181:0/590582648 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1d87ad1e0 con 0x55f1d8784c00 2023-10-09T03:56:27.887 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 conmon[160841]: 2023-10-09T03:56:27.803+0000 7f1a62787200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:56:27.887 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 conmon[160841]: 2023-10-09T03:56:27.886+0000 7f1a62787200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:56:28.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:27 smithi181 conmon[160841]: 2023-10-09T03:56:27.981+0000 7f1a62787200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:56:28.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:28 smithi181 conmon[160841]: 2023-10-09T03:56:28.098+0000 7f1a62787200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:56:28.690 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:28 smithi181 conmon[160841]: 2023-10-09T03:56:28.298+0000 7f1a62787200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:56:29.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:28 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:29.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:28 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:29.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:28 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:29.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:28 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1498172837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:29.022 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:28 smithi181 conmon[160841]: 2023-10-09T03:56:28.689+0000 7f1a62787200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:56:29.022 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:28 smithi181 conmon[160841]: 2023-10-09T03:56:28.837+0000 7f1a62787200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:56:29.022 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:28 smithi181 conmon[160841]: 2023-10-09T03:56:28.933+0000 7f1a62787200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:56:29.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1498172837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:29.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:28 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1498172837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:29.518 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:29 smithi181 conmon[160841]: 2023-10-09T03:56:29.034+0000 7f1a62787200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:56:30.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:29 smithi181 ceph-mon[105375]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:30.019 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:29 smithi181 conmon[160841]: 2023-10-09T03:56:29.533+0000 7f1a62787200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:56:30.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:29 smithi163 ceph-mon[118807]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:29 smithi163 ceph-mon[101022]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:30.355 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:30 smithi181 conmon[160841]: 2023-10-09T03:56:30.268+0000 7f1a62787200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:56:30.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:30 smithi181 conmon[160841]: 2023-10-09T03:56:30.354+0000 7f1a62787200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:56:31.328 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:31 smithi181 conmon[160841]: 2023-10-09T03:56:31.228+0000 7f1a62787200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:56:31.329 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:31 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.329 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:31 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.329 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:31 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3322599017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:31.329 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:31 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.329 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:31 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3322599017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3322599017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:31 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:31.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:31 smithi181 conmon[160841]: 2023-10-09T03:56:31.327+0000 7f1a62787200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:56:31.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:31 smithi181 conmon[160841]: 2023-10-09T03:56:31.510+0000 7f1a62787200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:56:32.072 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:31 smithi181 conmon[160841]: 2023-10-09T03:56:31.818+0000 7f1a62787200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:56:32.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:32 smithi181 ceph-mon[105375]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:32.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:32.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:32 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:32.331 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:32 smithi181 conmon[160841]: 2023-10-09T03:56:32.071+0000 7f1a62787200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:56:32.331 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:32 smithi181 conmon[160841]: 2023-10-09T03:56:32.237+0000 7f1a62787200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:56:32.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:32 smithi163 ceph-mon[101022]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:32.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:32.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:32 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:32 smithi163 ceph-mon[118807]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:32 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:32.653 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:32 smithi181 conmon[160841]: 2023-10-09T03:56:32.522+0000 7f1a62787200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:56:33.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:33 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:33 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:33.418 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:33 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:33.419 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:33 smithi181 conmon[160841]: 2023-10-09T03:56:33.264+0000 7f1a62787200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:56:33.750 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:33 smithi181 conmon[160841]: 2023-10-09T03:56:33.417+0000 7f1a62787200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:56:33.750 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:33 smithi181 conmon[160841]: 2023-10-09T03:56:33.627+0000 7f1a62787200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:56:33.750 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:33 smithi181 conmon[160841]: 2023-10-09T03:56:33.748+0000 7f1a62787200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:56:34.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:33 smithi181 conmon[160841]: 2023-10-09T03:56:33.979+0000 7f1a62787200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:56:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:34 smithi163 ceph-mon[101022]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:34 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/842916924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:34 smithi163 ceph-mon[118807]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:34 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/842916924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:34.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:34 smithi181 ceph-mon[105375]: pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:34.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:34 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/842916924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:35.067 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:34 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:34] "GET /metrics HTTP/1.1" 200 239723 "" "Prometheus/2.43.0" 2023-10-09T03:56:35.228 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:34 smithi181 conmon[160841]: [09/Oct/2023:03:56:34] ENGINE Bus STARTING 2023-10-09T03:56:35.228 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:34 smithi181 conmon[160841]: 2023-10-09T03:56:34.906+0000 7f1a62787200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:56:35.228 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: CherryPy Checker: 2023-10-09T03:56:35.228 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: The Application mounted at '' has an empty config. 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Serving on http://:::9283 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Bus STARTED 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:35 smithi181 ceph-mon[105375]: Standby manager daemon x restarted 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:35 smithi181 ceph-mon[105375]: Standby manager daemon x started 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:35 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:35 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:35 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:56:35.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:35 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:56:35.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[118807]: Standby manager daemon x restarted 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[118807]: Standby manager daemon x started 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[101022]: Standby manager daemon x restarted 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[101022]: Standby manager daemon x started 2023-10-09T03:56:35.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:56:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:56:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:56:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:56:35.509 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Bus STOPPING 2023-10-09T03:56:35.510 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:56:35.510 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Bus STOPPED 2023-10-09T03:56:35.510 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Bus STARTING 2023-10-09T03:56:35.755 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: 2023-10-09T03:56:35.421+0000 7fabc5dc4700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T03:56:35.755 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: ignoring --setuser ceph since I am not root 2023-10-09T03:56:35.755 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: ignoring --setgroup ceph since I am not root 2023-10-09T03:56:35.755 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 ceph-mgr[101474]: -- 172.21.15.163:0/2677803996 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5603187fc9c0 con 0x5603187d4c00 2023-10-09T03:56:35.755 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: 2023-10-09T03:56:35.587+0000 7f7a2248c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:56:35.756 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: 2023-10-09T03:56:35.669+0000 7f7a2248c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:56:35.756 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: 2023-10-09T03:56:35.755+0000 7f7a2248c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:56:35.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.569+0000 7f19bfbbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.569+0000 7f19bfbbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.569+0000 7f19bfbbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.569+0000 7f19bfbbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.569+0000 7f19bfbbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.570+0000 7f19bcbb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.570+0000 7f19bcbb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.570+0000 7f19bcbb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.570+0000 7f19bcbb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: 2023-10-09T03:56:35.570+0000 7f19bcbb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Serving on http://:::9283 2023-10-09T03:56:35.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: [09/Oct/2023:03:56:35] ENGINE Bus STARTED 2023-10-09T03:56:36.028 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:35 smithi163 conmon[101450]: 2023-10-09T03:56:35.849+0000 7f7a2248c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:56:36.082 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:35 smithi181 conmon[160841]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:35] "GET /metrics HTTP/1.1" 200 31021 "" "Prometheus/2.43.0" 2023-10-09T03:56:36.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:36 smithi163 conmon[101450]: 2023-10-09T03:56:36.028+0000 7f7a2248c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:56:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: mgrmap e20: y(active, since 3m), standbys: x 2023-10-09T03:56:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: mgrmap e21: x(active, starting, since 0.00721737s) 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:56:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:56:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: Manager daemon x is now available 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T03:56:36.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/682098508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: mgrmap e20: y(active, since 3m), standbys: x 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:56:36.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: mgrmap e21: x(active, starting, since 0.00721737s) 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:56:36.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:56:36.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: Manager daemon x is now available 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/682098508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:36 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: mgrmap e20: y(active, since 3m), standbys: x 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 172.21.15.163:0/3689330449' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T03:56:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.14457 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: mgrmap e21: x(active, starting, since 0.00721737s) 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:56:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: Manager daemon x is now available 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:36.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:56:36.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:56:36.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T03:56:36.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T03:56:36.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/682098508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:36.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:36 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:36.646 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:36 smithi163 conmon[101450]: 2023-10-09T03:56:36.336+0000 7f7a2248c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:56:36.646 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:36 smithi163 conmon[101450]: 2023-10-09T03:56:36.424+0000 7f7a2248c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:56:36.646 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:36 smithi163 conmon[101450]: 2023-10-09T03:56:36.553+0000 7f7a2248c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:56:37.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:36 smithi163 conmon[101450]: 2023-10-09T03:56:36.646+0000 7f7a2248c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:56:37.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:37 smithi163 conmon[101450]: 2023-10-09T03:56:37.062+0000 7f7a2248c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:56:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: Deploying cephadm binary to smithi181 2023-10-09T03:56:37.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: Deploying cephadm binary to smithi163 2023-10-09T03:56:37.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: [09/Oct/2023:03:56:36] ENGINE Bus STARTING 2023-10-09T03:56:37.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: mgrmap e22: x(active, since 1.01159s) 2023-10-09T03:56:37.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:37.985 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: [09/Oct/2023:03:56:36] ENGINE Serving on http://172.21.15.181:8765 2023-10-09T03:56:37.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: [09/Oct/2023:03:56:36] ENGINE Serving on https://172.21.15.181:7150 2023-10-09T03:56:37.986 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:37 smithi181 ceph-mon[105375]: [09/Oct/2023:03:56:36] ENGINE Bus STARTED 2023-10-09T03:56:37.987 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:37 smithi163 conmon[101450]: 2023-10-09T03:56:37.651+0000 7f7a2248c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:56:37.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: Deploying cephadm binary to smithi181 2023-10-09T03:56:37.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: Deploying cephadm binary to smithi163 2023-10-09T03:56:37.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: [09/Oct/2023:03:56:36] ENGINE Bus STARTING 2023-10-09T03:56:37.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: mgrmap e22: x(active, since 1.01159s) 2023-10-09T03:56:37.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: [09/Oct/2023:03:56:36] ENGINE Serving on http://172.21.15.181:8765 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: [09/Oct/2023:03:56:36] ENGINE Serving on https://172.21.15.181:7150 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[118807]: [09/Oct/2023:03:56:36] ENGINE Bus STARTED 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: Deploying cephadm binary to smithi181 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: Deploying cephadm binary to smithi163 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: [09/Oct/2023:03:56:36] ENGINE Bus STARTING 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: mgrmap e22: x(active, since 1.01159s) 2023-10-09T03:56:37.988 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: pgmap v3: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:37.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: [09/Oct/2023:03:56:36] ENGINE Serving on http://172.21.15.181:8765 2023-10-09T03:56:37.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: [09/Oct/2023:03:56:36] ENGINE Serving on https://172.21.15.181:7150 2023-10-09T03:56:37.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:37 smithi163 ceph-mon[101022]: [09/Oct/2023:03:56:36] ENGINE Bus STARTED 2023-10-09T03:56:38.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:37 smithi163 conmon[101450]: 2023-10-09T03:56:37.852+0000 7f7a2248c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:56:38.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:38 smithi163 conmon[101450]: 2023-10-09T03:56:38.724+0000 7f7a2248c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:56:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:38 smithi163 ceph-mon[118807]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:38 smithi163 ceph-mon[118807]: mgrmap e23: x(active, since 2s) 2023-10-09T03:56:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:38 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2852730355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:38.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:38 smithi163 ceph-mon[101022]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:38.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:38 smithi163 ceph-mon[101022]: mgrmap e23: x(active, since 2s) 2023-10-09T03:56:38.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:38 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2852730355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:39.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:38 smithi181 ceph-mon[105375]: pgmap v4: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:38 smithi181 ceph-mon[105375]: mgrmap e23: x(active, since 2s) 2023-10-09T03:56:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:38 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2852730355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:39.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:38 smithi181 conmon[155643]: ts=2023-10-09T03:56:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:56:39.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:38 smithi181 conmon[155643]: ts=2023-10-09T03:56:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephPGImbalance index=15 msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on (job) group_left () avg by (job) (ceph_osd_numpg\n > 0)) / on (job) group_left () avg by (job) (ceph_osd_numpg > 0)) * on (ceph_daemon)\n group_left (hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"172.21.15.163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:56:39.328 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:38 smithi163 conmon[101450]: 2023-10-09T03:56:38.835+0000 7f7a2248c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:56:39.328 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:39 smithi163 conmon[101450]: 2023-10-09T03:56:39.030+0000 7f7a2248c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:56:39.328 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:39 smithi163 conmon[101450]: 2023-10-09T03:56:39.328+0000 7f7a2248c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:56:39.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:39 smithi163 conmon[101450]: 2023-10-09T03:56:39.424+0000 7f7a2248c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:56:40.017 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:39 smithi163 conmon[101450]: 2023-10-09T03:56:39.724+0000 7f7a2248c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:56:40.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:40 smithi163 conmon[101450]: 2023-10-09T03:56:40.017+0000 7f7a2248c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:40 smithi163 conmon[101450]: 2023-10-09T03:56:40.771+0000 7f7a2248c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1515429204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1515429204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:40.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:40 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: pgmap v5: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1515429204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:41.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:40 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.208 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:40 smithi163 conmon[101450]: 2023-10-09T03:56:40.901+0000 7f7a2248c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:56:41.209 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:41 smithi163 conmon[101450]: 2023-10-09T03:56:41.093+0000 7f7a2248c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:56:41.550 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:41 smithi163 conmon[101450]: 2023-10-09T03:56:41.208+0000 7f7a2248c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:56:41.551 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:41 smithi163 conmon[101450]: 2023-10-09T03:56:41.424+0000 7f7a2248c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:56:41.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:41 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi163 to 2500M 2023-10-09T03:56:41.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:41 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:41 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:41 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi163 to 2500M 2023-10-09T03:56:41.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:41 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:41 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:41 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi163 to 2500M 2023-10-09T03:56:41.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:41 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:41.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:41 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:42.551 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: 2023-10-09T03:56:42.281+0000 7f7a2248c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:56:42.552 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: [09/Oct/2023:03:56:42] ENGINE Bus STARTING 2023-10-09T03:56:42.552 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: CherryPy Checker: 2023-10-09T03:56:42.552 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: The Application mounted at '' has an empty config. 2023-10-09T03:56:42.552 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: 2023-10-09T03:56:42.552 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: [09/Oct/2023:03:56:42] ENGINE Serving on http://:::9283 2023-10-09T03:56:42.552 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:42 smithi163 conmon[101450]: [09/Oct/2023:03:56:42] ENGINE Bus STARTED 2023-10-09T03:56:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[101022]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[101022]: Standby manager daemon y started 2023-10-09T03:56:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:56:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:56:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T03:56:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:56:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[118807]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[118807]: Standby manager daemon y started 2023-10-09T03:56:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:56:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:56:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T03:56:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:42 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:56:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:42 smithi181 ceph-mon[105375]: pgmap v6: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:42 smithi181 ceph-mon[105375]: Standby manager daemon y started 2023-10-09T03:56:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:42 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:56:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:42 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:56:43.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:42 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T03:56:43.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:42 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.163:0/3471781507' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:56:43.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:43 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2775315498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:43.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:43 smithi163 ceph-mon[118807]: mgrmap e24: x(active, since 7s), standbys: y 2023-10-09T03:56:43.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:43 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:56:43.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:43 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2775315498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:43.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:43 smithi163 ceph-mon[101022]: mgrmap e24: x(active, since 7s), standbys: y 2023-10-09T03:56:43.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:43 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:56:44.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:43 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2775315498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:43 smithi181 ceph-mon[105375]: mgrmap e24: x(active, since 7s), standbys: y 2023-10-09T03:56:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:43 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:56:44.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:56:44 smithi163 conmon[101450]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:56:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:56:44.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:44 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:56:45.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: pgmap v7: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:45.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:56:45.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:44 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: Adjusting osd_memory_target on smithi181 to 2500M 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/431931982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: Adjusting osd_memory_target on smithi181 to 2500M 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:56:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/431931982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:45.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:45.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:45 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:56:45 smithi181 conmon[160841]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:56:45] "GET /metrics HTTP/1.1" 200 31021 "" "Prometheus/2.43.0" 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: Adjusting osd_memory_target on smithi181 to 2500M 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/431931982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:46.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:45 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[118807]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:56:46.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:56:46.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:56:46.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:56:46.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T03:56:46.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T03:56:46.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:46 smithi163 ceph-mon[101022]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:47.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:56:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:56:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:56:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:56:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T03:56:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: Reconfiguring daemon alertmanager.a on smithi163 2023-10-09T03:56:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:46 smithi181 ceph-mon[105375]: pgmap v8: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:48.231 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:48 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:48.527 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:48 smithi163 conmon[177179]: ts=2023-10-09T03:56:48.231Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T03:56:48.527 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:48 smithi163 bash[198397]: 38b3c41a75cc16bd7c9a651726d43dedcbf4c09a24c6b3bfff327474e2add4ea 2023-10-09T03:56:48.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:48 smithi163 ceph-mon[118807]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:48.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:48 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1784748482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:48.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:48 smithi163 ceph-mon[101022]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:48.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:48 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1784748482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:48 smithi181 ceph-mon[105375]: pgmap v9: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:48 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1784748482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:49.331 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:48 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T03:56:49.332 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:48 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:49.332 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:48 smithi163 systemd[1]: Starting Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:50.001 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.614Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T03:56:50.001 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.614Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T03:56:50.001 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.616Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.163 port=9094 2023-10-09T03:56:50.001 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.617Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T03:56:50.001 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 bash[198734]: b04e9e96f8bb09f7e5ecd897d9b220334af258e994ff6eb448298e21e8041138 2023-10-09T03:56:50.002 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 systemd[1]: Started Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:50.002 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.640Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T03:56:50.002 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.641Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T03:56:50.002 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.643Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T03:56:50.002 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:49 smithi163 conmon[198775]: ts=2023-10-09T03:56:49.643Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T03:56:50.331 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:50 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:50.926 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:50 smithi163 bash[199033]: 8d858ae3dede3398a4617742d65ce069162aff4ac8f322906f126c2a52df8407 2023-10-09T03:56:50.926 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:50 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T03:56:50.927 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[101022]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:50.927 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:50.927 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:50.927 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3677239398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:50.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:50.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[118807]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:50.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:50.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:50.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3677239398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:50.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:50 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:51.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:50 smithi181 ceph-mon[105375]: pgmap v10: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:50 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:50 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:50 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3677239398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:50 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:56:51.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:50 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T03:56:51.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:50 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:51.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:50 smithi163 systemd[1]: Starting Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:51.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[101022]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T03:56:51.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[101022]: Reconfiguring daemon node-exporter.a on smithi163 2023-10-09T03:56:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[118807]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T03:56:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[118807]: Reconfiguring daemon node-exporter.a on smithi163 2023-10-09T03:56:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:51 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:51.834 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[198775]: ts=2023-10-09T03:56:51.617Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000381415s 2023-10-09T03:56:51.834 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.488Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T03:56:51.834 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.488Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T03:56:51.834 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.488Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.488Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T03:56:51.835 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T03:56:51.836 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T03:56:51.837 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T03:56:51.838 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.489Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.490Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 conmon[199397]: ts=2023-10-09T03:56:51.490Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T03:56:51.839 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 bash[199356]: 381d36940ced53d4c603b982f93ce59bccc0ee879e1367b2347a7f345de936e7 2023-10-09T03:56:51.840 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 03:56:51 smithi163 systemd[1]: Started Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:52.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:51 smithi181 ceph-mon[105375]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T03:56:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:51 smithi181 ceph-mon[105375]: Reconfiguring daemon node-exporter.a on smithi163 2023-10-09T03:56:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:51 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:51 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:53.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:52 smithi181 ceph-mon[105375]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:53.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:52 smithi181 ceph-mon[105375]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:56:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:52 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1341282207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:52 smithi163 ceph-mon[118807]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:52 smithi163 ceph-mon[118807]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:56:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:52 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1341282207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:52 smithi163 ceph-mon[101022]: pgmap v11: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:52 smithi163 ceph-mon[101022]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:56:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:52 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1341282207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:53.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:53 smithi181 ceph-mon[105375]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:56:54.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:53 smithi163 ceph-mon[118807]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:56:54.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:53 smithi163 ceph-mon[101022]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:56:54.442 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 systemd[1]: Stopping Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:54 smithi181 ceph-mon[105375]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.441Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T03:56:54.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.442Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T03:56:54.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.442Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T03:56:54.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.442Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T03:56:54.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 conmon[155643]: ts=2023-10-09T03:56:54.442Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T03:56:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:54 smithi163 ceph-mon[118807]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:55.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:54 smithi163 ceph-mon[101022]: pgmap v12: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:55.269 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:54 smithi181 bash[174125]: 7e99604366c169f37ce1cd562ff3e65ea92b5eafcb1bcf9c5eed49910b981459 2023-10-09T03:56:55.269 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T03:56:55.269 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 systemd[1]: Stopped Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:55.269 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 systemd[1]: Starting Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:56.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:55 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/257851835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 bash[174455]: cafec70e59d5555ed249abf8a7c355551f0be7d19c196dfc496b5653d868194a 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.638Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.639Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.639Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi181 (none))" 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.639Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.639Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 systemd[1]: Started Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.641Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T03:56:56.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.641Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.643Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.643Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.645Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.645Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.437µs 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.645Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.645Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.662Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.663Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.663Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.017µs wal_replay_duration=17.503648ms wbl_replay_duration=130ns total_replay_duration=17.559532ms 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.666Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.666Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T03:56:56.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.666Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T03:56:56.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.684Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.443648ms db_storage=939ns remote_storage=1.249µs web_handler=384ns query_engine=814ns scrape=497.092µs scrape_sd=92.051µs notify=34.011µs notify_sd=30.258µs rules=16.560161ms tracing=6.577µs 2023-10-09T03:56:56.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.684Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T03:56:56.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:56:55 smithi181 conmon[174496]: ts=2023-10-09T03:56:55.684Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T03:56:56.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:55 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/257851835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:56.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:55 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/257851835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:56 smithi181 ceph-mon[105375]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:56 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:56 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:56 smithi181 ceph-mon[105375]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T03:56:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:56 smithi181 ceph-mon[105375]: Reconfiguring daemon grafana.a on smithi181 2023-10-09T03:56:57.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[101022]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[101022]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[101022]: Reconfiguring daemon grafana.a on smithi181 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[118807]: pgmap v13: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:56:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[118807]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T03:56:57.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:56 smithi163 ceph-mon[118807]: Reconfiguring daemon grafana.a on smithi181 2023-10-09T03:56:58.018 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:57 smithi181 systemd[1]: Stopping Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:57 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3740783408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:56:57 smithi181 ceph-mon[105375]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:58.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:57 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3740783408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:58.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:56:57 smithi163 ceph-mon[118807]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:57 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3740783408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:56:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:56:57 smithi163 ceph-mon[101022]: pgmap v14: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:56:58.768 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:58 smithi181 bash[174973]: be20d4ebd53fee5217392a14b66b184c802b899f6eff9e171709f913fa54552b 2023-10-09T03:56:58.768 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:58 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T03:56:58.768 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:58 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T03:56:58.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:58 smithi181 systemd[1]: Stopped Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:58.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:58 smithi181 systemd[1]: Starting Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:56:59.518 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 bash[175309]: dd057a75b8e1e522a020cbc70ec6a2f4fedc8fdc69b3c2793ef7faa3eb90b162 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 systemd[1]: Started Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344608357Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344793507Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344811127Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344818642Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344824814Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344830887Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344837236Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344843463Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T03:56:59.519 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344849406Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344855419Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344860874Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.344866644Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.34487235Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=settings t=2023-10-09T03:56:59.34487889Z level=info msg="App mode production" 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=sqlstore t=2023-10-09T03:56:59.344930382Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=sqlstore t=2023-10-09T03:56:59.344947433Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=migrator t=2023-10-09T03:56:59.35960684Z level=info msg="Starting DB migrations" 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=migrator t=2023-10-09T03:56:59.364435167Z level=info msg="migrations completed" performed=0 skipped=481 duration=481.833µs 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=sqlstore t=2023-10-09T03:56:59.364846582Z level=info msg="Created default organization" 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=plugin.loader t=2023-10-09T03:56:59.432192936Z level=info msg="Plugin registered" pluginID=input 2023-10-09T03:56:59.520 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=plugin.loader t=2023-10-09T03:56:59.460171334Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T03:56:59.521 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=plugin.loader t=2023-10-09T03:56:59.460219739Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T03:56:59.521 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=secrets t=2023-10-09T03:56:59.460257096Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T03:56:59.521 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=query_data t=2023-10-09T03:56:59.463785617Z level=info msg="Query Service initialization" 2023-10-09T03:56:59.521 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:56:59 smithi181 conmon[175350]: logger=live.push_http t=2023-10-09T03:56:59.466591563Z level=info msg="Live Push Gateway initialization" 2023-10-09T03:56:59.942 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:56:59 smithi181 systemd[1]: Stopping Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:00.081 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 03:56:59 smithi163 conmon[198775]: ts=2023-10-09T03:56:59.619Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00217323s 2023-10-09T03:57:00.496 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:00 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T03:57:00.497 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:00 smithi181 bash[175581]: 02d966c4a8a947c08e2ef82588267be70b106eabfe9a249ba90a99fbbe17201f 2023-10-09T03:57:00.497 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=infra.usagestats.collector t=2023-10-09T03:57:00.222711474Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T03:57:00.497 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=provisioning.datasources t=2023-10-09T03:57:00.370304828Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-10-09T03:57:00.497 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=provisioning.datasources t=2023-10-09T03:57:00.370354728Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-10-09T03:57:00.497 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=provisioning.datasources t=2023-10-09T03:57:00.380645778Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T03:57:00.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:00 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:00.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:00 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:00.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:00 smithi181 ceph-mon[105375]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T03:57:00.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:00 smithi181 ceph-mon[105375]: Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T03:57:00.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:00 smithi181 ceph-mon[105375]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:00.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:00 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3812905567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[118807]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T03:57:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[118807]: Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T03:57:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[118807]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3812905567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:00.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:00.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:00.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[101022]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T03:57:00.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[101022]: Reconfiguring daemon node-exporter.b on smithi181 2023-10-09T03:57:00.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[101022]: pgmap v15: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:00.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:00 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3812905567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:00.769 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:00 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T03:57:00.769 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:00 smithi181 systemd[1]: Stopped Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:00.769 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:00 smithi181 systemd[1]: Starting Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:00.769 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=provisioning.alerting t=2023-10-09T03:57:00.529140188Z level=info msg="starting to provision alerting" 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=provisioning.alerting t=2023-10-09T03:57:00.529366527Z level=info msg="finished to provision alerting" 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=grafanaStorageLogger t=2023-10-09T03:57:00.529507538Z level=info msg="storage starting" 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=ngalert.state.manager t=2023-10-09T03:57:00.529493311Z level=info msg="Warming state cache for startup" 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=http.server t=2023-10-09T03:57:00.531435777Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=ngalert.state.manager t=2023-10-09T03:57:00.578648404Z level=info msg="State cache has been initialized" states=0 duration=49.153811ms 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=ticker t=2023-10-09T03:57:00.578728618Z level=info msg=starting first_tick=2023-10-09T03:57:10Z 2023-10-09T03:57:00.770 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 03:57:00 smithi181 conmon[175350]: logger=ngalert.multiorg.alertmanager t=2023-10-09T03:57:00.578718099Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T03:57:01.378 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STOPPING 2023-10-09T03:57:01.378 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STOPPED 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STARTING 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Serving on http://:::9283 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STARTED 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STOPPING 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STOPPED 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STARTING 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Serving on http://:::9283 2023-10-09T03:57:01.379 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STARTED 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.053Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.053Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T03:57:01.380 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T03:57:01.381 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T03:57:01.382 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T03:57:01.383 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.054Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.055Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[175961]: ts=2023-10-09T03:57:01.055Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 bash[175920]: 79b2ee0a484da009b46c07c10df30eb4c800df626e4a7c230f73af7e9557bc18 2023-10-09T03:57:01.384 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 03:57:01 smithi181 systemd[1]: Started Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:01.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STOPPING 2023-10-09T03:57:01.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:57:01.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STOPPED 2023-10-09T03:57:01.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STARTING 2023-10-09T03:57:01.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Serving on http://:::9283 2023-10-09T03:57:01.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:01 smithi181 conmon[160841]: [09/Oct/2023:03:57:01] ENGINE Bus STARTED 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181:9095"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181:9095"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181:3000"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181:3000"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T03:57:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3093378436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181:9095"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181:9095"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181:3000"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181:3000"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:02.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T03:57:02.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T03:57:02.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:02.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:02 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3093378436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi163.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181:9095"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181:9095"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181:3000"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi181:3000"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T03:57:02.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T03:57:02.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: pgmap v16: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:02.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:02 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3093378436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: Upgrade: Updating mgr.y 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:03.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: Deploying daemon mgr.y on smithi163 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[101022]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: Upgrade: Updating mgr.y 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: Deploying daemon mgr.y on smithi163 2023-10-09T03:57:03.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:03 smithi163 ceph-mon[118807]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:04.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:03 smithi181 conmon[160841]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:57:03] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.43.0" 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: Upgrade: Updating mgr.y 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: Deploying daemon mgr.y on smithi163 2023-10-09T03:57:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:03 smithi181 ceph-mon[105375]: pgmap v17: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:04.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:04 smithi163 systemd[1]: Stopping Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:04 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3824048499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:04 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3824048499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:04 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3824048499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:05.196 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:04 smithi163 bash[200170]: ec02a34c21a59f771821f1ca617699f3e4012831bae51a1607f60772c0cf652d 2023-10-09T03:57:05.197 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:04 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T03:57:05.197 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T03:57:05.197 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 systemd[1]: Stopped Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:05.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 systemd[1]: Starting Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:06.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:05 smithi163 ceph-mon[118807]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:06.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:05 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:06.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:05 smithi163 ceph-mon[101022]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:06.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:05 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:06.005 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 bash[200525]: 6bfa655af17a7509e2ebec4504ace4d0d95e0b23a6cbf6164fcc968a413ec7da 2023-10-09T03:57:06.005 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 ceph-mgr[200591]: -- 172.21.15.163:0/1562837804 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561fce0191e0 con 0x561fcdff0c00 2023-10-09T03:57:06.005 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 systemd[1]: Started Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:06.005 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:05 smithi163 conmon[200567]: 2023-10-09T03:57:05.918+0000 7fc011f33200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:57:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:05 smithi181 ceph-mon[105375]: pgmap v18: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:05 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:06.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:06 smithi163 conmon[200567]: 2023-10-09T03:57:06.003+0000 7fc011f33200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:06.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:06 smithi163 conmon[200567]: 2023-10-09T03:57:06.101+0000 7fc011f33200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:57:06.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:06 smithi163 conmon[200567]: 2023-10-09T03:57:06.220+0000 7fc011f33200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:57:06.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:06 smithi163 conmon[200567]: 2023-10-09T03:57:06.425+0000 7fc011f33200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:57:07.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:06 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:07.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:06 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:07.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:06 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:06 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:06 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:06 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:06 smithi163 conmon[200567]: 2023-10-09T03:57:06.839+0000 7fc011f33200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:06 smithi163 conmon[200567]: 2023-10-09T03:57:06.986+0000 7fc011f33200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:07 smithi163 conmon[200567]: 2023-10-09T03:57:07.082+0000 7fc011f33200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:57:07.185 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:07 smithi163 conmon[200567]: 2023-10-09T03:57:07.183+0000 7fc011f33200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:57:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:06 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:06 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:06 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:08.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:07 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2046649223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:08.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:07 smithi163 ceph-mon[118807]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:08.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:07 smithi163 conmon[200567]: 2023-10-09T03:57:07.684+0000 7fc011f33200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:57:08.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:07 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2046649223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:08.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:07 smithi163 ceph-mon[101022]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:07 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2046649223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:07 smithi181 ceph-mon[105375]: pgmap v19: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:08.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:08 smithi163 conmon[200567]: 2023-10-09T03:57:08.415+0000 7fc011f33200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:57:08.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:08 smithi163 conmon[200567]: 2023-10-09T03:57:08.501+0000 7fc011f33200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:57:09.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:08 smithi181 conmon[174496]: ts=2023-10-09T03:57:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:57:09.664 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:09 smithi163 conmon[200567]: 2023-10-09T03:57:09.363+0000 7fc011f33200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:09.665 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:09 smithi163 conmon[200567]: 2023-10-09T03:57:09.471+0000 7fc011f33200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:57:09.665 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:09 smithi163 conmon[200567]: 2023-10-09T03:57:09.663+0000 7fc011f33200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:09.947 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.947 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.947 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3946105707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:09.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[118807]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3946105707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:09.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:09 smithi163 ceph-mon[101022]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3946105707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:09 smithi181 ceph-mon[105375]: pgmap v20: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:10.230 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:09 smithi163 conmon[200567]: 2023-10-09T03:57:09.983+0000 7fc011f33200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:57:10.230 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:10 smithi163 conmon[200567]: 2023-10-09T03:57:10.228+0000 7fc011f33200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:57:10.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:10 smithi163 conmon[200567]: 2023-10-09T03:57:10.385+0000 7fc011f33200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:57:10.947 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:10 smithi163 conmon[200567]: 2023-10-09T03:57:10.661+0000 7fc011f33200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:57:11.562 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:11 smithi163 conmon[200567]: 2023-10-09T03:57:11.414+0000 7fc011f33200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:57:11.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:11 smithi163 conmon[200567]: 2023-10-09T03:57:11.561+0000 7fc011f33200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:57:11.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:11 smithi163 conmon[200567]: 2023-10-09T03:57:11.769+0000 7fc011f33200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:57:12.187 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:11 smithi163 conmon[200567]: 2023-10-09T03:57:11.904+0000 7fc011f33200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:12.187 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:12 smithi163 conmon[200567]: 2023-10-09T03:57:12.149+0000 7fc011f33200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:57:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:12 smithi163 ceph-mon[118807]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:12 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2351186201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:12 smithi163 ceph-mon[101022]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:12 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2351186201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:13.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:12 smithi181 ceph-mon[105375]: pgmap v21: 73 pgs: 73 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:13.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:12 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2351186201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:13.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: 2023-10-09T03:57:13.014+0000 7f1a55e34700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T03:57:13.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: ignoring --setuser ceph since I am not root 2023-10-09T03:57:13.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: ignoring --setgroup ceph since I am not root 2023-10-09T03:57:13.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mgr[160865]: -- 172.21.15.181:0/3852737128 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e3e79d51e0 con 0x55e3e79ad000 2023-10-09T03:57:13.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: 2023-10-09T03:57:13.178+0000 7f72cae82200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:57:13.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: 2023-10-09T03:57:13.257+0000 7f72cae82200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.072+0000 7fc011f33200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Bus STARTING 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: CherryPy Checker: 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: The Application mounted at '' has an empty config. 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Serving on http://:::9283 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Bus STARTED 2023-10-09T03:57:13.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Bus STOPPING 2023-10-09T03:57:13.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:57:13.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Bus STOPPED 2023-10-09T03:57:13.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Bus STARTING 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.334+0000 7fbf72bab700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.334+0000 7fbf72bab700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.334+0000 7fbf72bab700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.334+0000 7fbf72bab700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.334+0000 7fbf72bab700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.335+0000 7fbf70ba7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.335+0000 7fbf70ba7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.335+0000 7fbf70ba7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.335+0000 7fbf70ba7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: 2023-10-09T03:57:13.335+0000 7fbf70ba7700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:13.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Serving on http://:::9283 2023-10-09T03:57:13.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:13 smithi163 conmon[200567]: [09/Oct/2023:03:57:13] ENGINE Bus STARTED 2023-10-09T03:57:13.631 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: 2023-10-09T03:57:13.356+0000 7f72cae82200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:57:13.632 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: 2023-10-09T03:57:13.455+0000 7f72cae82200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:13 smithi181 conmon[160841]: 2023-10-09T03:57:13.631+0000 7f72cae82200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.24781 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: mgrmap e25: y(active, starting, since 0.00874418s) 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: Active manager daemon y restarted 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: Activating manager daemon y 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: mgrmap e26: y(active, starting, since 0.00662591s) 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:57:14.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:57:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: Manager daemon y is now available 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:14.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:14.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:14.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:14.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:14.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:13] ENGINE Bus STARTING 2023-10-09T03:57:14.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:13 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:13] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:57:14.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:14.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:14.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.172 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.24781 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: mgrmap e25: y(active, starting, since 0.00874418s) 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: Active manager daemon y restarted 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: Activating manager daemon y 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: mgrmap e26: y(active, starting, since 0.00662591s) 2023-10-09T03:57:14.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:57:14.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 172.21.15.181:0/3250485485' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T03:57:14.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.24781 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: mgrmap e25: y(active, starting, since 0.00874418s) 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: Active manager daemon y restarted 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: Activating manager daemon y 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: mgrmap e26: y(active, starting, since 0.00662591s) 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:57:14.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:57:14.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: Manager daemon y is now available 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: [09/Oct/2023:03:57:13] ENGINE Bus STARTING 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[101022]: [09/Oct/2023:03:57:13] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:57:14.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: Manager daemon y is now available 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:14.180 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:14.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:13] ENGINE Bus STARTING 2023-10-09T03:57:14.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:13 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:13] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:57:14.389 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:14 smithi181 conmon[160841]: 2023-10-09T03:57:14.048+0000 7f72cae82200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:57:14.390 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:14 smithi181 conmon[160841]: 2023-10-09T03:57:14.194+0000 7f72cae82200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:57:14.390 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:14 smithi181 conmon[160841]: 2023-10-09T03:57:14.287+0000 7f72cae82200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:57:15.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:14 smithi181 conmon[160841]: 2023-10-09T03:57:14.389+0000 7f72cae82200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:57:15.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:14 smithi181 conmon[160841]: 2023-10-09T03:57:14.854+0000 7f72cae82200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:57:15.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:15 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:13] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T03:57:15.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:15 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:13] ENGINE Bus STARTED 2023-10-09T03:57:15.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:15 smithi181 ceph-mon[105375]: mgrmap e27: y(active, since 1.0103s) 2023-10-09T03:57:15.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:15 smithi181 ceph-mon[105375]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:15.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:15 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1522587164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:15.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:13] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T03:57:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:13] ENGINE Bus STARTED 2023-10-09T03:57:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[118807]: mgrmap e27: y(active, since 1.0103s) 2023-10-09T03:57:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[118807]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1522587164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[101022]: [09/Oct/2023:03:57:13] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T03:57:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[101022]: [09/Oct/2023:03:57:13] ENGINE Bus STARTED 2023-10-09T03:57:15.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[101022]: mgrmap e27: y(active, since 1.0103s) 2023-10-09T03:57:15.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[101022]: pgmap v3: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:15.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:15 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1522587164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:16.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:15 smithi181 conmon[160841]: 2023-10-09T03:57:15.601+0000 7f72cae82200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:57:16.019 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:15 smithi181 conmon[160841]: 2023-10-09T03:57:15.705+0000 7f72cae82200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:57:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:16 smithi181 ceph-mon[105375]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:16.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:16 smithi163 ceph-mon[118807]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:16 smithi163 ceph-mon[101022]: pgmap v4: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:16.857 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:16 smithi181 conmon[160841]: 2023-10-09T03:57:16.571+0000 7f72cae82200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:16.857 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:16 smithi181 conmon[160841]: 2023-10-09T03:57:16.669+0000 7f72cae82200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:57:16.857 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:16 smithi181 conmon[160841]: 2023-10-09T03:57:16.857+0000 7f72cae82200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: mgrmap e28: y(active, since 3s) 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3490947632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.165 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:17 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.420 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: mgrmap e28: y(active, since 3s) 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3490947632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.421 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: mgrmap e28: y(active, since 3s) 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3490947632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:17 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:17.518 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:17 smithi181 conmon[160841]: 2023-10-09T03:57:17.164+0000 7f72cae82200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:57:17.518 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:17 smithi181 conmon[160841]: 2023-10-09T03:57:17.407+0000 7f72cae82200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:57:17.825 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:17 smithi181 conmon[160841]: 2023-10-09T03:57:17.557+0000 7f72cae82200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:57:18.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:17 smithi181 conmon[160841]: 2023-10-09T03:57:17.825+0000 7f72cae82200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:57:18.539 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:18 smithi163 ceph-mon[101022]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:18.539 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:18 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:18.539 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:18 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:18.540 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:18 smithi163 ceph-mon[118807]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:18.540 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:18 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:18.540 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:18 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:18.669 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:18 smithi181 conmon[160841]: 2023-10-09T03:57:18.533+0000 7f72cae82200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:57:18.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:18 smithi181 ceph-mon[105375]: pgmap v5: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:18.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:18 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:18.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:18 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:18.981 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:18 smithi181 conmon[160841]: 2023-10-09T03:57:18.668+0000 7f72cae82200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:57:18.981 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:18 smithi181 conmon[160841]: 2023-10-09T03:57:18.866+0000 7f72cae82200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:57:19.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:18 smithi181 conmon[160841]: 2023-10-09T03:57:18.980+0000 7f72cae82200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:19.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:19 smithi181 conmon[160841]: 2023-10-09T03:57:19.202+0000 7f72cae82200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:57:19.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:19 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1438496063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:19.705 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:19 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1438496063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:19 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1438496063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:20.373 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: 2023-10-09T03:57:20.085+0000 7f72cae82200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:57:20.374 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: [09/Oct/2023:03:57:20] ENGINE Bus STARTING 2023-10-09T03:57:20.374 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: CherryPy Checker: 2023-10-09T03:57:20.375 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: The Application mounted at '' has an empty config. 2023-10-09T03:57:20.375 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: 2023-10-09T03:57:20.375 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: [09/Oct/2023:03:57:20] ENGINE Serving on http://:::9283 2023-10-09T03:57:20.375 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:20 smithi181 conmon[160841]: [09/Oct/2023:03:57:20] ENGINE Bus STARTED 2023-10-09T03:57:20.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:20.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: Standby manager daemon x started 2023-10-09T03:57:20.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:20.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:20.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:20.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:20.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:20.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:20.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:20.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: Standby manager daemon x started 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:20 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:20.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: pgmap v6: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: Standby manager daemon x started 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/1787098576' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:20 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: mgrmap e29: y(active, since 7s), standbys: x 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:21 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: mgrmap e29: y(active, since 7s), standbys: x 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: mgrmap e29: y(active, since 7s), standbys: x 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:57:21.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:21.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:21 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:22 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:22 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:22 smithi181 ceph-mon[105375]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:22 smithi181 ceph-mon[105375]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:57:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:22 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/111288921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:22.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:22 smithi181 ceph-mon[105375]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:57:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[101022]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[101022]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[101022]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[101022]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:57:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/111288921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[101022]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:57:22.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:22.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:22.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[118807]: pgmap v7: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:22.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[118807]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T03:57:22.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/111288921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:22.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:22 smithi163 ceph-mon[118807]: Reconfiguring daemon prometheus.a on smithi181 2023-10-09T03:57:24.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:23 smithi181 conmon[160841]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T03:57:24.362 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 systemd[1]: Stopping Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:24 smithi181 ceph-mon[105375]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:24 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/1650020806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.362Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.362Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.362Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.362Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.362Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.362Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.363Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.363Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T03:57:24.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.363Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T03:57:24.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.363Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T03:57:24.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.363Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T03:57:24.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 conmon[174496]: ts=2023-10-09T03:57:24.363Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T03:57:24.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:24 smithi163 ceph-mon[118807]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:24.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:24 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/1650020806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:24.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:24 smithi163 ceph-mon[101022]: pgmap v8: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:24.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:24 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/1650020806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:25.268 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:24 smithi181 bash[179931]: cafec70e59d5555ed249abf8a7c355551f0be7d19c196dfc496b5653d868194a 2023-10-09T03:57:25.268 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T03:57:25.269 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 systemd[1]: Stopped Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:25.269 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 systemd[1]: Starting Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:26.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.532Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T03:57:26.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.532Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T03:57:26.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.532Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi181 (none))" 2023-10-09T03:57:26.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.533Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T03:57:26.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.533Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.534Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.534Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 bash[180257]: 3be07b4832ab95ce75a8e3201386223518f1be8e4223b79a793f91f400a64cd1 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.537Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.537Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.538Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.538Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.482µs 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.538Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.538Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 systemd[1]: Started Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:26.020 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.553Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.557Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.557Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.557Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.496µs wal_replay_duration=19.76721ms wbl_replay_duration=103ns total_replay_duration=19.805333ms 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.560Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.560Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.560Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.574Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.818499ms db_storage=859ns remote_storage=1.355µs web_handler=558ns query_engine=833ns scrape=261.489µs scrape_sd=125.349µs notify=10.106µs notify_sd=9.89µs rules=13.156991ms tracing=6.603µs 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.574Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T03:57:26.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:25 smithi181 conmon[180297]: ts=2023-10-09T03:57:25.574Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T03:57:26.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:25 smithi163 conmon[200567]: [09/Oct/2023:03:57:25] ENGINE Bus STOPPING 2023-10-09T03:57:26.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:25 smithi163 conmon[200567]: [09/Oct/2023:03:57:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T03:57:26.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:25 smithi163 conmon[200567]: [09/Oct/2023:03:57:25] ENGINE Bus STOPPED 2023-10-09T03:57:26.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:25 smithi163 conmon[200567]: [09/Oct/2023:03:57:25] ENGINE Bus STARTING 2023-10-09T03:57:26.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:25 smithi163 conmon[200567]: [09/Oct/2023:03:57:25] ENGINE Serving on http://:::9283 2023-10-09T03:57:26.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:25 smithi163 conmon[200567]: [09/Oct/2023:03:57:25] ENGINE Bus STARTED 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:57:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:26.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:26 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2466164717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2466164717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: pgmap v9: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi181.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:26.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:26.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:26 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2466164717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:27 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:27 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:27 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:57:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:27 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:57:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T03:57:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T03:57:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:27 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:28.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:28 smithi181 systemd[1]: Stopping Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:29.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:28 smithi181 ceph-mon[105375]: Upgrade: Updating mgr.x 2023-10-09T03:57:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:28 smithi181 ceph-mon[105375]: Deploying daemon mgr.x on smithi181 2023-10-09T03:57:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:28 smithi181 ceph-mon[105375]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:28 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[118807]: Upgrade: Updating mgr.x 2023-10-09T03:57:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[118807]: Deploying daemon mgr.x on smithi181 2023-10-09T03:57:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[118807]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[101022]: Upgrade: Updating mgr.x 2023-10-09T03:57:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[101022]: Deploying daemon mgr.x on smithi181 2023-10-09T03:57:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[101022]: pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:29.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:28 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:29.456 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:29 smithi181 bash[180931]: e813fb0d7da872895fc254561e94542ea68d323b2c66f2e6b61841e2b8f034ed 2023-10-09T03:57:29.456 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:29 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T03:57:29.456 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:29 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T03:57:29.456 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:29 smithi181 systemd[1]: Stopped Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:29.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:29 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3911014739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:29.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:29 smithi181 systemd[1]: Starting Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:29 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/3911014739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:29 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/3911014739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:30.141 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 ceph-mgr[181353]: -- 172.21.15.181:0/2647299140 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558b3fed91e0 con 0x558b3feb0800 2023-10-09T03:57:30.142 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 bash[181288]: cbeb189f4272e463553a104ef377f877376dc5c9d56726d740f1686e00914085 2023-10-09T03:57:30.142 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 systemd[1]: Started Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:30.429 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 conmon[181329]: 2023-10-09T03:57:30.140+0000 7f586e472200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:57:30.429 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 conmon[181329]: 2023-10-09T03:57:30.223+0000 7f586e472200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:30.429 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 conmon[181329]: 2023-10-09T03:57:30.318+0000 7f586e472200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:57:30.768 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 conmon[181329]: 2023-10-09T03:57:30.427+0000 7f586e472200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:57:30.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:30 smithi181 conmon[181329]: 2023-10-09T03:57:30.626+0000 7f586e472200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:57:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:30 smithi181 ceph-mon[105375]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:30 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:30 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:30 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:57:30] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2023-10-09T03:57:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[118807]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[101022]: pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:30 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:31.370 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:31 smithi181 conmon[181329]: 2023-10-09T03:57:31.027+0000 7f586e472200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:57:31.370 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:31 smithi181 conmon[181329]: 2023-10-09T03:57:31.174+0000 7f586e472200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:57:31.370 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:31 smithi181 conmon[181329]: 2023-10-09T03:57:31.268+0000 7f586e472200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:57:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:31 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2910534371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:31.769 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:31 smithi181 conmon[181329]: 2023-10-09T03:57:31.368+0000 7f586e472200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:57:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:31 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2910534371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:31 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2910534371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:32.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:31 smithi181 conmon[181329]: 2023-10-09T03:57:31.883+0000 7f586e472200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:57:32.717 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:32 smithi181 ceph-mon[105375]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:32.718 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:32 smithi181 conmon[181329]: 2023-10-09T03:57:32.631+0000 7f586e472200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:57:33.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:32 smithi181 conmon[181329]: 2023-10-09T03:57:32.716+0000 7f586e472200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:57:33.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:32 smithi163 ceph-mon[118807]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:32 smithi163 ceph-mon[101022]: pgmap v12: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:33.866 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:33 smithi181 conmon[181329]: 2023-10-09T03:57:33.581+0000 7f586e472200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:33.866 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:33 smithi181 conmon[181329]: 2023-10-09T03:57:33.680+0000 7f586e472200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:57:34.188 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:33 smithi181 conmon[181329]: 2023-10-09T03:57:33.864+0000 7f586e472200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:34.188 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:34 smithi181 conmon[181329]: 2023-10-09T03:57:34.186+0000 7f586e472200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:57:34.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:34 smithi181 ceph-mon[105375]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:34.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:34 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:34 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:34 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2973746463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:34.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:34 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.440 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:34 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[118807]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2973746463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[101022]: pgmap v13: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2973746463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:34.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:34 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:34.728 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:34 smithi181 conmon[181329]: 2023-10-09T03:57:34.438+0000 7f586e472200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:57:34.728 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:34 smithi181 conmon[181329]: 2023-10-09T03:57:34.597+0000 7f586e472200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:57:35.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:34 smithi181 conmon[181329]: 2023-10-09T03:57:34.893+0000 7f586e472200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:57:35.673 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:35 smithi181 conmon[181329]: 2023-10-09T03:57:35.645+0000 7f586e472200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:57:35.999 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:35 smithi181 conmon[181329]: 2023-10-09T03:57:35.793+0000 7f586e472200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:57:36.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:35 smithi181 conmon[181329]: 2023-10-09T03:57:35.998+0000 7f586e472200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:57:36.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:36 smithi181 conmon[181329]: 2023-10-09T03:57:36.123+0000 7f586e472200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:36 smithi163 ceph-mon[118807]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:36 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2118851314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:36.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:36 smithi163 ceph-mon[101022]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:36.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:36 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/2118851314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:36.709 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:36 smithi181 conmon[181329]: 2023-10-09T03:57:36.361+0000 7f586e472200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:57:36.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:36 smithi181 ceph-mon[105375]: pgmap v14: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:36.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:36 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2118851314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:37.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: 2023-10-09T03:57:37.287+0000 7f586e472200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:57:37.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: [09/Oct/2023:03:57:37] ENGINE Bus STARTING 2023-10-09T03:57:37.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: CherryPy Checker: 2023-10-09T03:57:37.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: The Application mounted at '' has an empty config. 2023-10-09T03:57:37.566 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: 2023-10-09T03:57:37.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: [09/Oct/2023:03:57:37] ENGINE Serving on http://:::9283 2023-10-09T03:57:37.567 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:37 smithi181 conmon[181329]: [09/Oct/2023:03:57:37] ENGINE Bus STARTED 2023-10-09T03:57:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: Standby manager daemon x restarted 2023-10-09T03:57:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: Standby manager daemon x started 2023-10-09T03:57:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: Standby manager daemon x restarted 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: Standby manager daemon x started 2023-10-09T03:57:37.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:37.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T03:57:37.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:37 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T03:57:38.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: Standby manager daemon x restarted 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: Standby manager daemon x started 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/150927632' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T03:57:38.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:37 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T03:57:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[101022]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[101022]: Upgrade: Setting container_image for all mgr 2023-10-09T03:57:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[101022]: Upgrade: It appears safe to stop mon.a 2023-10-09T03:57:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[101022]: mgrmap e30: y(active, since 24s), standbys: x 2023-10-09T03:57:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[101022]: from='client.? 172.21.15.163:0/581649562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[118807]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[118807]: Upgrade: Setting container_image for all mgr 2023-10-09T03:57:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[118807]: Upgrade: It appears safe to stop mon.a 2023-10-09T03:57:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[118807]: mgrmap e30: y(active, since 24s), standbys: x 2023-10-09T03:57:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:38 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/581649562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:39.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:38 smithi181 conmon[180297]: ts=2023-10-09T03:57:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:57:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:38 smithi181 ceph-mon[105375]: pgmap v15: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:38 smithi181 ceph-mon[105375]: Upgrade: Setting container_image for all mgr 2023-10-09T03:57:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:38 smithi181 ceph-mon[105375]: Upgrade: It appears safe to stop mon.a 2023-10-09T03:57:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:38 smithi181 ceph-mon[105375]: mgrmap e30: y(active, since 24s), standbys: x 2023-10-09T03:57:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:38 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/581649562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: Upgrade: Updating mon.a 2023-10-09T03:57:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:57:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:57:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: Deploying daemon mon.a on smithi163 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[118807]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: Upgrade: Updating mon.a 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: Deploying daemon mon.a on smithi163 2023-10-09T03:57:40.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 ceph-mon[101022]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:40.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: Upgrade: Updating mon.a 2023-10-09T03:57:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='mgr.y' 2023-10-09T03:57:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:57:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:57:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: from='mgr.25040 172.21.15.163:0/2401616105' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: Deploying daemon mon.a on smithi163 2023-10-09T03:57:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:40 smithi181 ceph-mon[105375]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:40.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 systemd[1]: Stopping Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:40.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 conmon[100998]: 2023-10-09T03:57:40.641+0000 7fe54e766700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T03:57:40.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:40 smithi163 conmon[100998]: 2023-10-09T03:57:40.641+0000 7fe54e766700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T03:57:40.777 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:57:40] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2023-10-09T03:57:41.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 bash[213421]: 51bd91cc44f86a966a2531891bb4ed396d01b78001a7aa0859e50e17f9acec93 2023-10-09T03:57:41.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T03:57:41.471 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 systemd[1]: Stopped Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:41.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 systemd[1]: Starting Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: pidfile_write: ignore empty --pid-file 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 bash[213776]: 8efafb4ec97da0aa3b40923217a63a1d11f35fa64d38f55f3d998b443979ba74 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: load: jerasure load: lrc 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: RocksDB version: 7.9.2 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Git sha 0 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T03:57:42.275 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: DB SUMMARY 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: DB Session ID: OJKWC9W0VYHMY6XGE7G1 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: CURRENT file: CURRENT 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: MANIFEST file: MANIFEST-000015 size: 778 Bytes 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000025.log size: 2525861 ; 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.error_if_exists: 0 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.create_if_missing: 0 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.paranoid_checks: 1 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T03:57:42.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.env: 0x55a6213e6720 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.info_log: 0x55a623501420 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.statistics: (nil) 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.use_fsync: 0 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_log_file_size: 0 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T03:57:42.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.allow_fallocate: 1 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.use_direct_reads: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.db_log_dir: 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.wal_dir: 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T03:57:42.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.write_buffer_manager: 0x55a622790c80 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T03:57:42.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.unordered_write: 0 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.row_cache: None 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.wal_filter: None 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T03:57:42.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.two_write_queues: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.wal_compression: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.atomic_flush: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.log_readahead_size: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T03:57:42.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_background_jobs: 2 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_background_compactions: -1 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_subcompactions: 1 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T03:57:42.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_open_files: -1 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_background_flushes: -1 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Compression algorithms supported: 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T03:57:42.283 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kZSTD supported: 0 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kXpressCompression supported: 0 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kZlibCompression supported: 1 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kSnappyCompression supported: 1 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kLZ4Compression supported: 1 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: kBZip2Compression supported: 0 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.merge_operator: 2023-10-09T03:57:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_filter: None 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a623501520) 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks: 1 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout: pin_top_level_index_and_filter: 1 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_type: 0 2023-10-09T03:57:42.285 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_index_type: 0 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_shortening: 1 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: checksum: 4 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: no_block_cache: 0 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache: 0x55a6228131f0 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_name: BinnedLRUCache 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_options: 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: capacity : 536870912 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: num_shard_bits : 4 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: strict_capacity_limit : 0 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: high_pri_pool_ratio: 0.000 2023-10-09T03:57:42.286 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_cache_compressed: (nil) 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: persistent_cache: (nil) 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size: 4096 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_size_deviation: 10 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_restart_interval: 16 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: index_block_restart_interval: 1 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: metadata_block_size: 4096 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: partition_filters: 0 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: use_delta_encoding: 1 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: filter_policy: bloomfilter 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: whole_key_filtering: 1 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: verify_compression: 0 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: read_amp_bytes_per_bit: 0 2023-10-09T03:57:42.287 INFO:journalctl@ceph.mon.a.smithi163.stdout: format_version: 5 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_index_compression: 1 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout: block_align: 0 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout: max_auto_readahead_size: 262144 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout: prepopulate_block_cache: 0 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout: initial_auto_readahead_size: 8192 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression: NoCompression 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T03:57:42.288 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.num_levels: 7 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T03:57:42.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T03:57:42.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T03:57:42.292 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.table_properties_collectors: 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.inplace_update_support: 0 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T03:57:42.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.bloom_locality: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.max_successive_merges: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.ttl: 2592000 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.enable_blob_files: false 2023-10-09T03:57:42.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.min_blob_size: 0 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 29, last_sequence is 10602, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T03:57:42.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 01771d03-7dbc-4eff-bfe5-33a3b7fdc374 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823861985147, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 systemd[1]: Started Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823861993101, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2280626, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10590, "largest_seqno": 11855, "table_properties": {"data_size": 2274461, "index_size": 3674, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 14086, "raw_average_key_size": 24, "raw_value_size": 2261897, "raw_average_value_size": 3899, "num_data_blocks": 169, "num_entries": 580, "num_filter_entries": 580, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696823861, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "01771d03-7dbc-4eff-bfe5-33a3b7fdc374", "db_session_id": "OJKWC9W0VYHMY6XGE7G1", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823861993195, "job": 1, "event": "recovery_finished"} 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a6228ae000 2023-10-09T03:57:42.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: DB pointer 0x55a62289c000 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** DB Stats ** 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.297 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: L0 1/0 2.17 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 554.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: L6 1/0 6.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: Sum 2/0 8.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 554.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 554.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:57:42.298 INFO:journalctl@ceph.mon.a.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 554.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: Cumulative compaction: 0.00 GB write, 177.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: Interval compaction: 0.00 GB write, 177.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:57:42.299 INFO:journalctl@ceph.mon.a.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: Block cache BinnedLRUCache@0x55a6228131f0#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: starting mon.a rank 0 at public addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] at bind addrs [v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???) e3 preinit fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).mds e9 new map 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).mds e9 print_map 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: e9 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T03:57:42.300 INFO:journalctl@ceph.mon.a.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: legacy client fscid: 1 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: Filesystem 'foo' (1) 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: fs_name foo 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: epoch 8 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: created 2023-10-09T03:54:13.223139+0000 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: modified 2023-10-09T03:54:28.181856+0000 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: tableserver 0 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: root 0 2023-10-09T03:57:42.301 INFO:journalctl@ceph.mon.a.smithi163.stdout: session_timeout 60 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: session_autoclose 300 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: max_file_size 1099511627776 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: required_client_features {} 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: last_failure 0 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: last_failure_osd_epoch 0 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: max_mds 1 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: in 0 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: up {0=24569} 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: failed 2023-10-09T03:57:42.302 INFO:journalctl@ceph.mon.a.smithi163.stdout: damaged 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: stopped 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: data_pools [4] 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: metadata_pool 3 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: inline_data disabled 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: balancer 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: bal_rank_mask -1 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: standby_count_wanted 1 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: [mds.foo.smithi163.crqkdj{0:24569} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: Standby daemons: 2023-10-09T03:57:42.303 INFO:journalctl@ceph.mon.a.smithi163.stdout: 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout: [mds.foo.smithi181.sboiwx{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).osd e79 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).osd e79 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:41 smithi163 ceph-mon[213841]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expand map: {default=false} 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta from 'false' to 'false' 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expanded map: {default=false} 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expand map: {default=info} 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta from 'info' to 'info' 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expanded map: {default=info} 2023-10-09T03:57:42.304 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expand map: {default=daemon} 2023-10-09T03:57:42.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T03:57:42.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T03:57:42.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expand map: {default=debug} 2023-10-09T03:57:42.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T03:57:42.305 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: expand_channel_meta expanded map: {default=debug} 2023-10-09T03:57:42.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: from='client.15261 172.21.15.163:0/1101449408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: mon.a calling monitor election 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: mgrmap e30: y(active, since 29s), standbys: x 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: mgrmap e31: y(active, since 29s), standbys: x 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: overall HEALTH_OK 2023-10-09T03:57:42.579 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[213841]: from='mgr.25040 ' entity='' 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: ignoring --setuser ceph since I am not root 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: ignoring --setgroup ceph since I am not root 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mgr[200591]: -- 172.21.15.163:0/278479701 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ca012b71e0 con 0x55ca0128f000 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: 2023-10-09T03:57:42.409+0000 7f40cda52200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: 2023-10-09T03:57:42.488+0000 7f40cda52200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: 2023-10-09T03:57:42.577+0000 7f40cda52200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:57:42.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: from='client.15261 172.21.15.163:0/1101449408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: mon.a calling monitor election 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: mgrmap e30: y(active, since 29s), standbys: x 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: mgrmap e31: y(active, since 29s), standbys: x 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: overall HEALTH_OK 2023-10-09T03:57:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:42 smithi163 ceph-mon[118807]: from='mgr.25040 ' entity='' 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: from='client.15261 172.21.15.163:0/1101449408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: mon.a calling monitor election 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T03:57:42.582 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: mgrmap e30: y(active, since 29s), standbys: x 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: mgrmap e31: y(active, since 29s), standbys: x 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: overall HEALTH_OK 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mon[105375]: from='mgr.25040 ' entity='' 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: ignoring --setuser ceph since I am not root 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: ignoring --setgroup ceph since I am not root 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 ceph-mgr[181353]: -- 172.21.15.181:0/2301608810 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bfb8ba11e0 con 0x55bfb8b78c00 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: 2023-10-09T03:57:42.411+0000 7f4b1a816200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: 2023-10-09T03:57:42.489+0000 7f4b1a816200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:42.583 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: 2023-10-09T03:57:42.578+0000 7f4b1a816200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T03:57:42.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: 2023-10-09T03:57:42.671+0000 7f40cda52200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:57:42.843 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: 2023-10-09T03:57:42.672+0000 7f4b1a816200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T03:57:43.093 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T03:57:43.101 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:42 smithi163 conmon[200567]: 2023-10-09T03:57:42.850+0000 7f40cda52200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:57:43.205 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:42 smithi181 conmon[181329]: 2023-10-09T03:57:42.842+0000 7f4b1a816200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T03:57:43.377 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:43 smithi163 conmon[200567]: 2023-10-09T03:57:43.236+0000 7f40cda52200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:57:43.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:43 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/693985576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:43.377 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/693985576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:43 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/693985576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:57:43.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:43 smithi181 conmon[181329]: 2023-10-09T03:57:43.204+0000 7f4b1a816200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T03:57:43.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:43 smithi181 conmon[181329]: 2023-10-09T03:57:43.335+0000 7f4b1a816200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:57:43.519 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:43 smithi181 conmon[181329]: 2023-10-09T03:57:43.423+0000 7f4b1a816200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:57:43.740 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:43 smithi163 conmon[200567]: 2023-10-09T03:57:43.375+0000 7f40cda52200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T03:57:43.741 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:43 smithi163 conmon[200567]: 2023-10-09T03:57:43.465+0000 7f40cda52200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T03:57:43.741 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:43 smithi163 conmon[200567]: 2023-10-09T03:57:43.560+0000 7f40cda52200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:57:43.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T03:57:43.940 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:43 smithi181 conmon[181329]: 2023-10-09T03:57:43.519+0000 7f4b1a816200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T03:57:44.015 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:44 smithi163 conmon[200567]: 2023-10-09T03:57:44.013+0000 7f40cda52200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:57:44.122 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:57:44.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:43 smithi181 conmon[181329]: 2023-10-09T03:57:43.939+0000 7f4b1a816200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T03:57:45.019 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:44 smithi181 conmon[181329]: 2023-10-09T03:57:44.595+0000 7f4b1a816200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:57:45.019 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:44 smithi181 conmon[181329]: 2023-10-09T03:57:44.679+0000 7f4b1a816200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:57:45.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:44 smithi163 conmon[200567]: 2023-10-09T03:57:44.729+0000 7f40cda52200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T03:57:45.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:44 smithi163 conmon[200567]: 2023-10-09T03:57:44.820+0000 7f40cda52200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T03:57:45.728 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:45 smithi181 conmon[181329]: 2023-10-09T03:57:45.456+0000 7f4b1a816200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:45.728 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:45 smithi181 conmon[181329]: 2023-10-09T03:57:45.548+0000 7f4b1a816200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:57:45.759 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:45 smithi163 conmon[200567]: 2023-10-09T03:57:45.657+0000 7f40cda52200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:46.013 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:45 smithi181 conmon[181329]: 2023-10-09T03:57:45.726+0000 7f4b1a816200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:46.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:45 smithi163 conmon[200567]: 2023-10-09T03:57:45.756+0000 7f40cda52200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T03:57:46.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:45 smithi163 conmon[200567]: 2023-10-09T03:57:45.936+0000 7f40cda52200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T03:57:46.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:46 smithi181 conmon[181329]: 2023-10-09T03:57:46.011+0000 7f4b1a816200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:57:46.268 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:46 smithi181 conmon[181329]: 2023-10-09T03:57:46.239+0000 7f4b1a816200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:57:46.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:46 smithi163 conmon[200567]: 2023-10-09T03:57:46.219+0000 7f40cda52200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T03:57:46.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:46 smithi163 conmon[200567]: 2023-10-09T03:57:46.448+0000 7f40cda52200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T03:57:46.657 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:46 smithi181 conmon[181329]: 2023-10-09T03:57:46.389+0000 7f4b1a816200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:57:46.868 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:46 smithi163 conmon[200567]: 2023-10-09T03:57:46.598+0000 7f40cda52200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T03:57:47.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:46 smithi181 conmon[181329]: 2023-10-09T03:57:46.655+0000 7f4b1a816200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:57:47.331 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:46 smithi163 conmon[200567]: 2023-10-09T03:57:46.867+0000 7f40cda52200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T03:57:47.696 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:47 smithi181 conmon[181329]: 2023-10-09T03:57:47.364+0000 7f4b1a816200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:57:47.696 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:47 smithi181 conmon[181329]: 2023-10-09T03:57:47.497+0000 7f4b1a816200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:57:47.916 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:47 smithi163 conmon[200567]: 2023-10-09T03:57:47.583+0000 7f40cda52200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T03:57:47.916 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:47 smithi163 conmon[200567]: 2023-10-09T03:57:47.717+0000 7f40cda52200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T03:57:48.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:47 smithi181 conmon[181329]: 2023-10-09T03:57:47.695+0000 7f4b1a816200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:57:48.018 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:47 smithi181 conmon[181329]: 2023-10-09T03:57:47.811+0000 7f4b1a816200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:48.256 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:47 smithi163 conmon[200567]: 2023-10-09T03:57:47.915+0000 7f40cda52200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T03:57:48.257 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:48 smithi163 conmon[200567]: 2023-10-09T03:57:48.031+0000 7f40cda52200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T03:57:48.518 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:48 smithi181 conmon[181329]: 2023-10-09T03:57:48.034+0000 7f4b1a816200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:57:48.923 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:48 smithi181 conmon[181329]: 2023-10-09T03:57:48.907+0000 7f4b1a816200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:57:48.924 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:48 smithi181 conmon[180297]: ts=2023-10-09T03:57:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:57:49.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:48 smithi181 ceph-mon[105375]: Standby manager daemon x restarted 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:48 smithi181 ceph-mon[105375]: Standby manager daemon x started 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:48 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:48 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:48 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:48 smithi181 ceph-mon[105375]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 conmon[105351]: 2023-10-09T03:57:49.153+0000 7f90164b2700 -1 mon.b@1(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:48 smithi181 conmon[181329]: [09/Oct/2023:03:57:48] ENGINE Bus STARTING 2023-10-09T03:57:49.269 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:49 smithi181 conmon[181329]: CherryPy Checker: 2023-10-09T03:57:49.270 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:49 smithi181 conmon[181329]: The Application mounted at '' has an empty config. 2023-10-09T03:57:49.270 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:49 smithi181 conmon[181329]: 2023-10-09T03:57:49.270 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:49 smithi181 conmon[181329]: [09/Oct/2023:03:57:49] ENGINE Serving on http://:::9283 2023-10-09T03:57:49.270 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 03:57:49 smithi181 conmon[181329]: [09/Oct/2023:03:57:49] ENGINE Bus STARTED 2023-10-09T03:57:49.294 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:48 smithi163 conmon[200567]: 2023-10-09T03:57:48.255+0000 7f40cda52200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T03:57:49.294 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.135+0000 7f40cda52200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T03:57:49.294 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: [09/Oct/2023:03:57:49] ENGINE Bus STARTING 2023-10-09T03:57:49.294 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.293+0000 7f402857e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[213841]: Standby manager daemon x restarted 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[213841]: Standby manager daemon x started 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[213841]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[213841]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[213841]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[213841]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[118807]: Standby manager daemon x restarted 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[118807]: Standby manager daemon x started 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T03:57:49.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T03:57:49.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:48 smithi163 ceph-mon[118807]: from='mgr.? 172.21.15.181:0/2385010062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T03:57:49.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[118783]: 2023-10-09T03:57:49.158+0000 7f595f82a700 -1 mon.c@2(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T03:57:49.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.293+0000 7f402857e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.293+0000 7f402857e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.293+0000 7f402857e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.293+0000 7f402857e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.295+0000 7f402c586700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.295+0000 7f402c586700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.295+0000 7f402c586700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.295+0000 7f402c586700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.295+0000 7f402c586700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: CherryPy Checker: 2023-10-09T03:57:49.582 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: The Application mounted at '' has an empty config. 2023-10-09T03:57:49.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: 2023-10-09T03:57:49.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: [09/Oct/2023:03:57:49] ENGINE Serving on http://:::9283 2023-10-09T03:57:49.583 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:49 smithi163 conmon[200567]: [09/Oct/2023:03:57:49] ENGINE Bus STARTED 2023-10-09T03:57:50.179 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T03:57:50.179 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (60s) - 4m 23.8M - 0.25.0 c8568f914cd2 b04e9e96f8bb 2023-10-09T03:57:50.179 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (51s) - 4m 84.5M - 9.4.7 2c41d148cca3 dd057a75b8e1 2023-10-09T03:57:50.179 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.crqkdj smithi163 running (3m) - 3m 16.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 8037aa8a1d7d 2023-10-09T03:57:50.179 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.sboiwx smithi181 running (3m) - 3m 14.4M - 17.2.6-1303-gb2903cb0 5517396c50d4 2c2246f2ef4b 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (20s) - 8m 45.9M - 18.2.0-598-gb4fa663f 3060897fdf97 cbeb189f4272 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (44s) - 9m 479M - 18.2.0-598-gb4fa663f 3060897fdf97 6bfa655af17a 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (9m) - 9m 101M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 51bd91cc44f8 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (8m) - 8m 72.0M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 4a4d2d630200 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (8m) - 8m 70.7M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 72ef10caabbb 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (59s) - 3m 8006k - 1.5.0 0da6a335fe13 381d36940ced 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (49s) - 3m 8502k - 1.5.0 0da6a335fe13 79b2ee0a484d 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (7m) - 7m 73.4M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 1d12e7087516 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (7m) - 7m 75.2M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 1b27d5ad7adb 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (6m) - 6m 147M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 e2ce5dc7d22b 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (6m) - 6m 67.0M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0e8301f836bc 2023-10-09T03:57:50.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (6m) - 6m 74.0M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 b4d4d04d2a3e 2023-10-09T03:57:50.181 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (5m) - 5m 96.0M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4ed298059fdf 2023-10-09T03:57:50.181 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (5m) - 5m 71.2M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4d54b928e978 2023-10-09T03:57:50.181 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (5m) - 5m 100M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 6ca0d6954129 2023-10-09T03:57:50.181 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (24s) - 4m 38.8M - 2.43.0 a07b618ecd1d 3be07b4832ab 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: mgrmap e32: y(active, since 35s), standbys: x 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: Active manager daemon y restarted 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: Activating manager daemon y 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: mgrmap e33: y(active, starting, since 0.010734s), standbys: x 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:57:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:57:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: Manager daemon y is now available 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:50.332 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:50 smithi163 conmon[200567]: 2023-10-09T03:57:50.167+0000 7f40871b1700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T03:57:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: mgrmap e32: y(active, since 35s), standbys: x 2023-10-09T03:57:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: Active manager daemon y restarted 2023-10-09T03:57:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: Activating manager daemon y 2023-10-09T03:57:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: mgrmap e33: y(active, starting, since 0.010734s), standbys: x 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:57:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: Manager daemon y is now available 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:50.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:50.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:50.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: mgrmap e32: y(active, since 35s), standbys: x 2023-10-09T03:57:50.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: Active manager daemon y restarted 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: Activating manager daemon y 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: mgrmap e33: y(active, starting, since 0.010734s), standbys: x 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.sboiwx"}]: dispatch 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi163.crqkdj"}]: dispatch 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T03:57:50.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: Manager daemon y is now available 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: failed to encode map e80 with expected crc 2023-10-09T03:57:50.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:49 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T03:57:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:57:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:57:50] "GET /metrics HTTP/1.1" 200 31236 "" "Prometheus/2.43.0" 2023-10-09T03:57:50.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T03:57:51.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: failed to encode map e80 with expected crc 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:49] ENGINE Bus STARTING 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:49] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:50] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: [09/Oct/2023:03:57:50] ENGINE Bus STARTED 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: mgrmap e34: y(active, since 1.01432s), standbys: x 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:57:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:50 smithi181 ceph-mon[105375]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: failed to encode map e80 with expected crc 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:49] ENGINE Bus STARTING 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:49] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:50] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: [09/Oct/2023:03:57:50] ENGINE Bus STARTED 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: mgrmap e34: y(active, since 1.01432s), standbys: x 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[118807]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: failed to encode map e80 with expected crc 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: [09/Oct/2023:03:57:49] ENGINE Bus STARTING 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: [09/Oct/2023:03:57:49] ENGINE Serving on http://172.21.15.163:8765 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: [09/Oct/2023:03:57:50] ENGINE Serving on https://172.21.15.163:7150 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: [09/Oct/2023:03:57:50] ENGINE Bus STARTED 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: mgrmap e34: y(active, since 1.01432s), standbys: x 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:57:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:50 smithi163 ceph-mon[213841]: pgmap v3: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:51.498 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:57:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:52 smithi181 ceph-mon[105375]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:52 smithi181 ceph-mon[105375]: mgrmap e35: y(active, since 2s), standbys: x 2023-10-09T03:57:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:52 smithi163 ceph-mon[118807]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:52 smithi163 ceph-mon[118807]: mgrmap e35: y(active, since 2s), standbys: x 2023-10-09T03:57:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:52 smithi163 ceph-mon[213841]: pgmap v4: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:52 smithi163 ceph-mon[213841]: mgrmap e35: y(active, since 2s), standbys: x 2023-10-09T03:57:53.425 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:53 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:53 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T03:57:54.419 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:57:54.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[118807]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:57:54.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[118807]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:57:54.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[213841]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:57:54.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[213841]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:57:54.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.958 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:54 smithi181 ceph-mon[105375]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:57:54.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:54 smithi181 ceph-mon[105375]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:57:54.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:54 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:54.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:54 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:55.966 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:55 smithi181 ceph-mon[105375]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:56.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:55 smithi163 ceph-mon[213841]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:56.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:55 smithi163 ceph-mon[118807]: pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:57.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:57.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:57 smithi181 ceph-mon[105375]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:57:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[213841]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.conf 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.conf 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi163:/etc/ceph/ceph.client.admin.keyring 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:57.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:57.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:57 smithi163 ceph-mon[118807]: Updating smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:58.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:57:58 smithi181 conmon[180297]: ts=2023-10-09T03:57:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:57:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:57:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:57:59.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:57:59 smithi163 ceph-mon[118807]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:57:59.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:57:59 smithi163 ceph-mon[213841]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:00.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:57:59 smithi181 ceph-mon[105375]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:01.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:58:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:58:00] "GET /metrics HTTP/1.1" 200 31236 "" "Prometheus/2.43.0" 2023-10-09T03:58:02.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:02 smithi181 ceph-mon[105375]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:02 smithi163 ceph-mon[118807]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:02 smithi163 ceph-mon[213841]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:04.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:04 smithi181 ceph-mon[105375]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:04 smithi163 ceph-mon[118807]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:04 smithi163 ceph-mon[213841]: pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:05.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:05 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:05.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:05 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:06.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:06 smithi181 ceph-mon[105375]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:06.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:06 smithi163 ceph-mon[118807]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:06.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:06 smithi163 ceph-mon[213841]: pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:08 smithi181 ceph-mon[105375]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:08 smithi163 ceph-mon[118807]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:08 smithi163 ceph-mon[213841]: pgmap v12: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:09.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:58:08 smithi181 conmon[180297]: ts=2023-10-09T03:58:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:58:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:10 smithi181 ceph-mon[105375]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:10 smithi163 ceph-mon[118807]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:10 smithi163 ceph-mon[213841]: pgmap v13: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:11.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:58:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:58:10] "GET /metrics HTTP/1.1" 200 33870 "" "Prometheus/2.43.0" 2023-10-09T03:58:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:12 smithi181 ceph-mon[105375]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:12 smithi163 ceph-mon[213841]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:12 smithi163 ceph-mon[118807]: pgmap v14: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:14 smithi181 ceph-mon[105375]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:14 smithi163 ceph-mon[118807]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:14 smithi163 ceph-mon[213841]: pgmap v15: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:16 smithi181 ceph-mon[105375]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:16 smithi163 ceph-mon[213841]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:16 smithi163 ceph-mon[118807]: pgmap v16: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:18 smithi163 ceph-mon[118807]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:18.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:18 smithi163 ceph-mon[213841]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:18 smithi181 ceph-mon[105375]: pgmap v17: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:19.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:58:18 smithi181 conmon[180297]: ts=2023-10-09T03:58:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:58:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:20 smithi181 ceph-mon[105375]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:20 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:20 smithi163 ceph-mon[118807]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:20 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:20 smithi163 ceph-mon[213841]: pgmap v18: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:58:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:58:20] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T03:58:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:22 smithi181 ceph-mon[105375]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:22 smithi163 ceph-mon[213841]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:22 smithi163 ceph-mon[118807]: pgmap v19: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:24.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:24 smithi181 ceph-mon[105375]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:24 smithi163 ceph-mon[213841]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:24 smithi163 ceph-mon[118807]: pgmap v20: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:26.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:26 smithi181 ceph-mon[105375]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:26 smithi163 ceph-mon[118807]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:26 smithi163 ceph-mon[213841]: pgmap v21: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:28.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:28 smithi163 ceph-mon[213841]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:28 smithi163 ceph-mon[118807]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:28.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:28 smithi181 ceph-mon[105375]: pgmap v22: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:29.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:58:28 smithi181 conmon[180297]: ts=2023-10-09T03:58:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:58:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:30 smithi181 ceph-mon[105375]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:30 smithi163 ceph-mon[213841]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:30 smithi163 ceph-mon[118807]: pgmap v23: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:58:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:58:30] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T03:58:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:32 smithi181 ceph-mon[105375]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:32 smithi163 ceph-mon[118807]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:32.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:32 smithi163 ceph-mon[213841]: pgmap v24: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:34.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:34 smithi181 ceph-mon[105375]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:34.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:34 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:34 smithi163 ceph-mon[118807]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:34 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:34 smithi163 ceph-mon[213841]: pgmap v25: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:36 smithi181 ceph-mon[105375]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:36 smithi163 ceph-mon[118807]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:36.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:36 smithi163 ceph-mon[213841]: pgmap v26: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:38 smithi163 ceph-mon[213841]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:38.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:38 smithi163 ceph-mon[118807]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:38.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:38 smithi181 ceph-mon[105375]: pgmap v27: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:39.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:58:38 smithi181 conmon[180297]: ts=2023-10-09T03:58:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:58:40.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:40 smithi181 ceph-mon[105375]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:40.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:40 smithi163 ceph-mon[213841]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:40 smithi163 ceph-mon[118807]: pgmap v28: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:41.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:58:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:58:40] "GET /metrics HTTP/1.1" 200 33868 "" "Prometheus/2.43.0" 2023-10-09T03:58:42.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:42 smithi181 ceph-mon[105375]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:42 smithi163 ceph-mon[118807]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:42.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:42 smithi163 ceph-mon[213841]: pgmap v29: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:44 smithi181 ceph-mon[105375]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:44 smithi163 ceph-mon[213841]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:44.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:44 smithi163 ceph-mon[118807]: pgmap v30: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:46 smithi181 ceph-mon[105375]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:46 smithi163 ceph-mon[118807]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:46 smithi163 ceph-mon[213841]: pgmap v31: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:48 smithi163 ceph-mon[118807]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:48 smithi163 ceph-mon[213841]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:48 smithi181 ceph-mon[105375]: pgmap v32: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:49.021 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:58:48 smithi181 conmon[180297]: ts=2023-10-09T03:58:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:58:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:50 smithi181 ceph-mon[105375]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:50 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:50 smithi163 ceph-mon[213841]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:50 smithi163 ceph-mon[118807]: pgmap v33: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:50 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:58:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:58:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:58:50] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T03:58:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:52 smithi181 ceph-mon[105375]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:52 smithi163 ceph-mon[118807]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:52 smithi163 ceph-mon[213841]: pgmap v34: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:54 smithi181 ceph-mon[105375]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:54 smithi163 ceph-mon[213841]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:54 smithi163 ceph-mon[118807]: pgmap v35: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:56.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T03:58:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:56 smithi181 ceph-mon[105375]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:56.546 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:58:56.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:56 smithi163 ceph-mon[213841]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:56 smithi163 ceph-mon[118807]: pgmap v36: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:58.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:58 smithi163 ceph-mon[118807]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:58.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:58 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:58:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:58 smithi163 ceph-mon[213841]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:58:58.222 INFO:teuthology.orchestra.run.smithi163.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:58:58.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:58 smithi181 ceph-mon[105375]: pgmap v37: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:58:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:58 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:58:58.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T03:58:59.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:58:58 smithi181 conmon[180297]: ts=2023-10-09T03:58:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:58:59.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[118807]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:58:59.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[118807]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:58:59.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[213841]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:58:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[213841]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:58:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:58:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.390 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:58:59.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:59 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:59 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:58:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:59 smithi181 ceph-mon[105375]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:58:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:59 smithi181 ceph-mon[105375]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:58:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:58:59 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:00.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:00 smithi163 ceph-mon[118807]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:00.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:00 smithi163 ceph-mon[213841]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:00 smithi181 ceph-mon[105375]: pgmap v38: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:00.739 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:59:00] "GET /metrics HTTP/1.1" 200 33867 "" "Prometheus/2.43.0" 2023-10-09T03:59:00.793 INFO:teuthology.orchestra.run.smithi163.stdout:wait for majority of mons to upgrade 2023-10-09T03:59:01.393 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T03:59:01.816 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:59:02.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:01 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:01 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:01 smithi181 ceph-mon[105375]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:02.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:01 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:02.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:01 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:02.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:01 smithi163 ceph-mon[118807]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:02.078 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:02.078 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:02.078 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:01 smithi163 ceph-mon[213841]: pgmap v39: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: Upgrade: It appears safe to stop mon.c 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[213841]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:03.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T03:59:03.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T03:59:03.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T03:59:03.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: Upgrade: It appears safe to stop mon.c 2023-10-09T03:59:03.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:03 smithi163 ceph-mon[118807]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T03:59:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: Upgrade: It appears safe to stop mon.c 2023-10-09T03:59:03.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:03 smithi181 ceph-mon[105375]: pgmap v40: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:04 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/2732266283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:04 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:04 smithi163 ceph-mon[118807]: from='client.? 172.21.15.163:0/2732266283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:04 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2732266283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: Upgrade: Updating mon.c 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: Deploying daemon mon.c on smithi163 2023-10-09T03:59:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[213841]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: Upgrade: Updating mon.c 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: Deploying daemon mon.c on smithi163 2023-10-09T03:59:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 ceph-mon[118807]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: Upgrade: Updating mon.c 2023-10-09T03:59:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T03:59:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T03:59:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: Deploying daemon mon.c on smithi163 2023-10-09T03:59:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:05 smithi181 ceph-mon[105375]: pgmap v41: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:05 smithi163 systemd[1]: Stopping Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:59:06.453 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:06 smithi163 conmon[118783]: 2023-10-09T03:59:06.118+0000 7f5965035700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T03:59:06.453 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:06 smithi163 conmon[118783]: 2023-10-09T03:59:06.118+0000 7f5965035700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T03:59:06.453 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:06 smithi163 bash[220639]: 72ef10caabbbdc9cddc0f33ff941648df08b26db51151a9480a9d693d45a66dd 2023-10-09T03:59:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:06 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T03:59:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:06 smithi163 systemd[1]: Stopped Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:59:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:06 smithi163 systemd[1]: Starting Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T03:59:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3705055922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T03:59:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T03:59:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 bash[221001]: 0f3bd992ec5ce3e2a852faed7b3c37a692e3be22c53134201d86305665919d86 2023-10-09T03:59:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: pidfile_write: ignore empty --pid-file 2023-10-09T03:59:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 systemd[1]: Started Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: load: jerasure load: lrc 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: RocksDB version: 7.9.2 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Git sha 0 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: DB SUMMARY 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: DB Session ID: IRYYMBMVGODT4ST0RA13 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: CURRENT file: CURRENT 2023-10-09T03:59:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T03:59:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1082 Bytes 2023-10-09T03:59:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T03:59:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 887855 ; 2023-10-09T03:59:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.error_if_exists: 0 2023-10-09T03:59:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.create_if_missing: 0 2023-10-09T03:59:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.paranoid_checks: 1 2023-10-09T03:59:07.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T03:59:07.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T03:59:07.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T03:59:07.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.env: 0x564e9bdfa720 2023-10-09T03:59:07.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T03:59:07.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.info_log: 0x564e9e1ab420 2023-10-09T03:59:07.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T03:59:07.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.statistics: (nil) 2023-10-09T03:59:07.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.use_fsync: 0 2023-10-09T03:59:07.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_log_file_size: 0 2023-10-09T03:59:07.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T03:59:07.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T03:59:07.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T03:59:07.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T03:59:07.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.allow_fallocate: 1 2023-10-09T03:59:07.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T03:59:07.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T03:59:07.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.use_direct_reads: 0 2023-10-09T03:59:07.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T03:59:07.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T03:59:07.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.db_log_dir: 2023-10-09T03:59:07.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.wal_dir: 2023-10-09T03:59:07.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T03:59:07.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T03:59:07.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T03:59:07.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.write_buffer_manager: 0x564e9d43ac80 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T03:59:07.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.unordered_write: 0 2023-10-09T03:59:07.842 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.row_cache: None 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.wal_filter: None 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T03:59:07.843 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.two_write_queues: 0 2023-10-09T03:59:07.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T03:59:07.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.wal_compression: 0 2023-10-09T03:59:07.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.atomic_flush: 0 2023-10-09T03:59:07.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T03:59:07.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T03:59:07.844 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T03:59:07.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.log_readahead_size: 0 2023-10-09T03:59:07.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T03:59:07.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T03:59:07.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T03:59:07.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T03:59:07.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T03:59:07.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T03:59:07.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T03:59:07.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_background_jobs: 2 2023-10-09T03:59:07.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_background_compactions: -1 2023-10-09T03:59:07.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_subcompactions: 1 2023-10-09T03:59:07.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T03:59:07.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T03:59:07.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T03:59:07.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T03:59:07.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T03:59:07.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T03:59:07.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T03:59:07.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T03:59:07.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_open_files: -1 2023-10-09T03:59:07.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T03:59:07.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T03:59:07.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T03:59:07.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T03:59:07.849 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_background_flushes: -1 2023-10-09T03:59:07.849 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Compression algorithms supported: 2023-10-09T03:59:07.849 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T03:59:07.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kZSTD supported: 0 2023-10-09T03:59:07.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kXpressCompression supported: 0 2023-10-09T03:59:07.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T03:59:07.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kZlibCompression supported: 1 2023-10-09T03:59:07.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kSnappyCompression supported: 1 2023-10-09T03:59:07.851 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kLZ4Compression supported: 1 2023-10-09T03:59:07.851 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: kBZip2Compression supported: 0 2023-10-09T03:59:07.851 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T03:59:07.851 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.merge_operator: 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_filter: None 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564e9e1ab520) 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout: cache_index_and_filter_blocks: 1 2023-10-09T03:59:07.852 INFO:journalctl@ceph.mon.c.smithi163.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: pin_top_level_index_and_filter: 1 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_type: 0 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_block_index_type: 0 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_shortening: 1 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: checksum: 4 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: no_block_cache: 0 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache: 0x564e9d4bd350 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_name: BinnedLRUCache 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_options: 2023-10-09T03:59:07.853 INFO:journalctl@ceph.mon.c.smithi163.stdout: capacity : 536870912 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: num_shard_bits : 4 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: strict_capacity_limit : 0 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: high_pri_pool_ratio: 0.000 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_cache_compressed: (nil) 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: persistent_cache: (nil) 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_size: 4096 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_size_deviation: 10 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_restart_interval: 16 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: index_block_restart_interval: 1 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: metadata_block_size: 4096 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: partition_filters: 0 2023-10-09T03:59:07.854 INFO:journalctl@ceph.mon.c.smithi163.stdout: use_delta_encoding: 1 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: filter_policy: bloomfilter 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: whole_key_filtering: 1 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: verify_compression: 0 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: read_amp_bytes_per_bit: 0 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: format_version: 5 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: enable_index_compression: 1 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: block_align: 0 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: max_auto_readahead_size: 262144 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: prepopulate_block_cache: 0 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: initial_auto_readahead_size: 8192 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T03:59:07.855 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression: NoCompression 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.num_levels: 7 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T03:59:07.856 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T03:59:07.857 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T03:59:07.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T03:59:07.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.table_properties_collectors: 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.inplace_update_support: 0 2023-10-09T03:59:07.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.bloom_locality: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.max_successive_merges: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.ttl: 2592000 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T03:59:07.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.enable_blob_files: false 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.min_blob_size: 0 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T03:59:07.862 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13428, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1abff181-15c2-414e-a468-41b0b1c2d725 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823947500032, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823947503146, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 568677, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13423, "largest_seqno": 14203, "table_properties": {"data_size": 565150, "index_size": 1873, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5575, "raw_average_key_size": 21, "raw_value_size": 559440, "raw_average_value_size": 2193, "num_data_blocks": 84, "num_entries": 255, "num_filter_entries": 255, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696823947, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1abff181-15c2-414e-a468-41b0b1c2d725", "db_session_id": "IRYYMBMVGODT4ST0RA13", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696823947503229, "job": 1, "event": "recovery_finished"} 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T03:59:07.863 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564e9d558000 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: DB pointer 0x564e9d546000 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** DB Stats ** 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T03:59:07.864 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: L0 1/0 555.35 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 336.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: L6 1/0 7.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: Sum 2/0 8.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 336.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 336.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** Compaction Stats [default] ** 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T03:59:07.865 INFO:journalctl@ceph.mon.c.smithi163.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 336.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: Cumulative compaction: 0.00 GB write, 99.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:59:07.866 INFO:journalctl@ceph.mon.c.smithi163.stdout: Interval compaction: 0.00 GB write, 99.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: Block cache BinnedLRUCache@0x564e9d4bd350#7 capacity: 512.00 MB usage: 2.80 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,2.08 KB,0.000396371%) Misc(1,0.00 KB,0%) 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: starting mon.c rank 2 at public addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] at bind addrs [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???) e3 preinit fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).mds e9 new map 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).mds e9 print_map 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: e9 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T03:59:07.867 INFO:journalctl@ceph.mon.c.smithi163.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: legacy client fscid: 1 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: Filesystem 'foo' (1) 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: fs_name foo 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: epoch 8 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: created 2023-10-09T03:54:13.223139+0000 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: modified 2023-10-09T03:54:28.181856+0000 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: tableserver 0 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: root 0 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: session_timeout 60 2023-10-09T03:59:07.868 INFO:journalctl@ceph.mon.c.smithi163.stdout: session_autoclose 300 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: max_file_size 1099511627776 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: required_client_features {} 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: last_failure 0 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: last_failure_osd_epoch 0 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: max_mds 1 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: in 0 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: up {0=24569} 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: failed 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: damaged 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: stopped 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: data_pools [4] 2023-10-09T03:59:07.869 INFO:journalctl@ceph.mon.c.smithi163.stdout: metadata_pool 3 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: inline_data disabled 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: balancer 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: bal_rank_mask -1 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: standby_count_wanted 1 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: [mds.foo.smithi163.crqkdj{0:24569} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: Standby daemons: 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout: [mds.foo.smithi181.sboiwx{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T03:59:07.870 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expand map: {default=false} 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta from 'false' to 'false' 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expanded map: {default=false} 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expand map: {default=info} 2023-10-09T03:59:07.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta from 'info' to 'info' 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expanded map: {default=info} 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expand map: {default=daemon} 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expand map: {default=debug} 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: expand_channel_meta expanded map: {default=debug} 2023-10-09T03:59:07.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3705055922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:07 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/3705055922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:08.611 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T03:59:09.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:59:08 smithi181 conmon[180297]: ts=2023-10-09T03:59:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: mon.c calling monitor election 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: mon.a calling monitor election 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: mgrmap e35: y(active, since 78s), standbys: x 2023-10-09T03:59:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: overall HEALTH_OK 2023-10-09T03:59:09.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:09.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:09.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:09.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:08 smithi181 ceph-mon[105375]: from='client.? 172.21.15.163:0/4138921970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: mon.c calling monitor election 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: mon.a calling monitor election 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: mgrmap e35: y(active, since 78s), standbys: x 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: overall HEALTH_OK 2023-10-09T03:59:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4138921970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: pgmap v42: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: mon.c calling monitor election 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: mon.a calling monitor election 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: monmap e3: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T03:59:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T03:59:09.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: mgrmap e35: y(active, since 78s), standbys: x 2023-10-09T03:59:09.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: overall HEALTH_OK 2023-10-09T03:59:09.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:09.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:09.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:09.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4138921970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T03:59:09.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T03:59:09.794 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:59:09.985 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:09 smithi163 ceph-mon[221067]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:09.985 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:09 smithi163 ceph-mon[213841]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:10.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:09 smithi181 ceph-mon[105375]: pgmap v43: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:11.083 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:59:10] "GET /metrics HTTP/1.1" 200 33865 "" "Prometheus/2.43.0" 2023-10-09T03:59:12.041 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T03:59:12.041 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (2m) - 5m 27.7M - 0.25.0 c8568f914cd2 b04e9e96f8bb 2023-10-09T03:59:12.041 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (2m) - 5m 87.2M - 9.4.7 2c41d148cca3 dd057a75b8e1 2023-10-09T03:59:12.041 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.crqkdj smithi163 running (4m) - 4m 17.8M - 17.2.6-1303-gb2903cb0 5517396c50d4 8037aa8a1d7d 2023-10-09T03:59:12.041 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.sboiwx smithi181 running (4m) - 4m 14.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 2c2246f2ef4b 2023-10-09T03:59:12.041 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (102s) - 9m 447M - 18.2.0-598-gb4fa663f 3060897fdf97 cbeb189f4272 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (2m) - 10m 501M - 18.2.0-598-gb4fa663f 3060897fdf97 6bfa655af17a 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (90s) - 11m 54.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8efafb4ec97d 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (9m) - 9m 78.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 4a4d2d630200 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (4s) - 9m 22.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0f3bd992ec5c 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (2m) - 5m 16.6M - 1.5.0 0da6a335fe13 381d36940ced 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (2m) - 5m 9063k - 1.5.0 0da6a335fe13 79b2ee0a484d 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (9m) - 9m 78.7M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 1d12e7087516 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (8m) - 8m 80.8M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 1b27d5ad7adb 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (8m) - 8m 150M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 e2ce5dc7d22b 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (7m) - 7m 72.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0e8301f836bc 2023-10-09T03:59:12.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (7m) - 7m 78.8M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 b4d4d04d2a3e 2023-10-09T03:59:12.043 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (7m) - 7m 99.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4ed298059fdf 2023-10-09T03:59:12.043 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (6m) - 6m 74.8M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4d54b928e978 2023-10-09T03:59:12.043 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (6m) - 6m 104M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 6ca0d6954129 2023-10-09T03:59:12.043 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (106s) - 6m 50.5M - 2.43.0 a07b618ecd1d 3be07b4832ab 2023-10-09T03:59:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[221067]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[213841]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:12 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:12 smithi181 ceph-mon[105375]: pgmap v44: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:12 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:12 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:12 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:12 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:12.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T03:59:12.788 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:12 smithi163 conmon[200567]: 2023-10-09T03:59:12.512+0000 7f40871b1700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T03:59:13.295 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:59:13.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:13 smithi163 ceph-mon[221067]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:59:13.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:13 smithi163 ceph-mon[213841]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:59:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:13 smithi181 ceph-mon[105375]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:59:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:14 smithi163 ceph-mon[221067]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:14 smithi163 ceph-mon[213841]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:14 smithi181 ceph-mon[105375]: pgmap v45: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:15.530 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:59:16.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:16.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T03:59:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:16 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: pgmap v46: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:16.743 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T03:59:17.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T03:59:17.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T03:59:18.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:18 smithi163 ceph-mon[213841]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:18 smithi163 ceph-mon[221067]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:18 smithi181 ceph-mon[105375]: pgmap v47: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:19.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:59:18 smithi181 conmon[180297]: ts=2023-10-09T03:59:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:59:19.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:19 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:20 smithi181 ceph-mon[105375]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:20 smithi163 ceph-mon[221067]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:20 smithi163 ceph-mon[213841]: pgmap v48: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:59:20] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2023-10-09T03:59:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:21 smithi163 ceph-mon[221067]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T03:59:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:21 smithi163 ceph-mon[213841]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T03:59:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:21 smithi181 ceph-mon[105375]: mgrmap e36: y(active, since 91s), standbys: x 2023-10-09T03:59:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:22 smithi163 ceph-mon[221067]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:22 smithi163 ceph-mon[213841]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:22 smithi181 ceph-mon[105375]: pgmap v49: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:24 smithi163 ceph-mon[221067]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:24 smithi163 ceph-mon[213841]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:24 smithi181 ceph-mon[105375]: pgmap v50: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:26 smithi163 ceph-mon[221067]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:26 smithi163 ceph-mon[213841]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:26 smithi181 ceph-mon[105375]: pgmap v51: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:28.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:28 smithi163 ceph-mon[213841]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:28 smithi163 ceph-mon[221067]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:28.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:28 smithi181 ceph-mon[105375]: pgmap v52: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:29.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:59:28 smithi181 conmon[180297]: ts=2023-10-09T03:59:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:59:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:30 smithi181 ceph-mon[105375]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:30 smithi163 ceph-mon[221067]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:30 smithi163 ceph-mon[213841]: pgmap v53: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:59:30] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2023-10-09T03:59:32.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:32 smithi181 ceph-mon[105375]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:32.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:32 smithi163 ceph-mon[221067]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:32.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:32 smithi163 ceph-mon[213841]: pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T03:59:34.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:34 smithi181 ceph-mon[105375]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:34.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:34 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:34 smithi163 ceph-mon[221067]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:34 smithi163 ceph-mon[213841]: pgmap v55: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:36 smithi181 ceph-mon[105375]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:36 smithi163 ceph-mon[221067]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:36.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:36 smithi163 ceph-mon[213841]: pgmap v56: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:38.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:38 smithi163 ceph-mon[221067]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:38 smithi163 ceph-mon[213841]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:38.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:38 smithi181 ceph-mon[105375]: pgmap v57: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:39.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:59:38 smithi181 conmon[180297]: ts=2023-10-09T03:59:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:59:40.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:40 smithi181 ceph-mon[105375]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:40.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:40 smithi163 ceph-mon[213841]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:40 smithi163 ceph-mon[221067]: pgmap v58: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:41.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:59:40] "GET /metrics HTTP/1.1" 200 33864 "" "Prometheus/2.43.0" 2023-10-09T03:59:42.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:42 smithi181 ceph-mon[105375]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:42 smithi163 ceph-mon[221067]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:42.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:42 smithi163 ceph-mon[213841]: pgmap v59: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:44 smithi181 ceph-mon[105375]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:44.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:44 smithi163 ceph-mon[221067]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:44 smithi163 ceph-mon[213841]: pgmap v60: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:46 smithi181 ceph-mon[105375]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:46 smithi163 ceph-mon[213841]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:46 smithi163 ceph-mon[221067]: pgmap v61: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:48 smithi163 ceph-mon[213841]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:48 smithi163 ceph-mon[221067]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:48 smithi181 ceph-mon[105375]: pgmap v62: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:49.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:59:48 smithi181 conmon[180297]: ts=2023-10-09T03:59:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T03:59:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:49.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:49 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T03:59:50.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:50 smithi163 ceph-mon[221067]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:50 smithi163 ceph-mon[213841]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:50 smithi181 ceph-mon[105375]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 03:59:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:03:59:50] "GET /metrics HTTP/1.1" 200 33860 "" "Prometheus/2.43.0" 2023-10-09T03:59:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:52 smithi163 ceph-mon[221067]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:52 smithi163 ceph-mon[213841]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:52.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:52 smithi181 ceph-mon[105375]: pgmap v64: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:54 smithi163 ceph-mon[213841]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:54 smithi163 ceph-mon[221067]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:54 smithi181 ceph-mon[105375]: pgmap v65: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:56 smithi163 ceph-mon[221067]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:56 smithi163 ceph-mon[213841]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:56 smithi181 ceph-mon[105375]: pgmap v66: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:58.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 03:59:58 smithi163 ceph-mon[213841]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:58.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 03:59:58 smithi163 ceph-mon[221067]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:58.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 03:59:58 smithi181 ceph-mon[105375]: pgmap v67: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T03:59:59.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 03:59:58 smithi181 conmon[180297]: ts=2023-10-09T03:59:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:00:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:00 smithi181 ceph-mon[105375]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:00.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:00 smithi181 ceph-mon[105375]: overall HEALTH_OK 2023-10-09T04:00:00.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:00 smithi163 ceph-mon[213841]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:00.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:00 smithi163 ceph-mon[213841]: overall HEALTH_OK 2023-10-09T04:00:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:00 smithi163 ceph-mon[221067]: pgmap v68: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:00 smithi163 ceph-mon[221067]: overall HEALTH_OK 2023-10-09T04:00:01.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:00:00] "GET /metrics HTTP/1.1" 200 33860 "" "Prometheus/2.43.0" 2023-10-09T04:00:02.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:02 smithi181 ceph-mon[105375]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:02 smithi163 ceph-mon[221067]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:02 smithi163 ceph-mon[213841]: pgmap v69: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:04.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:04 smithi181 ceph-mon[105375]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:04 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:04 smithi163 ceph-mon[221067]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:04 smithi163 ceph-mon[213841]: pgmap v70: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:06 smithi181 ceph-mon[105375]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:06.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:06 smithi163 ceph-mon[221067]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:06.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:06 smithi163 ceph-mon[213841]: pgmap v71: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:08 smithi163 ceph-mon[221067]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:08 smithi163 ceph-mon[213841]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:08 smithi181 ceph-mon[105375]: pgmap v72: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:09.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:08 smithi181 conmon[180297]: ts=2023-10-09T04:00:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:00:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:10 smithi181 ceph-mon[105375]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:10 smithi163 ceph-mon[213841]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:10 smithi163 ceph-mon[221067]: pgmap v73: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:11.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:00:10] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2023-10-09T04:00:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:12 smithi181 ceph-mon[105375]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:12 smithi163 ceph-mon[213841]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:12 smithi163 ceph-mon[221067]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:14 smithi181 ceph-mon[105375]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:14 smithi163 ceph-mon[213841]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:14 smithi163 ceph-mon[221067]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:16 smithi181 ceph-mon[105375]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:16 smithi163 ceph-mon[221067]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:16 smithi163 ceph-mon[213841]: pgmap v76: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:17 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:18 smithi181 ceph-mon[105375]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:18 smithi163 ceph-mon[221067]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:18 smithi163 ceph-mon[213841]: pgmap v77: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:18.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:00:19.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:18 smithi181 conmon[180297]: ts=2023-10-09T04:00:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:00:19.396 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:00:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:19 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[221067]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[213841]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:20 smithi181 ceph-mon[105375]: pgmap v78: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:20 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:20 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:20 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:20 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:20.976 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:00:20] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2023-10-09T04:00:21.417 INFO:teuthology.orchestra.run.smithi163.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:21 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:21 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:21 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:22.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:00:22.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:00:22.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:00:22.334 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: pgmap v79: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:00:22.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:22 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:00:22.403 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:00:23.541 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: Upgrade: It appears safe to stop mon.b 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:23.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: Upgrade: It appears safe to stop mon.b 2023-10-09T04:00:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:23.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: Upgrade: It appears safe to stop mon.b 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:23.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:23 smithi181 ceph-mon[105375]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:23.637 INFO:teuthology.orchestra.run.smithi163.stdout:wait for all mons to upgrade 2023-10-09T04:00:24.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T04:00:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:24 smithi163 ceph-mon[213841]: Upgrade: Updating mon.b 2023-10-09T04:00:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:24 smithi163 ceph-mon[213841]: Deploying daemon mon.b on smithi181 2023-10-09T04:00:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:24 smithi163 ceph-mon[213841]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:24 smithi163 ceph-mon[221067]: Upgrade: Updating mon.b 2023-10-09T04:00:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:24 smithi163 ceph-mon[221067]: Deploying daemon mon.b on smithi181 2023-10-09T04:00:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:24 smithi163 ceph-mon[221067]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:24.583 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:00:24.690 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:24 smithi181 ceph-mon[105375]: Upgrade: Updating mon.b 2023-10-09T04:00:24.691 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:24 smithi181 ceph-mon[105375]: Deploying daemon mon.b on smithi181 2023-10-09T04:00:24.691 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:24 smithi181 ceph-mon[105375]: pgmap v80: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:24.691 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:24 smithi181 systemd[1]: Stopping Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:00:24.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:24 smithi181 conmon[105351]: 2023-10-09T04:00:24.688+0000 7f901bcbd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:00:24.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:24 smithi181 conmon[105351]: 2023-10-09T04:00:24.688+0000 7f901bcbd700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:00:25.737 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:25 smithi181 bash[196564]: 4a4d2d630200392dac4ea0b9e22e30faf7f1ccb21a2e2456db2652f26aef0657 2023-10-09T04:00:25.738 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:25 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T04:00:25.738 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:25 smithi181 systemd[1]: Stopped Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:00:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:25 smithi181 systemd[1]: Starting Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: pidfile_write: ignore empty --pid-file 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 bash[196914]: 48065a0413d3bf0b4e7c3019910ed706495daee9c8f9fa3aacd623bd748de8bd 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: load: jerasure load: lrc 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: RocksDB version: 7.9.2 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Git sha 0 2023-10-09T04:00:26.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: DB SUMMARY 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: DB Session ID: SS5CL2F3TPBMITYY6G68 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1082 Bytes 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 2552395 ; 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:00:26.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.env: 0x55d99f59a720 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.info_log: 0x55d9a237d420 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.statistics: (nil) 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.use_fsync: 0 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:00:26.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.db_log_dir: 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.wal_dir: 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:00:26.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.write_buffer_manager: 0x55d9a160cc80 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:00:26.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.unordered_write: 0 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.row_cache: None 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.wal_filter: None 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:00:26.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.wal_compression: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:00:26.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:00:26.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_open_files: -1 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Compression algorithms supported: 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kZSTD supported: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:00:26.528 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.merge_operator: 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_filter: None 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:00:26.529 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d9a237d520) 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: index_type: 0 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: data_block_index_type: 0 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: index_shortening: 1 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:00:26.530 INFO:journalctl@ceph.mon.b.smithi181.stdout: checksum: 4 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: no_block_cache: 0 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache: 0x55d9a168f1f0 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache_options: 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: capacity : 536870912 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: num_shard_bits : 4 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: strict_capacity_limit : 0 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_cache_compressed: (nil) 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: persistent_cache: (nil) 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_size: 4096 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_size_deviation: 10 2023-10-09T04:00:26.531 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_restart_interval: 16 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: index_block_restart_interval: 1 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: metadata_block_size: 4096 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: partition_filters: 0 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: use_delta_encoding: 1 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: filter_policy: bloomfilter 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: whole_key_filtering: 1 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: verify_compression: 0 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: format_version: 5 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: enable_index_compression: 1 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: block_align: 0 2023-10-09T04:00:26.532 INFO:journalctl@ceph.mon.b.smithi181.stdout: max_auto_readahead_size: 262144 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout: prepopulate_block_cache: 0 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout: initial_auto_readahead_size: 8192 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression: NoCompression 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.num_levels: 7 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:00:26.533 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:00:26.534 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:00:26.535 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:00:26.536 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:00:26.537 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:00:26.538 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.ttl: 2592000 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:00:26.539 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:00:26.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:00:26.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:00:26.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:00:26.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T04:00:26.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13504, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 65c8ac4d-9284-4af0-95ca-64d0c9984012 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824026260764, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 systemd[1]: Started Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824026268712, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1573721, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13499, "largest_seqno": 15674, "table_properties": {"data_size": 1566089, "index_size": 4439, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2181, "raw_key_size": 17354, "raw_average_key_size": 20, "raw_value_size": 1549544, "raw_average_value_size": 1795, "num_data_blocks": 204, "num_entries": 863, "num_filter_entries": 863, "num_deletions": 251, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696824026, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "65c8ac4d-9284-4af0-95ca-64d0c9984012", "db_session_id": "SS5CL2F3TPBMITYY6G68", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824026268806, "job": 1, "event": "recovery_finished"} 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:00:26.541 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d9a172a000 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: DB pointer 0x55d9a1718000 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** DB Stats ** 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:00:26.542 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** Compaction Stats [default] ** 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: L0 1/0 1.50 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 495.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: L6 1/0 7.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: Sum 2/0 9.31 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 495.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 495.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** Compaction Stats [default] ** 2023-10-09T04:00:26.543 INFO:journalctl@ceph.mon.b.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 495.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:00:26.544 INFO:journalctl@ceph.mon.b.smithi181.stdout: Cumulative compaction: 0.00 GB write, 136.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: Interval compaction: 0.00 GB write, 136.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: Block cache BinnedLRUCache@0x55d9a168f1f0#7 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: starting mon.b rank 1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???) e3 preinit fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).mds e9 new map 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).mds e9 print_map 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: e9 2023-10-09T04:00:26.545 INFO:journalctl@ceph.mon.b.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: legacy client fscid: 1 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: Filesystem 'foo' (1) 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: fs_name foo 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: epoch 8 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: created 2023-10-09T03:54:13.223139+0000 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: modified 2023-10-09T03:54:28.181856+0000 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: tableserver 0 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: root 0 2023-10-09T04:00:26.546 INFO:journalctl@ceph.mon.b.smithi181.stdout: session_timeout 60 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: session_autoclose 300 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: max_file_size 1099511627776 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: required_client_features {} 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: last_failure 0 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: last_failure_osd_epoch 0 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: max_mds 1 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: in 0 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: up {0=24569} 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: failed 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: damaged 2023-10-09T04:00:26.547 INFO:journalctl@ceph.mon.b.smithi181.stdout: stopped 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: data_pools [4] 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: metadata_pool 3 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: inline_data disabled 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: balancer 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: bal_rank_mask -1 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: standby_count_wanted 1 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: [mds.foo.smithi163.crqkdj{0:24569} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.163:6834/3616402083,v1:172.21.15.163:6835/3616402083] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: Standby daemons: 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: 2023-10-09T04:00:26.548 INFO:journalctl@ceph.mon.b.smithi181.stdout: [mds.foo.smithi181.sboiwx{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.181:6832/3363379780,v1:172.21.15.181:6833/3363379780] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expand map: {default=false} 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta from 'false' to 'false' 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expanded map: {default=false} 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expand map: {default=info} 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta from 'info' to 'info' 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expanded map: {default=info} 2023-10-09T04:00:26.549 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expand map: {default=daemon} 2023-10-09T04:00:26.550 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T04:00:26.550 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T04:00:26.550 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expand map: {default=debug} 2023-10-09T04:00:26.550 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T04:00:26.550 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:26 smithi181 ceph-mon[196979]: expand_channel_meta expanded map: {default=debug} 2023-10-09T04:00:29.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:28 smithi181 conmon[180297]: ts=2023-10-09T04:00:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:00:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:00:30] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2023-10-09T04:00:31.581 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:31 smithi163 conmon[200567]: 2023-10-09T04:00:31.279+0000 7f40871b1700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T04:00:32.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: overall HEALTH_OK 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: mon.a calling monitor election 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: mon.b calling monitor election 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: mon.c (rank 2) addr [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] is down (out of quorum) 2023-10-09T04:00:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:32.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: overall HEALTH_OK 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: mon.a calling monitor election 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: mon.b calling monitor election 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:00:32.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: mon.c (rank 2) addr [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] is down (out of quorum) 2023-10-09T04:00:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:32.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:32.274 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:00:32.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:00:33.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:32 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:32 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4073471970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:00:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:32 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:32 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4073471970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:00:33.293 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:00:34.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:33 smithi181 ceph-mon[196979]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:34.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:33 smithi163 ceph-mon[213841]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (3m) - 7m 27.7M - 0.25.0 c8568f914cd2 b04e9e96f8bb 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (3m) - 6m 93.9M - 9.4.7 2c41d148cca3 dd057a75b8e1 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.crqkdj smithi163 running (6m) - 6m 18.1M - 17.2.6-1303-gb2903cb0 5517396c50d4 8037aa8a1d7d 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.sboiwx smithi181 running (6m) - 6m 15.5M - 17.2.6-1303-gb2903cb0 5517396c50d4 2c2246f2ef4b 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (3m) - 10m 448M - 18.2.0-598-gb4fa663f 3060897fdf97 cbeb189f4272 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (3m) - 12m 504M - 18.2.0-598-gb4fa663f 3060897fdf97 6bfa655af17a 2023-10-09T04:00:34.885 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (2m) - 12m 61.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8efafb4ec97d 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (8s) - 11m 27.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 48065a0413d3 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (87s) - 11m 37.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0f3bd992ec5c 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (3m) - 6m 16.9M - 1.5.0 0da6a335fe13 381d36940ced 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (3m) - 6m 18.1M - 1.5.0 0da6a335fe13 79b2ee0a484d 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (10m) - 10m 81.8M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 1d12e7087516 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (10m) - 10m 83.7M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 1b27d5ad7adb 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (9m) - 9m 152M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 e2ce5dc7d22b 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (9m) - 9m 75.6M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0e8301f836bc 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (9m) - 9m 83.2M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 b4d4d04d2a3e 2023-10-09T04:00:34.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (8m) - 8m 103M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4ed298059fdf 2023-10-09T04:00:34.887 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (8m) - 8m 78.3M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4d54b928e978 2023-10-09T04:00:34.887 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (7m) - 7m 108M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 6ca0d6954129 2023-10-09T04:00:34.887 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (3m) - 7m 60.0M - 2.43.0 a07b618ecd1d 3be07b4832ab 2023-10-09T04:00:35.500 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: overall HEALTH_OK 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: mon.a calling monitor election 2023-10-09T04:00:35.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: mon.b calling monitor election 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: pgmap v82: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: pgmap v83: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: pgmap v84: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T04:00:35.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: mon.c (rank 2) addr [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0] is down (out of quorum) 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4073471970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: pgmap v85: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:35.503 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.504 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:00:35.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:35.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:35.852 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: mon.c calling monitor election 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: mon.a calling monitor election 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: mon.b calling monitor election 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T04:00:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: overall HEALTH_OK 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: mon.c calling monitor election 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:36.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: mon.a calling monitor election 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: mon.b calling monitor election 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: overall HEALTH_OK 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:36.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: mon.c calling monitor election 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: pgmap v86: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: mon.a calling monitor election 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: mon.b calling monitor election 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: monmap e4: 3 mons at {a=[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0],b=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],c=[v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]} removed_ranks: {} 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: fsmap foo:1 {0=foo.smithi163.crqkdj=up:active} 1 up:standby 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: mgrmap e36: y(active, since 2m), standbys: x 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:00:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: overall HEALTH_OK 2023-10-09T04:00:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:36.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:37.437 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:38.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:00:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:38 smithi181 ceph-mon[196979]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:38.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:38 smithi163 ceph-mon[213841]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:38.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:38 smithi163 ceph-mon[221067]: pgmap v87: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:38.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:38.434 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:00:38.768 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:38 smithi181 conmon[180297]: ts=2023-10-09T04:00:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:39.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:40 smithi163 ceph-mon[221067]: Reconfiguring mon.a (monmap changed)... 2023-10-09T04:00:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:40 smithi163 ceph-mon[221067]: Reconfiguring daemon mon.a on smithi163 2023-10-09T04:00:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:40 smithi163 ceph-mon[221067]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:40 smithi163 ceph-mon[213841]: Reconfiguring mon.a (monmap changed)... 2023-10-09T04:00:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:40 smithi163 ceph-mon[213841]: Reconfiguring daemon mon.a on smithi163 2023-10-09T04:00:40.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:40 smithi163 ceph-mon[213841]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:40 smithi181 ceph-mon[196979]: Reconfiguring mon.a (monmap changed)... 2023-10-09T04:00:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:40 smithi181 ceph-mon[196979]: Reconfiguring daemon mon.a on smithi163 2023-10-09T04:00:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:40 smithi181 ceph-mon[196979]: pgmap v88: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:40.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:00:40] "GET /metrics HTTP/1.1" 200 33934 "" "Prometheus/2.43.0" 2023-10-09T04:00:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:00:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:00:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: Reconfiguring daemon mgr.y on smithi163 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[213841]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:00:41.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:00:41.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:41.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: Reconfiguring daemon mgr.y on smithi163 2023-10-09T04:00:41.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:41 smithi163 ceph-mon[221067]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: Reconfiguring daemon mgr.y on smithi163 2023-10-09T04:00:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:41 smithi181 ceph-mon[196979]: pgmap v89: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[221067]: Reconfiguring daemon mon.c on smithi163 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:00:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:42.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:42.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:42.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:42 smithi163 ceph-mon[213841]: Reconfiguring daemon mon.c on smithi163 2023-10-09T04:00:43.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:00:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:43.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:43.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:42 smithi181 ceph-mon[196979]: Reconfiguring daemon mon.c on smithi163 2023-10-09T04:00:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:00:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:00:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.0 on smithi163 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[213841]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.0 on smithi163 2023-10-09T04:00:44.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:43 smithi163 ceph-mon[221067]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:44.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:00:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:00:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.0 on smithi163 2023-10-09T04:00:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:43 smithi181 ceph-mon[196979]: pgmap v90: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[213841]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:00:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:00:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.1 on smithi163 2023-10-09T04:00:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[221067]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:00:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:00:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:45.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:45 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.1 on smithi163 2023-10-09T04:00:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:45 smithi181 ceph-mon[196979]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:00:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:00:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:45 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.1 on smithi163 2023-10-09T04:00:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:46 smithi181 ceph-mon[196979]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:00:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[213841]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:00:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[221067]: pgmap v91: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:46.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:46.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:46.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:00:46.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:47 smithi181 ceph-mon[196979]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:00:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:47 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.2 on smithi163 2023-10-09T04:00:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:00:47.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:47.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[221067]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:00:47.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.2 on smithi163 2023-10-09T04:00:47.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:47.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:47.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[213841]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.2 on smithi163 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:00:47.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.3 on smithi163 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:00:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:48.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:00:48.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.3 on smithi163 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.3 on smithi163 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: pgmap v92: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:48.526 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:48.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:48.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi163.crqkdj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:00:48.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:49.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:48 smithi181 conmon[180297]: ts=2023-10-09T04:00:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:00:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:49 smithi181 ceph-mon[196979]: Reconfiguring mds.foo.smithi163.crqkdj (monmap changed)... 2023-10-09T04:00:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:49 smithi181 ceph-mon[196979]: Reconfiguring daemon mds.foo.smithi163.crqkdj on smithi163 2023-10-09T04:00:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[213841]: Reconfiguring mds.foo.smithi163.crqkdj (monmap changed)... 2023-10-09T04:00:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[213841]: Reconfiguring daemon mds.foo.smithi163.crqkdj on smithi163 2023-10-09T04:00:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[221067]: Reconfiguring mds.foo.smithi163.crqkdj (monmap changed)... 2023-10-09T04:00:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[221067]: Reconfiguring daemon mds.foo.smithi163.crqkdj on smithi163 2023-10-09T04:00:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:00:50.489 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:50 smithi181 conmon[180297]: ts=2023-10-09T04:00:50.086Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", public_addr=\"172.21.15.163\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.163\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-09T04:00:50.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:50.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:50.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:50.489 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:00:50.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:50.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:50.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:50.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:50 smithi181 ceph-mon[196979]: Reconfiguring daemon mon.b on smithi181 2023-10-09T04:00:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[213841]: Reconfiguring daemon mon.b on smithi181 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: pgmap v93: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:00:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:00:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:00:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:50 smithi163 ceph-mon[221067]: Reconfiguring daemon mon.b on smithi181 2023-10-09T04:00:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:00:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:00:50] "GET /metrics HTTP/1.1" 200 33933 "" "Prometheus/2.43.0" 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: Reconfiguring daemon mgr.x on smithi181 2023-10-09T04:00:51.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:51 smithi181 ceph-mon[196979]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: Reconfiguring daemon mgr.x on smithi181 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[221067]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:00:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:00:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:00:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: Reconfiguring daemon mgr.x on smithi181 2023-10-09T04:00:51.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:51 smithi163 ceph-mon[213841]: pgmap v94: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:53.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:52 smithi181 ceph-mon[196979]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:00:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:00:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:52 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.4 on smithi181 2023-10-09T04:00:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[221067]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:00:53.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.4 on smithi181 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[213841]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:53.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:52 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.4 on smithi181 2023-10-09T04:00:54.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:00:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:00:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.5 on smithi181 2023-10-09T04:00:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:53 smithi181 ceph-mon[196979]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.5 on smithi181 2023-10-09T04:00:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[221067]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.5 on smithi181 2023-10-09T04:00:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:53 smithi163 ceph-mon[213841]: pgmap v95: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[221067]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:00:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:00:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.6 on smithi181 2023-10-09T04:00:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[213841]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:00:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:00:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:55 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.6 on smithi181 2023-10-09T04:00:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:55 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:55 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:55.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:55 smithi181 ceph-mon[196979]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:00:55.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:55 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:00:55.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:55 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:55.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:55 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.6 on smithi181 2023-10-09T04:00:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:56 smithi181 ceph-mon[196979]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:00:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[221067]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:00:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[213841]: pgmap v96: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:00:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:57.503 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:57 smithi181 ceph-mon[196979]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:00:57.503 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:57 smithi181 ceph-mon[196979]: Reconfiguring daemon osd.7 on smithi181 2023-10-09T04:00:57.504 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:57.504 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:57.504 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:00:57.504 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:57.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[221067]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:00:57.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[221067]: Reconfiguring daemon osd.7 on smithi181 2023-10-09T04:00:57.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:57.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:57.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:00:57.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[213841]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:00:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[213841]: Reconfiguring daemon osd.7 on smithi181 2023-10-09T04:00:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.sboiwx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:00:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: Reconfiguring mds.foo.smithi181.sboiwx (monmap changed)... 2023-10-09T04:00:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: Reconfiguring daemon mds.foo.smithi181.sboiwx on smithi181 2023-10-09T04:00:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: Reconfiguring mds.foo.smithi181.sboiwx (monmap changed)... 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: Reconfiguring daemon mds.foo.smithi181.sboiwx on smithi181 2023-10-09T04:00:58.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:00:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: Reconfiguring mds.foo.smithi181.sboiwx (monmap changed)... 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: Reconfiguring daemon mds.foo.smithi181.sboiwx on smithi181 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: pgmap v97: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:58.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:00:58.663 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:00:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:00:59.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:00:58 smithi181 conmon[180297]: ts=2023-10-09T04:00:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:01:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:00 smithi163 ceph-mon[221067]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:00.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:00 smithi163 ceph-mon[213841]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:00.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:00 smithi181 ceph-mon[196979]: pgmap v98: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:01.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:01:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:01:00] "GET /metrics HTTP/1.1" 200 33933 "" "Prometheus/2.43.0" 2023-10-09T04:01:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:02 smithi163 ceph-mon[213841]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:02 smithi163 ceph-mon[221067]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:02.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:02 smithi181 ceph-mon[196979]: pgmap v99: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:04 smithi163 ceph-mon[213841]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:04 smithi163 ceph-mon[221067]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:04 smithi181 ceph-mon[196979]: pgmap v100: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:06.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:06 smithi163 ceph-mon[221067]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:06.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:06 smithi163 ceph-mon[213841]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:06.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:06 smithi181 ceph-mon[196979]: pgmap v101: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:08 smithi163 ceph-mon[213841]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:08 smithi163 ceph-mon[221067]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:08 smithi181 ceph-mon[196979]: pgmap v102: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:09.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:01:08 smithi181 conmon[180297]: ts=2023-10-09T04:01:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:01:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:10 smithi163 ceph-mon[221067]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:10 smithi163 ceph-mon[213841]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:10.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:10 smithi181 ceph-mon[196979]: pgmap v103: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:11.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:01:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:01:10] "GET /metrics HTTP/1.1" 200 33933 "" "Prometheus/2.43.0" 2023-10-09T04:01:12.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:12 smithi163 ceph-mon[221067]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:12.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:12 smithi163 ceph-mon[213841]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:12 smithi181 ceph-mon[196979]: pgmap v104: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:14 smithi163 ceph-mon[213841]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:14 smithi163 ceph-mon[221067]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:14 smithi181 ceph-mon[196979]: pgmap v105: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:16 smithi163 ceph-mon[221067]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:16 smithi163 ceph-mon[213841]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:16.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:16 smithi181 ceph-mon[196979]: pgmap v106: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:18.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:18 smithi163 ceph-mon[221067]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:18.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:18 smithi163 ceph-mon[213841]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:18 smithi181 ceph-mon[196979]: pgmap v107: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:19.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:01:18 smithi181 conmon[180297]: ts=2023-10-09T04:01:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:01:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:20 smithi163 ceph-mon[221067]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:20 smithi163 ceph-mon[213841]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:20 smithi181 ceph-mon[196979]: pgmap v108: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:01:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:01:20] "GET /metrics HTTP/1.1" 200 33929 "" "Prometheus/2.43.0" 2023-10-09T04:01:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:22 smithi163 ceph-mon[213841]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:22 smithi163 ceph-mon[221067]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:22 smithi181 ceph-mon[196979]: pgmap v109: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:24.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:24 smithi163 ceph-mon[221067]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:24 smithi163 ceph-mon[213841]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:24 smithi181 ceph-mon[196979]: pgmap v110: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:26 smithi163 ceph-mon[221067]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:26 smithi163 ceph-mon[213841]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:26 smithi181 ceph-mon[196979]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:28 smithi163 ceph-mon[221067]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:28.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:28 smithi163 ceph-mon[213841]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:28.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:28 smithi181 ceph-mon[196979]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:29.018 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:01:28 smithi181 conmon[180297]: ts=2023-10-09T04:01:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:01:30.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:30 smithi163 ceph-mon[213841]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:30 smithi163 ceph-mon[221067]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:30 smithi181 ceph-mon[196979]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:01:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:01:30] "GET /metrics HTTP/1.1" 200 33929 "" "Prometheus/2.43.0" 2023-10-09T04:01:32.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:32 smithi163 ceph-mon[213841]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:32 smithi163 ceph-mon[221067]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:32 smithi181 ceph-mon[196979]: pgmap v114: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:34 smithi163 ceph-mon[221067]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:34 smithi163 ceph-mon[213841]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:34 smithi181 ceph-mon[196979]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:36 smithi163 ceph-mon[221067]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:36.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:36 smithi163 ceph-mon[213841]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:36.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:36 smithi181 ceph-mon[196979]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:38.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:38 smithi163 ceph-mon[221067]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:38 smithi163 ceph-mon[213841]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:38.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:38 smithi181 ceph-mon[196979]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:39.019 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:01:38 smithi181 conmon[180297]: ts=2023-10-09T04:01:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi181.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.163\", device_class=\"ssd\", hostname=\"smithi163\", instance=\"smithi163:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.163\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:01:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:40 smithi163 ceph-mon[221067]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:40.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:40 smithi163 ceph-mon[213841]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:40 smithi181 ceph-mon[196979]: pgmap v118: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:40.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:01:41.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:01:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:01:40] "GET /metrics HTTP/1.1" 200 33928 "" "Prometheus/2.43.0" 2023-10-09T04:01:41.185 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:01:42.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:42 smithi163 ceph-mon[221067]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:42.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:42 smithi163 ceph-mon[213841]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:42 smithi181 ceph-mon[196979]: pgmap v119: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:42.839 INFO:teuthology.orchestra.run.smithi163.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:01:43.450 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T04:01:44.041 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:01:44.233 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[221067]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[221067]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[221067]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[213841]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[213841]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:44.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:43 smithi163 ceph-mon[213841]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:44.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:43 smithi181 ceph-mon[196979]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:43 smithi181 ceph-mon[196979]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:01:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:43 smithi181 ceph-mon[196979]: pgmap v120: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:45.583 INFO:teuthology.orchestra.run.smithi163.stdout:wait for half of osds to upgrade 2023-10-09T04:01:46.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T04:01:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:46 smithi163 ceph-mon[221067]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:46 smithi163 ceph-mon[213841]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:46.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:46.737 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:01:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:46 smithi181 ceph-mon[196979]: pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:48.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:48.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:01:48.363 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:01:48.364 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:48.365 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:01:48.366 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:01:48.367 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:01:48.367 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:01:48.367 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.367 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.367 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:01:48.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:48.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:01:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:49 smithi181 ceph-mon[196979]: Upgrade: Setting container_image for all mon 2023-10-09T04:01:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:49 smithi181 ceph-mon[196979]: Upgrade: Setting container_image for all crash 2023-10-09T04:01:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:49 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:01:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:49 smithi181 ceph-mon[196979]: Upgrade: osd.0 is safe to restart 2023-10-09T04:01:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/910225734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[221067]: Upgrade: Setting container_image for all mon 2023-10-09T04:01:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[221067]: Upgrade: Setting container_image for all crash 2023-10-09T04:01:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:01:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[221067]: Upgrade: osd.0 is safe to restart 2023-10-09T04:01:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/910225734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[213841]: Upgrade: Setting container_image for all mon 2023-10-09T04:01:49.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[213841]: Upgrade: Setting container_image for all crash 2023-10-09T04:01:49.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:01:49.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[213841]: Upgrade: osd.0 is safe to restart 2023-10-09T04:01:49.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/910225734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:50.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[213841]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:50.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:50.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:50.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:01:50.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:50 smithi181 ceph-mon[196979]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:01:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[221067]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:01:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:01:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:01:51.055 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:01:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:01:50] "GET /metrics HTTP/1.1" 200 33929 "" "Prometheus/2.43.0" 2023-10-09T04:01:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:51 smithi163 ceph-mon[221067]: Upgrade: Updating osd.0 2023-10-09T04:01:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:51 smithi163 ceph-mon[221067]: Deploying daemon osd.0 on smithi163 2023-10-09T04:01:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:51 smithi163 ceph-mon[213841]: Upgrade: Updating osd.0 2023-10-09T04:01:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:51 smithi163 ceph-mon[213841]: Deploying daemon osd.0 on smithi163 2023-10-09T04:01:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:51 smithi181 ceph-mon[196979]: Upgrade: Updating osd.0 2023-10-09T04:01:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:51 smithi181 ceph-mon[196979]: Deploying daemon osd.0 on smithi163 2023-10-09T04:01:52.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:52 smithi163 ceph-mon[221067]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:52.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:52 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3784491134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:52.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:52 smithi163 ceph-mon[213841]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:52.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:52 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3784491134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:52 smithi181 ceph-mon[196979]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:52 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3784491134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:52.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:52 smithi163 systemd[1]: Stopping Ceph osd.0 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:01:52.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:52 smithi163 conmon[125110]: 2023-10-09T04:01:52.529+0000 7fda1953a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:01:52.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:52 smithi163 conmon[125110]: 2023-10-09T04:01:52.529+0000 7fda1953a700 -1 osd.0 80 *** Got signal Terminated *** 2023-10-09T04:01:52.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:52 smithi163 conmon[125110]: 2023-10-09T04:01:52.529+0000 7fda1953a700 -1 osd.0 80 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:01:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:53 smithi181 ceph-mon[196979]: osd.0 marked itself down and dead 2023-10-09T04:01:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:53 smithi163 ceph-mon[213841]: osd.0 marked itself down and dead 2023-10-09T04:01:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:53 smithi163 ceph-mon[221067]: osd.0 marked itself down and dead 2023-10-09T04:01:54.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:53 smithi163 bash[236730]: 1d12e7087516b3be643787048cff48f66fb948eb7e106d81cfe30ef6b25f30ea 2023-10-09T04:01:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:54 smithi181 ceph-mon[196979]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:01:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:54 smithi181 ceph-mon[196979]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:54 smithi181 ceph-mon[196979]: failed to encode map e81 with expected crc 2023-10-09T04:01:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:54 smithi181 ceph-mon[196979]: failed to encode map e81 with expected crc 2023-10-09T04:01:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:54 smithi181 ceph-mon[196979]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T04:01:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/941072678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[213841]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[213841]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[213841]: failed to encode map e81 with expected crc 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[213841]: failed to encode map e81 with expected crc 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[213841]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/941072678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[221067]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:01:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[221067]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:01:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[221067]: failed to encode map e81 with expected crc 2023-10-09T04:01:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[221067]: failed to encode map e81 with expected crc 2023-10-09T04:01:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[221067]: osdmap e81: 8 total, 7 up, 8 in 2023-10-09T04:01:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/941072678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:55.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:55 smithi181 ceph-mon[196979]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[213841]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: osdmap e82: 8 total, 7 up, 8 in 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:55.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:55 smithi163 ceph-mon[221067]: failed to encode map e82 with expected crc 2023-10-09T04:01:56.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:55 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T04:01:56.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:55 smithi163 systemd[1]: Stopped Ceph osd.0 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:01:56.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:55 smithi163 systemd[1]: Starting Ceph osd.0 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:01:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:56 smithi181 ceph-mon[196979]: pgmap v128: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:01:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:56 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:01:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3488058046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:56 smithi163 ceph-mon[213841]: pgmap v128: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:01:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:56 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:01:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3488058046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:56 smithi163 ceph-mon[221067]: pgmap v128: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:01:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:56 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:01:56.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3488058046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:57.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:01:57.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:01:57.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T04:01:57.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:01:57.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:01:57.082 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T04:01:57.083 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:01:57.083 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:56 smithi163 bash[237422]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T04:01:58.068 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:57 smithi163 bash[237848]: 663e3d75cc4756b9055330f64b604d7837cbb1f244e2e1aa10fb6e49206a88ef 2023-10-09T04:01:58.068 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:57 smithi163 ceph-osd[237913]: -- 172.21.15.163:0/751133422 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ed80daa4e0 con 0x55ed80d7cc00 2023-10-09T04:01:58.068 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:57 smithi163 systemd[1]: Started Ceph osd.0 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:01:58.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[221067]: pgmap v129: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:01:58.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:58.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:58.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[213841]: pgmap v129: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:01:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:58.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:58 smithi181 ceph-mon[196979]: pgmap v129: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:01:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:01:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:58.832 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:01:58 smithi163 conmon[237889]: 2023-10-09T04:01:58.402+0000 7fa7abe08640 -1 Falling back to public interface 2023-10-09T04:01:59.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:01:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1323789422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:01:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1323789422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:01:59.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:01:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1323789422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:00.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:00 smithi163 ceph-mon[221067]: pgmap v130: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:00.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:00 smithi163 ceph-mon[213841]: pgmap v130: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:00 smithi181 ceph-mon[196979]: pgmap v130: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:01.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:02:00] "GET /metrics HTTP/1.1" 200 33929 "" "Prometheus/2.43.0" 2023-10-09T04:02:01.556 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:02:01 smithi163 conmon[237889]: 2023-10-09T04:02:01.182+0000 7fa7abe08640 -1 osd.0 80 log_to_monitors true 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3416673479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.821 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3416673479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:01 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:01.823 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:02:01 smithi163 conmon[237889]: 2023-10-09T04:02:01.588+0000 7fa79ba71700 -1 osd.0 80 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:02:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3416673479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:02:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: pgmap v131: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:01 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:02.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[213841]: failed to encode map e83 with expected crc 2023-10-09T04:02:02.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[213841]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:02:02.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[213841]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[213841]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[213841]: failed to encode map e83 with expected crc 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[221067]: failed to encode map e83 with expected crc 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[221067]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[221067]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[221067]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:02:02.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:02 smithi163 ceph-mon[221067]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:02 smithi181 ceph-mon[196979]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:02 smithi181 ceph-mon[196979]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:02:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:02 smithi181 ceph-mon[196979]: osdmap e83: 8 total, 7 up, 8 in 2023-10-09T04:02:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:02 smithi181 ceph-mon[196979]: from='osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:02:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:02 smithi181 ceph-mon[196979]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:02:03.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: failed to encode map e84 with expected crc 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252] boot 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2908843047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: pgmap v134: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[221067]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T04:02:03.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: failed to encode map e84 with expected crc 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: failed to encode map e83 with expected crc 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252] boot 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2908843047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: pgmap v134: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:03.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:03 smithi163 ceph-mon[213841]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T04:02:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: failed to encode map e83 with expected crc 2023-10-09T04:02:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:02:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: failed to encode map e84 with expected crc 2023-10-09T04:02:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: failed to encode map e83 with expected crc 2023-10-09T04:02:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: osd.0 [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252] boot 2023-10-09T04:02:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:02:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:02:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2908843047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: pgmap v134: 73 pgs: 7 active+undersized+degraded, 19 active+undersized, 47 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-09T04:02:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:03 smithi181 ceph-mon[196979]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T04:02:04.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:04 smithi163 conmon[200567]: 2023-10-09T04:02:04.772+0000 7f40871b1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:02:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: failed to encode map e85 with expected crc 2023-10-09T04:02:04.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:04.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: failed to encode map e85 with expected crc 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:06.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:02:06.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:06.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3611092566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:06.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3611092566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3611092566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:07.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:06 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:02:07.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:06 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:02:07.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:06 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:02:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:08 smithi163 ceph-mon[213841]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1903039240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:08 smithi163 ceph-mon[221067]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1903039240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:08 smithi181 ceph-mon[196979]: pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:08.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1903039240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:09.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:09 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T04:02:09.073 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:09 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:02:09.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:09 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T04:02:09.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:09 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:02:09.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:09 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded) 2023-10-09T04:02:09.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:09 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:02:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:10 smithi163 ceph-mon[221067]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/313549283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:10 smithi163 ceph-mon[213841]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/313549283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:10 smithi181 ceph-mon[196979]: pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/313549283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:11.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:02:10] "GET /metrics HTTP/1.1" 200 33847 "" "Prometheus/2.43.0" 2023-10-09T04:02:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:12 smithi163 ceph-mon[221067]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1621055077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:12 smithi163 ceph-mon[213841]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1621055077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:12 smithi181 ceph-mon[196979]: pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1621055077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:14.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:14 smithi163 ceph-mon[213841]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:14 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:14 smithi163 ceph-mon[221067]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:14 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:14 smithi181 ceph-mon[196979]: pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:14 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/523109666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/523109666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/523109666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:16 smithi163 ceph-mon[221067]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:16 smithi163 ceph-mon[213841]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:16.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:16 smithi181 ceph-mon[196979]: pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3269824793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3269824793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:17.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3269824793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:18 smithi163 ceph-mon[213841]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:18 smithi163 ceph-mon[221067]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:18.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:18 smithi181 ceph-mon[196979]: pgmap v142: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1902433886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1902433886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1902433886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:20.531 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:20 smithi163 ceph-mon[221067]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:20.531 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:20.531 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:20 smithi163 ceph-mon[213841]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:20.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:20 smithi181 ceph-mon[196979]: pgmap v143: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:20.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:02:20] "GET /metrics HTTP/1.1" 200 33995 "" "Prometheus/2.43.0" 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[221067]: Upgrade: osd.1 is safe to restart 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[213841]: Upgrade: osd.1 is safe to restart 2023-10-09T04:02:21.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:21.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:02:21.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:21 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:02:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:21 smithi181 ceph-mon[196979]: Upgrade: osd.1 is safe to restart 2023-10-09T04:02:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:02:21.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[221067]: Upgrade: Updating osd.1 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[221067]: Deploying daemon osd.1 on smithi163 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[221067]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2527416494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[213841]: Upgrade: Updating osd.1 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[213841]: Deploying daemon osd.1 on smithi163 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[213841]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2527416494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:22 smithi181 ceph-mon[196979]: Upgrade: Updating osd.1 2023-10-09T04:02:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:22 smithi181 ceph-mon[196979]: Deploying daemon osd.1 on smithi163 2023-10-09T04:02:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:22 smithi181 ceph-mon[196979]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2527416494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:24.319 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:23 smithi163 systemd[1]: Stopping Ceph osd.1 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:24.319 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:24 smithi163 conmon[134840]: 2023-10-09T04:02:24.126+0000 7f19aa73c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:02:24.319 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:24 smithi163 conmon[134840]: 2023-10-09T04:02:24.126+0000 7f19aa73c700 -1 osd.1 85 *** Got signal Terminated *** 2023-10-09T04:02:24.319 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:24 smithi163 conmon[134840]: 2023-10-09T04:02:24.126+0000 7f19aa73c700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:02:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:24 smithi163 ceph-mon[213841]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1484978254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:24 smithi163 ceph-mon[213841]: osd.1 marked itself down and dead 2023-10-09T04:02:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:24 smithi163 ceph-mon[221067]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1484978254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:24 smithi163 ceph-mon[221067]: osd.1 marked itself down and dead 2023-10-09T04:02:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:24 smithi181 ceph-mon[196979]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1484978254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:24 smithi181 ceph-mon[196979]: osd.1 marked itself down and dead 2023-10-09T04:02:25.582 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:25 smithi163 bash[245135]: 1b27d5ad7adbb690242a649058eebd0058f2125784148704e8528c745b4fcf81 2023-10-09T04:02:25.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[221067]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:02:25.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[221067]: failed to encode map e86 with expected crc 2023-10-09T04:02:25.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[221067]: failed to encode map e86 with expected crc 2023-10-09T04:02:25.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[221067]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:02:25.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[213841]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:02:25.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[213841]: failed to encode map e86 with expected crc 2023-10-09T04:02:25.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[213841]: failed to encode map e86 with expected crc 2023-10-09T04:02:25.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:25 smithi163 ceph-mon[213841]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:02:25.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:25 smithi181 ceph-mon[196979]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:02:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:25 smithi181 ceph-mon[196979]: failed to encode map e86 with expected crc 2023-10-09T04:02:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:25 smithi181 ceph-mon[196979]: failed to encode map e86 with expected crc 2023-10-09T04:02:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:25 smithi181 ceph-mon[196979]: osdmap e86: 8 total, 7 up, 8 in 2023-10-09T04:02:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: pgmap v147: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1151898813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: pgmap v147: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1151898813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: pgmap v147: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: osdmap e87: 8 total, 7 up, 8 in 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:26.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1151898813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:27.323 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:26 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T04:02:27.323 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:26 smithi163 systemd[1]: Stopped Ceph osd.1 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:27.323 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 systemd[1]: Starting Ceph osd.1 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:27 smithi163 ceph-mon[221067]: failed to encode map e87 with expected crc 2023-10-09T04:02:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:27 smithi163 ceph-mon[213841]: failed to encode map e87 with expected crc 2023-10-09T04:02:27.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:27 smithi181 ceph-mon[196979]: failed to encode map e87 with expected crc 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:02:28.324 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:27 smithi163 bash[245791]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T04:02:28.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:28 smithi163 ceph-mon[221067]: pgmap v149: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:28.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:28 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:28 smithi163 ceph-mon[213841]: pgmap v149: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:28 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:28 smithi181 ceph-mon[196979]: pgmap v149: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:28 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:29.326 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:29 smithi163 bash[246237]: e0fb1e6b8d2362088024bfe435f4cb76ac239c43926590d37e136557d9f5a2a8 2023-10-09T04:02:29.326 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-osd[246303]: -- 172.21.15.163:0/2424914401 <== mon.2 v2:172.21.15.163:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560dcd2504e0 con 0x560dcd222c00 2023-10-09T04:02:29.326 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:29 smithi163 systemd[1]: Started Ceph osd.1 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:29.581 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:29 smithi163 conmon[246279]: 2023-10-09T04:02:29.519+0000 7f429d759640 -1 Falling back to public interface 2023-10-09T04:02:29.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/473259369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:29.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:29.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:29.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:29.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/473259369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:29.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:29.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:29.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:29 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:29.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/473259369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:29.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:29 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:29.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:29 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:29.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:29 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:30 smithi163 ceph-mon[213841]: pgmap v150: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:30 smithi163 ceph-mon[221067]: pgmap v150: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:30 smithi181 ceph-mon[196979]: pgmap v150: 73 pgs: 1 active+undersized+degraded, 28 peering, 3 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:02:30] "GET /metrics HTTP/1.1" 200 33995 "" "Prometheus/2.43.0" 2023-10-09T04:02:31.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2333786500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:31.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2333786500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2333786500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:32.582 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:32 smithi163 conmon[246279]: 2023-10-09T04:02:32.258+0000 7f429d759640 -1 osd.1 85 log_to_monitors true 2023-10-09T04:02:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[221067]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2023-10-09T04:02:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[221067]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:02:32.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[213841]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:32.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2023-10-09T04:02:32.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:32.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:32.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:32 smithi163 ceph-mon[213841]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:02:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:32 smithi181 ceph-mon[196979]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:32 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2023-10-09T04:02:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:32 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:32 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:32 smithi181 ceph-mon[196979]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:02:33.176 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:02:33 smithi163 conmon[246279]: 2023-10-09T04:02:33.092+0000 7f428abbd700 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:02:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:02:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T04:02:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:02:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2537422346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[213841]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2537422346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:33 smithi163 ceph-mon[221067]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: from='osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2537422346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:33 smithi181 ceph-mon[196979]: pgmap v153: 73 pgs: 3 active+undersized+degraded, 29 active+undersized, 41 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215] boot 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:02:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:02:34.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:34.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215] boot 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:34.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: failed to encode map e88 with expected crc 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: osd.1 [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215] boot 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: osdmap e89: 8 total, 8 up, 8 in 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:34.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:35.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: failed to encode map e89 with expected crc 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[221067]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 12 active+undersized, 60 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:35.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: failed to encode map e89 with expected crc 2023-10-09T04:02:35.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T04:02:35.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:35 smithi163 ceph-mon[213841]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 12 active+undersized, 60 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:35.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: failed to encode map e89 with expected crc 2023-10-09T04:02:35.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: osdmap e90: 8 total, 8 up, 8 in 2023-10-09T04:02:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:35 smithi181 ceph-mon[196979]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 12 active+undersized, 60 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-09T04:02:36.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:35 smithi163 conmon[200567]: 2023-10-09T04:02:35.743+0000 7f40871b1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3513827743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:36.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-10-09T04:02:36.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:36 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:36.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3513827743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-10-09T04:02:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: failed to encode map e90 with expected crc 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3513827743' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-10-09T04:02:36.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:36 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:38.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:38 smithi163 ceph-mon[221067]: pgmap v157: 73 pgs: 10 active+undersized, 63 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:38.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/261040844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:38 smithi163 ceph-mon[213841]: pgmap v157: 73 pgs: 10 active+undersized, 63 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/261040844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:38.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:38 smithi181 ceph-mon[196979]: pgmap v157: 73 pgs: 10 active+undersized, 63 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/261040844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:39.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:39 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T04:02:39.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:39 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:02:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:39 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T04:02:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:39 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:02:39.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:39 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-09T04:02:39.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:39 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:02:40.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:40 smithi163 ceph-mon[221067]: pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:40 smithi163 ceph-mon[213841]: pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:40 smithi181 ceph-mon[196979]: pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:41.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:02:40] "GET /metrics HTTP/1.1" 200 33843 "" "Prometheus/2.43.0" 2023-10-09T04:02:41.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3740226923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3740226923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:41.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3740226923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:42.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:42 smithi181 ceph-mon[196979]: pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:42.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:42 smithi163 ceph-mon[221067]: pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:42.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:42 smithi163 ceph-mon[213841]: pgmap v159: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2901789383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2901789383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:43.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2901789383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:44 smithi181 ceph-mon[196979]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:44 smithi163 ceph-mon[213841]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:44.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:44 smithi163 ceph-mon[221067]: pgmap v160: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:45.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:45.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:45 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3761812323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:45.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:45.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:45 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3761812323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:45.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:45 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3761812323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:46 smithi163 ceph-mon[221067]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:46 smithi163 ceph-mon[213841]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:46 smithi181 ceph-mon[196979]: pgmap v161: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1882881582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:48 smithi163 ceph-mon[221067]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1882881582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:48 smithi163 ceph-mon[213841]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1882881582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:48 smithi181 ceph-mon[196979]: pgmap v162: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:50 smithi181 ceph-mon[196979]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/675464720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[221067]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/675464720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[213841]: pgmap v163: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:50.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:02:50.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/675464720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:02:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:02:50] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.43.0" 2023-10-09T04:02:51.514 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:51.515 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: Upgrade: osd.2 is safe to restart 2023-10-09T04:02:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/163609544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:02:52.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: Upgrade: osd.2 is safe to restart 2023-10-09T04:02:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/163609544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: Upgrade: osd.2 is safe to restart 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: pgmap v164: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/163609544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:02:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:53 smithi181 ceph-mon[196979]: Upgrade: Updating osd.2 2023-10-09T04:02:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:53 smithi181 ceph-mon[196979]: Deploying daemon osd.2 on smithi163 2023-10-09T04:02:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:53 smithi163 ceph-mon[213841]: Upgrade: Updating osd.2 2023-10-09T04:02:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:53 smithi163 ceph-mon[213841]: Deploying daemon osd.2 on smithi163 2023-10-09T04:02:53.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:53 smithi163 ceph-mon[221067]: Upgrade: Updating osd.2 2023-10-09T04:02:53.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:53 smithi163 ceph-mon[221067]: Deploying daemon osd.2 on smithi163 2023-10-09T04:02:54.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:54 smithi163 ceph-mon[213841]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:54.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2032627284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:54.535 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:54 smithi163 ceph-mon[221067]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:54.535 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2032627284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:54 smithi181 ceph-mon[196979]: pgmap v165: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2032627284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:55.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:54 smithi163 systemd[1]: Stopping Ceph osd.2 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:55.083 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:55 smithi163 conmon[143762]: 2023-10-09T04:02:55.039+0000 7f692494d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:02:55.083 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:55 smithi163 conmon[143762]: 2023-10-09T04:02:55.039+0000 7f692494d700 -1 osd.2 90 *** Got signal Terminated *** 2023-10-09T04:02:55.083 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:55 smithi163 conmon[143762]: 2023-10-09T04:02:55.039+0000 7f692494d700 -1 osd.2 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:02:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:55 smithi163 ceph-mon[213841]: osd.2 marked itself down and dead 2023-10-09T04:02:55.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:55 smithi163 ceph-mon[221067]: osd.2 marked itself down and dead 2023-10-09T04:02:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:55 smithi181 ceph-mon[196979]: osd.2 marked itself down and dead 2023-10-09T04:02:56.081 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:55 smithi163 bash[252877]: e2ce5dc7d22b0f048d176bdf4e9c2d27ea56f4a91b132de0ee84d27820a7aeb9 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[221067]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[221067]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[221067]: failed to encode map e91 with expected crc 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[221067]: failed to encode map e91 with expected crc 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[221067]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[213841]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[213841]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:02:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[213841]: failed to encode map e91 with expected crc 2023-10-09T04:02:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[213841]: failed to encode map e91 with expected crc 2023-10-09T04:02:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:56 smithi163 ceph-mon[213841]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T04:02:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:56 smithi181 ceph-mon[196979]: pgmap v166: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:02:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:56 smithi181 ceph-mon[196979]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:02:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:56 smithi181 ceph-mon[196979]: failed to encode map e91 with expected crc 2023-10-09T04:02:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:56 smithi181 ceph-mon[196979]: failed to encode map e91 with expected crc 2023-10-09T04:02:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:56 smithi181 ceph-mon[196979]: osdmap e91: 8 total, 7 up, 8 in 2023-10-09T04:02:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2741409803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2741409803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: osdmap e92: 8 total, 7 up, 8 in 2023-10-09T04:02:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: failed to encode map e92 with expected crc 2023-10-09T04:02:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2741409803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:58.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:57 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T04:02:58.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:57 smithi163 systemd[1]: Stopped Ceph osd.2 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:58.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:57 smithi163 systemd[1]: Starting Ceph osd.2 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:58.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:58 smithi163 ceph-mon[213841]: pgmap v169: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+undersized, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:02:58.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:58 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:58 smithi163 ceph-mon[221067]: pgmap v169: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+undersized, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:02:58.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:58 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:58 smithi181 ceph-mon[196979]: pgmap v169: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+undersized, 69 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:02:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:58 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:02:59.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:02:59.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:02:59.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T04:02:59.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:02:59.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:02:59.083 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T04:02:59.083 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:02:59.083 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:02:58 smithi163 bash[253539]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T04:02:59.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:02:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2870891855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:59.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:02:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2870891855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:02:59.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:02:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2870891855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:00.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[221067]: pgmap v170: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:00.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:00.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:00.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:00.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-osd[254054]: -- 172.21.15.163:0/2875857669 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55802dd584e0 con 0x55802dd2b000 2023-10-09T04:03:00.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:03:00 smithi163 bash[253989]: e242e555fe1b87e6b4e8c92fd76f92a5376ac32458cfe90c8a0c093af4d95aa5 2023-10-09T04:03:00.332 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:03:00 smithi163 systemd[1]: Started Ceph osd.2 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:03:00.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[213841]: pgmap v170: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:00.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:00.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:00.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:00 smithi181 ceph-mon[196979]: pgmap v170: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:01.082 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:03:00 smithi163 conmon[254030]: 2023-10-09T04:03:00.584+0000 7f0554de8640 -1 Falling back to public interface 2023-10-09T04:03:01.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:03:00] "GET /metrics HTTP/1.1" 200 33990 "" "Prometheus/2.43.0" 2023-10-09T04:03:01.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2745793054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2745793054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:01.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2745793054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:02.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:02 smithi163 ceph-mon[221067]: pgmap v171: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:02 smithi163 ceph-mon[213841]: pgmap v171: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:02 smithi181 ceph-mon[196979]: pgmap v171: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:03.676 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:03:03 smithi163 conmon[254030]: 2023-10-09T04:03:03.389+0000 7f0554de8640 -1 osd.2 90 log_to_monitors true 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: pgmap v172: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:03:03.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1334515754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: pgmap v172: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:03:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:03 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1334515754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: pgmap v172: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:03:04.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:03 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1334515754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:04.659 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:03:04 smithi163 conmon[254030]: 2023-10-09T04:03:04.347+0000 7f0545a53700 -1 osd.2 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:05 smithi181 ceph-mon[196979]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[213841]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:05.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:03:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: osdmap e93: 8 total, 7 up, 8 in 2023-10-09T04:03:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: from='osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:03:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: failed to encode map e93 with expected crc 2023-10-09T04:03:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:05 smithi163 ceph-mon[221067]: failed to encode map e93 with expected crc 2023-10-09T04:03:06.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: pgmap v174: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:06.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:03:06.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: failed to encode map e94 with expected crc 2023-10-09T04:03:06.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: failed to encode map e94 with expected crc 2023-10-09T04:03:06.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669] boot 2023-10-09T04:03:06.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4286576120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: pgmap v174: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: failed to encode map e94 with expected crc 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: failed to encode map e94 with expected crc 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669] boot 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:03:06.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4286576120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: pgmap v174: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: failed to encode map e94 with expected crc 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: failed to encode map e94 with expected crc 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: osd.2 [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669] boot 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:03:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:06.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4286576120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:06.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:06 smithi163 conmon[200567]: 2023-10-09T04:03:06.714+0000 7f40871b1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:07.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: failed to encode map e95 with expected crc 2023-10-09T04:03:07.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:07.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:08.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:08 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:08 smithi181 ceph-mon[196979]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:03:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:08 smithi181 ceph-mon[196979]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 16 active+undersized, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2379818089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:08.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:08.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[221067]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:03:08.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[221067]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 16 active+undersized, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:08.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2379818089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[213841]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-09T04:03:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[213841]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 16 active+undersized, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2379818089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:10.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:10 smithi163 ceph-mon[213841]: pgmap v178: 73 pgs: 1 active+undersized+degraded, 12 active+undersized, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:10 smithi163 ceph-mon[221067]: pgmap v178: 73 pgs: 1 active+undersized+degraded, 12 active+undersized, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:10 smithi181 ceph-mon[196979]: pgmap v178: 73 pgs: 1 active+undersized+degraded, 12 active+undersized, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:03:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:11.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:03:10] "GET /metrics HTTP/1.1" 200 34053 "" "Prometheus/2.43.0" 2023-10-09T04:03:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1843575827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:11.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1843575827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:11.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1843575827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:12.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:12 smithi163 ceph-mon[221067]: pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:12 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T04:03:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:12 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:03:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:12 smithi163 ceph-mon[213841]: pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:12 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T04:03:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:12 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:03:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:12 smithi181 ceph-mon[196979]: pgmap v179: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:12 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-09T04:03:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:12 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:03:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2703943456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2703943456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:13.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2703943456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:14 smithi163 ceph-mon[221067]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:14 smithi163 ceph-mon[213841]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:14 smithi181 ceph-mon[196979]: pgmap v180: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/454193985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/454193985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/454193985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:16 smithi163 ceph-mon[221067]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:16 smithi163 ceph-mon[213841]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:16.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:16 smithi181 ceph-mon[196979]: pgmap v181: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:18 smithi181 ceph-mon[196979]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1097639601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:18 smithi163 ceph-mon[221067]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1097639601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:18 smithi163 ceph-mon[213841]: pgmap v182: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1097639601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:20 smithi181 ceph-mon[196979]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2308073835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[221067]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2308073835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[213841]: pgmap v183: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2308073835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:03:20] "GET /metrics HTTP/1.1" 200 34053 "" "Prometheus/2.43.0" 2023-10-09T04:03:22.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:22 smithi163 ceph-mon[221067]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:22.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:22.502 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1425434509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:22.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:22 smithi163 ceph-mon[213841]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:22.503 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:22.503 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1425434509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:22 smithi181 ceph-mon[196979]: pgmap v184: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1425434509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:23 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:23 smithi181 ceph-mon[196979]: Upgrade: osd.3 is safe to restart 2023-10-09T04:03:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:03:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[213841]: Upgrade: osd.3 is safe to restart 2023-10-09T04:03:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:03:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:03:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[221067]: Upgrade: osd.3 is safe to restart 2023-10-09T04:03:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:03:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:24.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:24 smithi181 ceph-mon[196979]: Upgrade: Updating osd.3 2023-10-09T04:03:24.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:24 smithi181 ceph-mon[196979]: Deploying daemon osd.3 on smithi163 2023-10-09T04:03:24.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:24 smithi181 ceph-mon[196979]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:24 smithi163 ceph-mon[221067]: Upgrade: Updating osd.3 2023-10-09T04:03:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:24 smithi163 ceph-mon[221067]: Deploying daemon osd.3 on smithi163 2023-10-09T04:03:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:24 smithi163 ceph-mon[221067]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:24 smithi163 ceph-mon[213841]: Upgrade: Updating osd.3 2023-10-09T04:03:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:24 smithi163 ceph-mon[213841]: Deploying daemon osd.3 on smithi163 2023-10-09T04:03:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:24 smithi163 ceph-mon[213841]: pgmap v185: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:25.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2598726147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:25.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2598726147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:25.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2598726147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:26.263 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:26 smithi163 systemd[1]: Stopping Ceph osd.3 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:03:26.263 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:26 smithi163 conmon[153552]: 2023-10-09T04:03:26.233+0000 7f815ab0d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:03:26.263 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:26 smithi163 conmon[153552]: 2023-10-09T04:03:26.233+0000 7f815ab0d700 -1 osd.3 95 *** Got signal Terminated *** 2023-10-09T04:03:26.263 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:26 smithi163 conmon[153552]: 2023-10-09T04:03:26.233+0000 7f815ab0d700 -1 osd.3 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:03:26.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:26 smithi181 ceph-mon[196979]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:26.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:26 smithi181 ceph-mon[196979]: osd.3 marked itself down and dead 2023-10-09T04:03:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:26 smithi163 ceph-mon[221067]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:26 smithi163 ceph-mon[221067]: osd.3 marked itself down and dead 2023-10-09T04:03:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:26 smithi163 ceph-mon[213841]: pgmap v186: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:26 smithi163 ceph-mon[213841]: osd.3 marked itself down and dead 2023-10-09T04:03:27.082 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:26 smithi163 bash[260463]: 0e8301f836bccc71e6f3d0780a0bc66d875f44b1f89752ccd2802a3e7520bc45 2023-10-09T04:03:27.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:27 smithi181 ceph-mon[196979]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:03:27.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:27 smithi181 ceph-mon[196979]: failed to encode map e96 with expected crc 2023-10-09T04:03:27.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:27 smithi181 ceph-mon[196979]: failed to encode map e96 with expected crc 2023-10-09T04:03:27.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:27 smithi181 ceph-mon[196979]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T04:03:27.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2256262541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[213841]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:03:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[213841]: failed to encode map e96 with expected crc 2023-10-09T04:03:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[213841]: failed to encode map e96 with expected crc 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[213841]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2256262541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[221067]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[221067]: failed to encode map e96 with expected crc 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[221067]: failed to encode map e96 with expected crc 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[221067]: osdmap e96: 8 total, 7 up, 8 in 2023-10-09T04:03:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2256262541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: pgmap v188: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T04:03:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: pgmap v188: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:28 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:03:28.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: pgmap v188: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: osdmap e97: 8 total, 7 up, 8 in 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: failed to encode map e97 with expected crc 2023-10-09T04:03:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:28 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:03:29.082 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:28 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T04:03:29.083 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:28 smithi163 systemd[1]: Stopped Ceph osd.3 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:03:29.083 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:28 smithi163 systemd[1]: Starting Ceph osd.3 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:03:29.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4239539711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:29.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4239539711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:29.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4239539711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:30.082 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:03:30.082 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T04:03:30.082 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T04:03:30.082 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T04:03:30.083 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T04:03:30.083 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T04:03:30.083 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T04:03:30.083 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:29 smithi163 bash[261147]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T04:03:30.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:30 smithi163 ceph-mon[213841]: pgmap v190: 73 pgs: 7 peering, 8 stale+active+clean, 6 active+undersized, 52 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:30 smithi163 ceph-mon[221067]: pgmap v190: 73 pgs: 7 peering, 8 stale+active+clean, 6 active+undersized, 52 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:30 smithi181 ceph-mon[196979]: pgmap v190: 73 pgs: 7 peering, 8 stale+active+clean, 6 active+undersized, 52 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:30.902 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:03:30] "GET /metrics HTTP/1.1" 200 34053 "" "Prometheus/2.43.0" 2023-10-09T04:03:31.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:30 smithi163 ceph-osd[261615]: -- 172.21.15.163:0/2675859776 <== mon.0 v2:172.21.15.163:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a63a544e0 con 0x559a63a26c00 2023-10-09T04:03:31.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:30 smithi163 bash[261549]: 859a75049c4733676092e751e2cccc1b34cdec12b6706bdf13225dea4431a4f5 2023-10-09T04:03:31.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:30 smithi163 systemd[1]: Started Ceph osd.3 for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:03:31.831 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:31 smithi163 conmon[261591]: 2023-10-09T04:03:31.423+0000 7fe530c7f640 -1 Falling back to public interface 2023-10-09T04:03:32.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:32.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:32.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:32.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:31 smithi181 ceph-mon[196979]: pgmap v191: 73 pgs: 2 active+undersized+degraded, 7 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:03:32.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/193989180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:32.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[221067]: pgmap v191: 73 pgs: 2 active+undersized+degraded, 7 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:03:32.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/193989180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:32.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:32.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:32.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:32.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[213841]: pgmap v191: 73 pgs: 2 active+undersized+degraded, 7 peering, 22 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:03:32.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/193989180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:33.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:32 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:33.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:32 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:32 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:34.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:33 smithi181 ceph-mon[196979]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:34.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:33 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:34.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:33 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:34.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2018929383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:34.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:34 smithi163 conmon[261591]: 2023-10-09T04:03:34.192+0000 7fe530c7f640 -1 osd.3 95 log_to_monitors true 2023-10-09T04:03:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[221067]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2018929383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[213841]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2018929383' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:35.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:03:35.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[221067]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:03:35.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.074 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:35.075 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:03:35.075 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.075 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[213841]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:03:35.075 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.075 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.075 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:35.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:34 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:03:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:34 smithi181 ceph-mon[196979]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:03:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:35.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:03:35 smithi163 conmon[261591]: 2023-10-09T04:03:35.221+0000 7fe5218ea700 -1 osd.3 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:03:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: osdmap e98: 8 total, 7 up, 8 in 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3979381362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: osdmap e98: 8 total, 7 up, 8 in 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3979381362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: osdmap e98: 8 total, 7 up, 8 in 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: from='osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi163", "root=default"]}]: dispatch 2023-10-09T04:03:36.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: failed to encode map e98 with expected crc 2023-10-09T04:03:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3979381362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:37.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:03:37.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: failed to encode map e99 with expected crc 2023-10-09T04:03:37.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: failed to encode map e99 with expected crc 2023-10-09T04:03:37.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858] boot 2023-10-09T04:03:37.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:03:37.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:03:37.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: failed to encode map e99 with expected crc 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: failed to encode map e99 with expected crc 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858] boot 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:03:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: failed to encode map e99 with expected crc 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: failed to encode map e99 with expected crc 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: osd.3 [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858] boot 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:03:37.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:03:38.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:37 smithi163 conmon[200567]: 2023-10-09T04:03:37.834+0000 7f40871b1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:03:38.298 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T04:03:38.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: pgmap v197: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:38.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.484 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: pgmap v197: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: failed to encode map e100 with expected crc 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: pgmap v197: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:03:38.914 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:03:39.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:39 smithi163 ceph-mon[213841]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:03:39.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:39 smithi163 ceph-mon[213841]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:03:39.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/439924499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:39.263 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:39.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:39 smithi163 ceph-mon[221067]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:03:39.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:39 smithi163 ceph-mon[221067]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:03:39.467 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/439924499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:39.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:39 smithi181 ceph-mon[196979]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:03:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:39 smithi181 ceph-mon[196979]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:03:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/439924499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:03:40.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:40 smithi163 ceph-mon[221067]: pgmap v198: 73 pgs: 5 peering, 3 active+undersized+degraded, 20 active+undersized, 45 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:40.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:40 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:40.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:40 smithi163 ceph-mon[213841]: pgmap v198: 73 pgs: 5 peering, 3 active+undersized+degraded, 20 active+undersized, 45 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:40.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:40 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:40 smithi181 ceph-mon[196979]: pgmap v198: 73 pgs: 5 peering, 3 active+undersized+degraded, 20 active+undersized, 45 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:03:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:40 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:40.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:03:40] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.43.0" 2023-10-09T04:03:40.924 INFO:teuthology.orchestra.run.smithi163.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:03:41.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:03:41.942 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:42.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:41 smithi181 ceph-mon[196979]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:41 smithi181 ceph-mon[196979]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:03:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:41 smithi181 ceph-mon[196979]: pgmap v199: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:42.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[221067]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:42.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[221067]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:03:42.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:42.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[221067]: pgmap v199: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[213841]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[213841]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:03:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:42.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:41 smithi163 ceph-mon[213841]: pgmap v199: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:43.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:42 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T04:03:43.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:42 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:03:43.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:42 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T04:03:43.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:42 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:03:43.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:42 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T04:03:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:42 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (6m) - 10m 27.7M - 0.25.0 c8568f914cd2 b04e9e96f8bb 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (6m) - 9m 96.0M - 9.4.7 2c41d148cca3 dd057a75b8e1 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.crqkdj smithi163 running (9m) - 9m 19.1M - 17.2.6-1303-gb2903cb0 5517396c50d4 8037aa8a1d7d 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.sboiwx smithi181 running (9m) - 9m 16.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 2c2246f2ef4b 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 cbeb189f4272 2023-10-09T04:03:43.571 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-598-gb4fa663f 3060897fdf97 6bfa655af17a 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (6m) - 15m 86.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8efafb4ec97d 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (3m) - 14m 53.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 48065a0413d3 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (4m) - 14m 62.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0f3bd992ec5c 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (6m) - 9m 17.2M - 1.5.0 0da6a335fe13 381d36940ced 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (6m) - 9m 17.9M - 1.5.0 0da6a335fe13 79b2ee0a484d 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (105s) - 13m 56.8M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 663e3d75cc47 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (74s) - 13m 55.2M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 e0fb1e6b8d23 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (43s) - 12m 49.3M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 e242e555fe1b 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (12s) - 12m 12.4M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 859a75049c47 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (12m) - 12m 98.3M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 b4d4d04d2a3e 2023-10-09T04:03:43.572 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (11m) - 11m 112M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4ed298059fdf 2023-10-09T04:03:43.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (11m) - 11m 88.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4d54b928e978 2023-10-09T04:03:43.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (11m) - 11m 121M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 6ca0d6954129 2023-10-09T04:03:43.573 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (6m) - 10m 70.3M - 2.43.0 a07b618ecd1d 3be07b4832ab 2023-10-09T04:03:44.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:43 smithi163 ceph-mon[213841]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:44.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:43 smithi163 ceph-mon[221067]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:44.254 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:03:44.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:43 smithi181 ceph-mon[196979]: pgmap v200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:44.601 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:45.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:44 smithi163 ceph-mon[221067]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:44 smithi163 ceph-mon[213841]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:45.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:44 smithi181 ceph-mon[196979]: from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:46.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:45 smithi163 ceph-mon[221067]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:46.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:45 smithi163 ceph-mon[213841]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:46.249 INFO:teuthology.orchestra.run.smithi163.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:03:46.249 INFO:teuthology.orchestra.run.smithi163.stdout:alertmanager.a smithi163 *:9093,9094 running (6m) - 10m 27.7M - 0.25.0 c8568f914cd2 b04e9e96f8bb 2023-10-09T04:03:46.249 INFO:teuthology.orchestra.run.smithi163.stdout:grafana.a smithi181 *:3000 running (6m) - 9m 96.0M - 9.4.7 2c41d148cca3 dd057a75b8e1 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi163.crqkdj smithi163 running (9m) - 9m 19.1M - 17.2.6-1303-gb2903cb0 5517396c50d4 8037aa8a1d7d 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mds.foo.smithi181.sboiwx smithi181 running (9m) - 9m 16.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 2c2246f2ef4b 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.x smithi181 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 cbeb189f4272 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mgr.y smithi163 *:8443,9283,8765 running (6m) - 15m 515M - 18.2.0-598-gb4fa663f 3060897fdf97 6bfa655af17a 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a smithi163 running (6m) - 15m 86.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8efafb4ec97d 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b smithi181 running (3m) - 14m 53.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 48065a0413d3 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c smithi163 running (4m) - 14m 62.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 0f3bd992ec5c 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.a smithi163 *:9100 running (6m) - 9m 17.2M - 1.5.0 0da6a335fe13 381d36940ced 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:node-exporter.b smithi181 *:9100 running (6m) - 9m 17.9M - 1.5.0 0da6a335fe13 79b2ee0a484d 2023-10-09T04:03:46.250 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 smithi163 running (108s) - 13m 56.8M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 663e3d75cc47 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 smithi163 running (77s) - 13m 55.2M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 e0fb1e6b8d23 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 smithi163 running (46s) - 12m 49.3M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 e242e555fe1b 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 smithi163 running (15s) - 12m 12.4M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 859a75049c47 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 smithi181 running (12m) - 12m 98.3M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 b4d4d04d2a3e 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 smithi181 running (11m) - 11m 112M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4ed298059fdf 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 smithi181 running (11m) - 11m 88.5M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4d54b928e978 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 smithi181 running (11m) - 11m 121M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 6ca0d6954129 2023-10-09T04:03:46.251 INFO:teuthology.orchestra.run.smithi163.stdout:prometheus.a smithi181 *:9095 running (6m) - 10m 70.3M - 2.43.0 a07b618ecd1d 3be07b4832ab 2023-10-09T04:03:46.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:45 smithi181 ceph-mon[196979]: pgmap v201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:46.871 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T04:03:47.223 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:47.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:46 smithi181 ceph-mon[196979]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:46 smithi163 ceph-mon[221067]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:46 smithi163 ceph-mon[213841]: from='client.44456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:48.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:48 smithi163 ceph-mon[221067]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:48.440 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:48 smithi163 ceph-mon[213841]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:48 smithi181 ceph-mon[196979]: pgmap v202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:48.908 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:03:48.908 INFO:teuthology.orchestra.run.smithi163.stdout: "mon": { 2023-10-09T04:03:48.908 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:03:48.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr": { 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "osd": { 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 4, 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "mds": { 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 2 2023-10-09T04:03:48.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:03:48.910 INFO:teuthology.orchestra.run.smithi163.stdout: "overall": { 2023-10-09T04:03:48.910 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 6, 2023-10-09T04:03:48.910 INFO:teuthology.orchestra.run.smithi163.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T04:03:48.910 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:03:48.910 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:03:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3383393447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3383393447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3383393447' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:03:49.579 DEBUG:teuthology.parallel:result is None 2023-10-09T04:03:49.579 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:03:49.589 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T04:03:49.590 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T04:03:49.598 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:03:49.598 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:03:49.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:03:49.944 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:50.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:50.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:50.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[213841]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:50.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:50.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:50.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:50.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[221067]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:50.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:03:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:50 smithi181 ceph-mon[196979]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:03:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:03:50] "GET /metrics HTTP/1.1" 200 34044 "" "Prometheus/2.43.0" 2023-10-09T04:03:51.559 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:03:52.193 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":36,"active_gid":34104,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":2698166629},{"type":"v1","addr":"172.21.15.163:6801","nonce":2698166629}]},"active_addr":"172.21.15.163:6801/2698166629","active_change":"2023-10-09T03:57:49.149475+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25039,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":80,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":2061487337}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3165078582}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":3169340142}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.163:0","nonce":2280121025}]}]} 2023-10-09T04:03:52.201 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:03:52.201 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:03:52.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:03:52.550 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:52.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:52 smithi163 ceph-mon[221067]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:52 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3694247777' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:03:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:52 smithi163 ceph-mon[213841]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:52 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3694247777' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:03:52.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:52 smithi181 ceph-mon[196979]: pgmap v204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:52 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3694247777' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:03:54.111 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:03:54.111 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":100,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T04:03:37.200609+0000","last_up_change":"2023-10-09T04:03:36.195924+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T03:54:11.510171+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T03:54:12.437589+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":99,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6803","nonce":658095252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6805","nonce":658095252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6809","nonce":658095252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6807","nonce":658095252}]},"public_addr":"172.21.15.163:6803/658095252","cluster_addr":"172.21.15.163:6805/658095252","heartbeat_back_addr":"172.21.15.163:6809/658095252","heartbeat_front_addr":"172.21.15.163:6807/658095252","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":99,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6811","nonce":3761898215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6813","nonce":3761898215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6817","nonce":3761898215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6815","nonce":3761898215}]},"public_addr":"172.21.15.163:6811/3761898215","cluster_addr":"172.21.15.163:6813/3761898215","heartbeat_back_addr":"172.21.15.163:6817/3761898215","heartbeat_front_addr":"172.21.15.163:6815/3761898215","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":99,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6819","nonce":96613669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6821","nonce":96613669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6825","nonce":96613669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6823","nonce":96613669}]},"public_addr":"172.21.15.163:6819/96613669","cluster_addr":"172.21.15.163:6821/96613669","heartbeat_back_addr":"172.21.15.163:6825/96613669","heartbeat_front_addr":"172.21.15.163:6823/96613669","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":95,"up_from":99,"up_thru":99,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6827","nonce":3467616858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6829","nonce":3467616858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6833","nonce":3467616858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6831","nonce":3467616858}]},"public_addr":"172.21.15.163:6827/3467616858","cluster_addr":"172.21.15.163:6829/3467616858","heartbeat_back_addr":"172.21.15.163:6833/3467616858","heartbeat_front_addr":"172.21.15.163:6831/3467616858","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.181:0/3987314134":"2023-10-10T03:57:13.005970+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2462309240":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1847962187":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1802556470":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/3524034575":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6801/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.181:6835/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/814100112":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/2543837887":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/4218140698":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1275790360":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.181:6834/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.181:0/1808329322":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2380156011":"2023-10-10T03:57:49.149190+0000","172.21.15.181:0/647401329":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/1229473398":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6800/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6801/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.181:0/1614027396":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/3405897408":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1361101496":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:03:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:54 smithi163 ceph-mon[221067]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1825867973' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:03:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:54 smithi163 ceph-mon[213841]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1825867973' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:03:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:54 smithi181 ceph-mon[196979]: pgmap v205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1825867973' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:03:54.777 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:03:54.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:03:55.122 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:56.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:56 smithi163 ceph-mon[221067]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:56.381 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:56 smithi163 ceph-mon[213841]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:56 smithi181 ceph-mon[196979]: pgmap v206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:56.797 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:03:56.797 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":100,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T04:03:37.200609+0000","last_up_change":"2023-10-09T04:03:36.195924+0000","last_in_change":"2023-10-09T03:52:28.035250+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T03:54:11.510171+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T03:54:12.437589+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":99,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6803","nonce":658095252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6805","nonce":658095252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6809","nonce":658095252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6807","nonce":658095252}]},"public_addr":"172.21.15.163:6803/658095252","cluster_addr":"172.21.15.163:6805/658095252","heartbeat_back_addr":"172.21.15.163:6809/658095252","heartbeat_front_addr":"172.21.15.163:6807/658095252","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":99,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6811","nonce":3761898215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6813","nonce":3761898215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6817","nonce":3761898215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6815","nonce":3761898215}]},"public_addr":"172.21.15.163:6811/3761898215","cluster_addr":"172.21.15.163:6813/3761898215","heartbeat_back_addr":"172.21.15.163:6817/3761898215","heartbeat_front_addr":"172.21.15.163:6815/3761898215","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":99,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6819","nonce":96613669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6821","nonce":96613669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6825","nonce":96613669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6823","nonce":96613669}]},"public_addr":"172.21.15.163:6819/96613669","cluster_addr":"172.21.15.163:6821/96613669","heartbeat_back_addr":"172.21.15.163:6825/96613669","heartbeat_front_addr":"172.21.15.163:6823/96613669","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":95,"up_from":99,"up_thru":99,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6827","nonce":3467616858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6829","nonce":3467616858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6833","nonce":3467616858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6831","nonce":3467616858}]},"public_addr":"172.21.15.163:6827/3467616858","cluster_addr":"172.21.15.163:6829/3467616858","heartbeat_back_addr":"172.21.15.163:6833/3467616858","heartbeat_front_addr":"172.21.15.163:6831/3467616858","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":99,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.181:0/3987314134":"2023-10-10T03:57:13.005970+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2462309240":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1847962187":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1802556470":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/3524034575":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6801/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.181:6835/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/814100112":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/2543837887":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/4218140698":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1275790360":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.181:6834/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.181:0/1808329322":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2380156011":"2023-10-10T03:57:49.149190+0000","172.21.15.181:0/647401329":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/1229473398":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6800/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6801/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.181:0/1614027396":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/3405897408":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1361101496":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:03:57.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:03:57.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:03:57.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:03:57.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:03:57.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:03:57.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:03:57.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:03:57.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:03:57.590 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2762062502' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:03:57.596 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2762062502' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:03:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2762062502' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:03:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:03:58 smithi163 ceph-mon[221067]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:58.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:03:58 smithi163 ceph-mon[213841]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:58.674 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:03:58 smithi181 ceph-mon[196979]: pgmap v207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:03:58.782 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:58.879 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:58.951 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:58.981 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:59.184 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:59.281 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:59.499 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:00.433 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:00 smithi163 ceph-mon[213841]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:00.433 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:00 smithi163 ceph-mon[221067]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:00.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:00 smithi181 ceph-mon[196979]: pgmap v208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:00.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:04:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:04:00] "GET /metrics HTTP/1.1" 200 34044 "" "Prometheus/2.43.0" 2023-10-09T04:04:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:02 smithi163 ceph-mon[221067]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:02 smithi163 ceph-mon[213841]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:02 smithi181 ceph-mon[196979]: pgmap v209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:04.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:04 smithi163 ceph-mon[221067]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:04.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:04.495 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:04 smithi163 ceph-mon[213841]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:04.495 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:04 smithi181 ceph-mon[196979]: pgmap v210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:05.178 INFO:teuthology.orchestra.run.smithi163.stdout:360777252890 2023-10-09T04:04:05.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:04:06.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:06 smithi163 ceph-mon[221067]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:06.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:06 smithi163 ceph-mon[213841]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:06.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:06 smithi181 ceph-mon[196979]: pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:07.300 INFO:teuthology.orchestra.run.smithi163.stdout:425201762311 2023-10-09T04:04:07.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:04:08.429 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:08 smithi163 ceph-mon[213841]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:08.430 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:08 smithi163 ceph-mon[221067]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:08.431 INFO:teuthology.orchestra.run.smithi163.stdout:133143986328 2023-10-09T04:04:08.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:04:08.484 INFO:teuthology.orchestra.run.smithi163.stdout:167503724692 2023-10-09T04:04:08.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:04:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:08 smithi181 ceph-mon[196979]: pgmap v212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:09.211 INFO:teuthology.orchestra.run.smithi163.stdout:201863463056 2023-10-09T04:04:09.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:04:09.641 INFO:teuthology.orchestra.run.smithi163.stdout:236223201419 2023-10-09T04:04:09.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:04:09.687 INFO:teuthology.orchestra.run.smithi163.stdout:382252089364 2023-10-09T04:04:09.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:04:09.741 INFO:teuthology.orchestra.run.smithi163.stdout:403726925838 2023-10-09T04:04:09.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:04:09.784 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:10.022 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:10.027 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:10.027 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:10.028 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:10.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:10 smithi163 ceph-mon[221067]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:10.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:10 smithi163 ceph-mon[213841]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:10.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:10 smithi181 ceph-mon[196979]: pgmap v213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:10.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:04:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:04:10] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.43.0" 2023-10-09T04:04:12.142 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:12.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:12 smithi163 ceph-mon[221067]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:12 smithi163 ceph-mon[213841]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:12 smithi181 ceph-mon[196979]: pgmap v214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:13.570 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:13.570 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:14.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:14 smithi163 ceph-mon[221067]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:14.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:14 smithi163 ceph-mon[213841]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:14 smithi181 ceph-mon[196979]: pgmap v215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:15.410 INFO:teuthology.orchestra.run.smithi163.stdout:425201762314 2023-10-09T04:04:15.522 INFO:teuthology.orchestra.run.smithi163.stdout:167503724694 2023-10-09T04:04:16.118 INFO:teuthology.orchestra.run.smithi163.stdout:201863463058 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[221067]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3770575141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2648388475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1216681348' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2400246897' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[213841]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:16.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3770575141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:04:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2648388475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:04:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1216681348' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:04:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2400246897' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:04:16.670 INFO:teuthology.orchestra.run.smithi163.stdout:133143986330 2023-10-09T04:04:16.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:16 smithi181 ceph-mon[196979]: pgmap v216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3770575141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:04:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2648388475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:04:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1216681348' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:04:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2400246897' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:04:17.590 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986328 got 133143986330 for osd.4 2023-10-09T04:04:17.590 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:17.786 INFO:teuthology.orchestra.run.smithi163.stdout:360777252892 2023-10-09T04:04:17.904 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724692 got 167503724694 for osd.5 2023-10-09T04:04:17.904 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:18.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:18 smithi163 ceph-mon[221067]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:18.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/457439331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:04:18.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:18 smithi163 ceph-mon[213841]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:18.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/457439331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:04:18.277 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762311 got 425201762314 for osd.3 2023-10-09T04:04:18.278 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:18 smithi181 ceph-mon[196979]: pgmap v217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/457439331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:04:18.783 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463056 got 201863463058 for osd.6 2023-10-09T04:04:18.783 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:19.491 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777252890 got 360777252892 for osd.0 2023-10-09T04:04:19.491 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:20.091 INFO:teuthology.orchestra.run.smithi163.stdout:403726925840 2023-10-09T04:04:20.121 INFO:teuthology.orchestra.run.smithi163.stdout:382252089367 2023-10-09T04:04:20.366 INFO:teuthology.orchestra.run.smithi163.stdout:236223201421 2023-10-09T04:04:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:20 smithi163 ceph-mon[221067]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3540071291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:04:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1283337650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:04:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:20 smithi163 ceph-mon[213841]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3540071291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:04:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1283337650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:04:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:20 smithi181 ceph-mon[196979]: pgmap v218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3540071291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:04:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1283337650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:04:20.965 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:04:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:04:20] "GET /metrics HTTP/1.1" 200 34044 "" "Prometheus/2.43.0" 2023-10-09T04:04:21.306 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925838 got 403726925840 for osd.2 2023-10-09T04:04:21.306 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:21.344 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201419 got 236223201421 for osd.7 2023-10-09T04:04:21.344 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:21.544 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2424121231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:04:21.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2424121231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:04:21.545 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089364 got 382252089367 for osd.1 2023-10-09T04:04:21.546 DEBUG:teuthology.parallel:result is None 2023-10-09T04:04:21.546 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:04:21.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:04:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2424121231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:04:21.890 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:22.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:22 smithi163 ceph-mon[213841]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:22.416 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:22 smithi163 ceph-mon[221067]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:22 smithi181 ceph-mon[196979]: pgmap v219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:23.538 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:04:23.539 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:04:24.165 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":220,"stamp":"2023-10-09T04:04:23.263874+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333544,"kb_used_data":9768,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627672,"statfs":{"total":767960285184,"available":766594736128,"internally_reserved":0,"allocated":10002432,"data_stored":6138143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002978"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206546+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:03:36.206546+0000","last_peered":"2023-10-09T04:03:36.206546+0000","last_clean":"2023-10-09T04:03:36.206546+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:03:36.206546+0000","last_undegraded":"2023-10-09T04:03:36.206546+0000","last_fullsized":"2023-10-09T04:03:36.206546+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207470+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:03:37.207470+0000","last_peered":"2023-10-09T04:03:37.207470+0000","last_clean":"2023-10-09T04:03:37.207470+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:03:37.207470+0000","last_undegraded":"2023-10-09T04:03:37.207470+0000","last_fullsized":"2023-10-09T04:03:37.207470+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215833+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:03:37.215833+0000","last_peered":"2023-10-09T04:03:37.215833+0000","last_clean":"2023-10-09T04:03:37.215833+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:03:37.215833+0000","last_undegraded":"2023-10-09T04:03:37.215833+0000","last_fullsized":"2023-10-09T04:03:37.215833+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221959+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:03:37.221959+0000","last_peered":"2023-10-09T04:03:37.221959+0000","last_clean":"2023-10-09T04:03:37.221959+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:03:37.221959+0000","last_undegraded":"2023-10-09T04:03:37.221959+0000","last_fullsized":"2023-10-09T04:03:37.221959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.224382+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:03:37.224382+0000","last_peered":"2023-10-09T04:03:37.224382+0000","last_clean":"2023-10-09T04:03:37.224382+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:03:37.224382+0000","last_undegraded":"2023-10-09T04:03:37.224382+0000","last_fullsized":"2023-10-09T04:03:37.224382+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213576+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:03:36.213576+0000","last_peered":"2023-10-09T04:03:36.213576+0000","last_clean":"2023-10-09T04:03:36.213576+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:03:36.213576+0000","last_undegraded":"2023-10-09T04:03:36.213576+0000","last_fullsized":"2023-10-09T04:03:36.213576+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225542+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:03:37.225542+0000","last_peered":"2023-10-09T04:03:37.225542+0000","last_clean":"2023-10-09T04:03:37.225542+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:03:37.225542+0000","last_undegraded":"2023-10-09T04:03:37.225542+0000","last_fullsized":"2023-10-09T04:03:37.225542+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212908+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:03:36.212908+0000","last_peered":"2023-10-09T04:03:36.212908+0000","last_clean":"2023-10-09T04:03:36.212908+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:03:36.212908+0000","last_undegraded":"2023-10-09T04:03:36.212908+0000","last_fullsized":"2023-10-09T04:03:36.212908+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215366+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:03:37.215366+0000","last_peered":"2023-10-09T04:03:37.215366+0000","last_clean":"2023-10-09T04:03:37.215366+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:03:37.215366+0000","last_undegraded":"2023-10-09T04:03:37.215366+0000","last_fullsized":"2023-10-09T04:03:37.215366+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211445+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:03:36.211445+0000","last_peered":"2023-10-09T04:03:36.211445+0000","last_clean":"2023-10-09T04:03:36.211445+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:03:36.211445+0000","last_undegraded":"2023-10-09T04:03:36.211445+0000","last_fullsized":"2023-10-09T04:03:36.211445+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206640+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:03:36.206640+0000","last_peered":"2023-10-09T04:03:36.206640+0000","last_clean":"2023-10-09T04:03:36.206640+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:03:36.206640+0000","last_undegraded":"2023-10-09T04:03:36.206640+0000","last_fullsized":"2023-10-09T04:03:36.206640+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208063+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:03:37.208063+0000","last_peered":"2023-10-09T04:03:37.208063+0000","last_clean":"2023-10-09T04:03:37.208063+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:03:37.208063+0000","last_undegraded":"2023-10-09T04:03:37.208063+0000","last_fullsized":"2023-10-09T04:03:37.208063+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208745+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:03:36.208745+0000","last_peered":"2023-10-09T04:03:36.208745+0000","last_clean":"2023-10-09T04:03:36.208745+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:03:36.208745+0000","last_undegraded":"2023-10-09T04:03:36.208745+0000","last_fullsized":"2023-10-09T04:03:36.208745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221060+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:03:37.221060+0000","last_peered":"2023-10-09T04:03:37.221060+0000","last_clean":"2023-10-09T04:03:37.221060+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:03:37.221060+0000","last_undegraded":"2023-10-09T04:03:37.221060+0000","last_fullsized":"2023-10-09T04:03:37.221060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211820+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:03:36.211820+0000","last_peered":"2023-10-09T04:03:36.211820+0000","last_clean":"2023-10-09T04:03:36.211820+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:03:36.211820+0000","last_undegraded":"2023-10-09T04:03:36.211820+0000","last_fullsized":"2023-10-09T04:03:36.211820+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217253+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:03:37.217253+0000","last_peered":"2023-10-09T04:03:37.217253+0000","last_clean":"2023-10-09T04:03:37.217253+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:03:37.217253+0000","last_undegraded":"2023-10-09T04:03:37.217253+0000","last_fullsized":"2023-10-09T04:03:37.217253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207368+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:03:37.207368+0000","last_peered":"2023-10-09T04:03:37.207368+0000","last_clean":"2023-10-09T04:03:37.207368+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:03:37.207368+0000","last_undegraded":"2023-10-09T04:03:37.207368+0000","last_fullsized":"2023-10-09T04:03:37.207368+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211705+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:03:36.211705+0000","last_peered":"2023-10-09T04:03:36.211705+0000","last_clean":"2023-10-09T04:03:36.211705+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:03:36.211705+0000","last_undegraded":"2023-10-09T04:03:36.211705+0000","last_fullsized":"2023-10-09T04:03:36.211705+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214698+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:03:36.214698+0000","last_peered":"2023-10-09T04:03:36.214698+0000","last_clean":"2023-10-09T04:03:36.214698+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:03:36.214698+0000","last_undegraded":"2023-10-09T04:03:36.214698+0000","last_fullsized":"2023-10-09T04:03:36.214698+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208137+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:03:37.208137+0000","last_peered":"2023-10-09T04:03:37.208137+0000","last_clean":"2023-10-09T04:03:37.208137+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:03:37.208137+0000","last_undegraded":"2023-10-09T04:03:37.208137+0000","last_fullsized":"2023-10-09T04:03:37.208137+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220143+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:03:37.220143+0000","last_peered":"2023-10-09T04:03:37.220143+0000","last_clean":"2023-10-09T04:03:37.220143+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:03:37.220143+0000","last_undegraded":"2023-10-09T04:03:37.220143+0000","last_fullsized":"2023-10-09T04:03:37.220143+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212510+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:03:36.212510+0000","last_peered":"2023-10-09T04:03:36.212510+0000","last_clean":"2023-10-09T04:03:36.212510+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:03:36.212510+0000","last_undegraded":"2023-10-09T04:03:36.212510+0000","last_fullsized":"2023-10-09T04:03:36.212510+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.209067+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:03:36.209067+0000","last_peered":"2023-10-09T04:03:36.209067+0000","last_clean":"2023-10-09T04:03:36.209067+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:03:36.209067+0000","last_undegraded":"2023-10-09T04:03:36.209067+0000","last_fullsized":"2023-10-09T04:03:36.209067+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221180+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:03:37.221180+0000","last_peered":"2023-10-09T04:03:37.221180+0000","last_clean":"2023-10-09T04:03:37.221180+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:03:37.221180+0000","last_undegraded":"2023-10-09T04:03:37.221180+0000","last_fullsized":"2023-10-09T04:03:37.221180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":440,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212413+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:03:36.212413+0000","last_peered":"2023-10-09T04:03:36.212413+0000","last_clean":"2023-10-09T04:03:36.212413+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:03:36.212413+0000","last_undegraded":"2023-10-09T04:03:36.212413+0000","last_fullsized":"2023-10-09T04:03:36.212413+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211739+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:03:36.211739+0000","last_peered":"2023-10-09T04:03:36.211739+0000","last_clean":"2023-10-09T04:03:36.211739+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:03:36.211739+0000","last_undegraded":"2023-10-09T04:03:36.211739+0000","last_fullsized":"2023-10-09T04:03:36.211739+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.218572+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:03:36.218572+0000","last_peered":"2023-10-09T04:03:36.218572+0000","last_clean":"2023-10-09T04:03:36.218572+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:03:36.218572+0000","last_undegraded":"2023-10-09T04:03:36.218572+0000","last_fullsized":"2023-10-09T04:03:36.218572+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213218+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:03:36.213218+0000","last_peered":"2023-10-09T04:03:36.213218+0000","last_clean":"2023-10-09T04:03:36.213218+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:03:36.213218+0000","last_undegraded":"2023-10-09T04:03:36.213218+0000","last_fullsized":"2023-10-09T04:03:36.213218+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215356+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:03:37.215356+0000","last_peered":"2023-10-09T04:03:37.215356+0000","last_clean":"2023-10-09T04:03:37.215356+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:03:37.215356+0000","last_undegraded":"2023-10-09T04:03:37.215356+0000","last_fullsized":"2023-10-09T04:03:37.215356+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.218452+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:03:37.218452+0000","last_peered":"2023-10-09T04:03:37.218452+0000","last_clean":"2023-10-09T04:03:37.218452+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:03:37.218452+0000","last_undegraded":"2023-10-09T04:03:37.218452+0000","last_fullsized":"2023-10-09T04:03:37.218452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207441+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:03:36.207441+0000","last_peered":"2023-10-09T04:03:36.207441+0000","last_clean":"2023-10-09T04:03:36.207441+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:03:36.207441+0000","last_undegraded":"2023-10-09T04:03:36.207441+0000","last_fullsized":"2023-10-09T04:03:36.207441+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214621+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:03:36.214621+0000","last_peered":"2023-10-09T04:03:36.214621+0000","last_clean":"2023-10-09T04:03:36.214621+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:03:36.214621+0000","last_undegraded":"2023-10-09T04:03:36.214621+0000","last_fullsized":"2023-10-09T04:03:36.214621+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211952+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:03:36.211952+0000","last_peered":"2023-10-09T04:03:36.211952+0000","last_clean":"2023-10-09T04:03:36.211952+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:03:36.211952+0000","last_undegraded":"2023-10-09T04:03:36.211952+0000","last_fullsized":"2023-10-09T04:03:36.211952+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220381+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:03:37.220381+0000","last_peered":"2023-10-09T04:03:37.220381+0000","last_clean":"2023-10-09T04:03:37.220381+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220381+0000","last_undegraded":"2023-10-09T04:03:37.220381+0000","last_fullsized":"2023-10-09T04:03:37.220381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":71,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211615+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:03:36.211615+0000","last_peered":"2023-10-09T04:03:36.211615+0000","last_clean":"2023-10-09T04:03:36.211615+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:03:36.211615+0000","last_undegraded":"2023-10-09T04:03:36.211615+0000","last_fullsized":"2023-10-09T04:03:36.211615+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.219140+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:03:37.219140+0000","last_peered":"2023-10-09T04:03:37.219140+0000","last_clean":"2023-10-09T04:03:37.219140+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:03:37.219140+0000","last_undegraded":"2023-10-09T04:03:37.219140+0000","last_fullsized":"2023-10-09T04:03:37.219140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221254+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:03:37.221254+0000","last_peered":"2023-10-09T04:03:37.221254+0000","last_clean":"2023-10-09T04:03:37.221254+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:03:37.221254+0000","last_undegraded":"2023-10-09T04:03:37.221254+0000","last_fullsized":"2023-10-09T04:03:37.221254+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206997+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:03:36.206997+0000","last_peered":"2023-10-09T04:03:36.206997+0000","last_clean":"2023-10-09T04:03:36.206997+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:03:36.206997+0000","last_undegraded":"2023-10-09T04:03:36.206997+0000","last_fullsized":"2023-10-09T04:03:36.206997+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206551+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:03:36.206551+0000","last_peered":"2023-10-09T04:03:36.206551+0000","last_clean":"2023-10-09T04:03:36.206551+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:03:36.206551+0000","last_undegraded":"2023-10-09T04:03:36.206551+0000","last_fullsized":"2023-10-09T04:03:36.206551+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208191+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:03:36.208191+0000","last_peered":"2023-10-09T04:03:36.208191+0000","last_clean":"2023-10-09T04:03:36.208191+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:03:36.208191+0000","last_undegraded":"2023-10-09T04:03:36.208191+0000","last_fullsized":"2023-10-09T04:03:36.208191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206838+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:03:36.206838+0000","last_peered":"2023-10-09T04:03:36.206838+0000","last_clean":"2023-10-09T04:03:36.206838+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:03:36.206838+0000","last_undegraded":"2023-10-09T04:03:36.206838+0000","last_fullsized":"2023-10-09T04:03:36.206838+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225364+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:03:37.225364+0000","last_peered":"2023-10-09T04:03:37.225364+0000","last_clean":"2023-10-09T04:03:37.225364+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:03:37.225364+0000","last_undegraded":"2023-10-09T04:03:37.225364+0000","last_fullsized":"2023-10-09T04:03:37.225364+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220457+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:03:37.220457+0000","last_peered":"2023-10-09T04:03:37.220457+0000","last_clean":"2023-10-09T04:03:37.220457+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:03:37.220457+0000","last_undegraded":"2023-10-09T04:03:37.220457+0000","last_fullsized":"2023-10-09T04:03:37.220457+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207539+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:03:37.207539+0000","last_peered":"2023-10-09T04:03:37.207539+0000","last_clean":"2023-10-09T04:03:37.207539+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:03:37.207539+0000","last_undegraded":"2023-10-09T04:03:37.207539+0000","last_fullsized":"2023-10-09T04:03:37.207539+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221861+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:03:37.221861+0000","last_peered":"2023-10-09T04:03:37.221861+0000","last_clean":"2023-10-09T04:03:37.221861+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:03:37.221861+0000","last_undegraded":"2023-10-09T04:03:37.221861+0000","last_fullsized":"2023-10-09T04:03:37.221861+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206686+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:03:36.206686+0000","last_peered":"2023-10-09T04:03:36.206686+0000","last_clean":"2023-10-09T04:03:36.206686+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:03:36.206686+0000","last_undegraded":"2023-10-09T04:03:36.206686+0000","last_fullsized":"2023-10-09T04:03:36.206686+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214808+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:03:36.214808+0000","last_peered":"2023-10-09T04:03:36.214808+0000","last_clean":"2023-10-09T04:03:36.214808+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:03:36.214808+0000","last_undegraded":"2023-10-09T04:03:36.214808+0000","last_fullsized":"2023-10-09T04:03:36.214808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225348+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:03:37.225348+0000","last_peered":"2023-10-09T04:03:37.225348+0000","last_clean":"2023-10-09T04:03:37.225348+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:03:37.225348+0000","last_undegraded":"2023-10-09T04:03:37.225348+0000","last_fullsized":"2023-10-09T04:03:37.225348+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211069+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:03:36.211069+0000","last_peered":"2023-10-09T04:03:36.211069+0000","last_clean":"2023-10-09T04:03:36.211069+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:03:36.211069+0000","last_undegraded":"2023-10-09T04:03:36.211069+0000","last_fullsized":"2023-10-09T04:03:36.211069+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211887+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:03:36.211887+0000","last_peered":"2023-10-09T04:03:36.211887+0000","last_clean":"2023-10-09T04:03:36.211887+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:03:36.211887+0000","last_undegraded":"2023-10-09T04:03:36.211887+0000","last_fullsized":"2023-10-09T04:03:36.211887+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213608+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:03:36.213608+0000","last_peered":"2023-10-09T04:03:36.213608+0000","last_clean":"2023-10-09T04:03:36.213608+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:03:36.213608+0000","last_undegraded":"2023-10-09T04:03:36.213608+0000","last_fullsized":"2023-10-09T04:03:36.213608+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217150+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:03:37.217150+0000","last_peered":"2023-10-09T04:03:37.217150+0000","last_clean":"2023-10-09T04:03:37.217150+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:03:37.217150+0000","last_undegraded":"2023-10-09T04:03:37.217150+0000","last_fullsized":"2023-10-09T04:03:37.217150+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206089+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:03:36.206089+0000","last_peered":"2023-10-09T04:03:36.206089+0000","last_clean":"2023-10-09T04:03:36.206089+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:03:36.206089+0000","last_undegraded":"2023-10-09T04:03:36.206089+0000","last_fullsized":"2023-10-09T04:03:36.206089+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223252+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:03:37.223252+0000","last_peered":"2023-10-09T04:03:37.223252+0000","last_clean":"2023-10-09T04:03:37.223252+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:03:37.223252+0000","last_undegraded":"2023-10-09T04:03:37.223252+0000","last_fullsized":"2023-10-09T04:03:37.223252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.210575+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:03:36.210575+0000","last_peered":"2023-10-09T04:03:36.210575+0000","last_clean":"2023-10-09T04:03:36.210575+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:03:36.210575+0000","last_undegraded":"2023-10-09T04:03:36.210575+0000","last_fullsized":"2023-10-09T04:03:36.210575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220488+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:03:37.220488+0000","last_peered":"2023-10-09T04:03:37.220488+0000","last_clean":"2023-10-09T04:03:37.220488+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:03:37.220488+0000","last_undegraded":"2023-10-09T04:03:37.220488+0000","last_fullsized":"2023-10-09T04:03:37.220488+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220100+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:03:37.220100+0000","last_peered":"2023-10-09T04:03:37.220100+0000","last_clean":"2023-10-09T04:03:37.220100+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:03:37.220100+0000","last_undegraded":"2023-10-09T04:03:37.220100+0000","last_fullsized":"2023-10-09T04:03:37.220100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.222047+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:03:37.222047+0000","last_peered":"2023-10-09T04:03:37.222047+0000","last_clean":"2023-10-09T04:03:37.222047+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:03:37.222047+0000","last_undegraded":"2023-10-09T04:03:37.222047+0000","last_fullsized":"2023-10-09T04:03:37.222047+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211450+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:03:36.211450+0000","last_peered":"2023-10-09T04:03:36.211450+0000","last_clean":"2023-10-09T04:03:36.211450+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:03:36.211450+0000","last_undegraded":"2023-10-09T04:03:36.211450+0000","last_fullsized":"2023-10-09T04:03:36.211450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207346+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:03:36.207346+0000","last_peered":"2023-10-09T04:03:36.207346+0000","last_clean":"2023-10-09T04:03:36.207346+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:03:36.207346+0000","last_undegraded":"2023-10-09T04:03:36.207346+0000","last_fullsized":"2023-10-09T04:03:36.207346+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208541+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:03:36.208541+0000","last_peered":"2023-10-09T04:03:36.208541+0000","last_clean":"2023-10-09T04:03:36.208541+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:03:36.208541+0000","last_undegraded":"2023-10-09T04:03:36.208541+0000","last_fullsized":"2023-10-09T04:03:36.208541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211626+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:03:36.211626+0000","last_peered":"2023-10-09T04:03:36.211626+0000","last_clean":"2023-10-09T04:03:36.211626+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:03:36.211626+0000","last_undegraded":"2023-10-09T04:03:36.211626+0000","last_fullsized":"2023-10-09T04:03:36.211626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220454+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:03:37.220454+0000","last_peered":"2023-10-09T04:03:37.220454+0000","last_clean":"2023-10-09T04:03:37.220454+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:03:37.220454+0000","last_undegraded":"2023-10-09T04:03:37.220454+0000","last_fullsized":"2023-10-09T04:03:37.220454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221468+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:03:37.221468+0000","last_peered":"2023-10-09T04:03:37.221468+0000","last_clean":"2023-10-09T04:03:37.221468+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:03:37.221468+0000","last_undegraded":"2023-10-09T04:03:37.221468+0000","last_fullsized":"2023-10-09T04:03:37.221468+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212749+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:03:36.212749+0000","last_peered":"2023-10-09T04:03:36.212749+0000","last_clean":"2023-10-09T04:03:36.212749+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:03:36.212749+0000","last_undegraded":"2023-10-09T04:03:36.212749+0000","last_fullsized":"2023-10-09T04:03:36.212749+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221476+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:03:37.221476+0000","last_peered":"2023-10-09T04:03:37.221476+0000","last_clean":"2023-10-09T04:03:37.221476+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:03:37.221476+0000","last_undegraded":"2023-10-09T04:03:37.221476+0000","last_fullsized":"2023-10-09T04:03:37.221476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217414+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:03:37.217414+0000","last_peered":"2023-10-09T04:03:37.217414+0000","last_clean":"2023-10-09T04:03:37.217414+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:03:37.217414+0000","last_undegraded":"2023-10-09T04:03:37.217414+0000","last_fullsized":"2023-10-09T04:03:37.217414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208807+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:03:36.208807+0000","last_peered":"2023-10-09T04:03:36.208807+0000","last_clean":"2023-10-09T04:03:36.208807+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:03:36.208807+0000","last_undegraded":"2023-10-09T04:03:36.208807+0000","last_fullsized":"2023-10-09T04:03:36.208807+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.216503+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:03:37.216503+0000","last_peered":"2023-10-09T04:03:37.216503+0000","last_clean":"2023-10-09T04:03:37.216503+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:03:37.216503+0000","last_undegraded":"2023-10-09T04:03:37.216503+0000","last_fullsized":"2023-10-09T04:03:37.216503+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223692+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:03:37.223692+0000","last_peered":"2023-10-09T04:03:37.223692+0000","last_clean":"2023-10-09T04:03:37.223692+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:03:37.223692+0000","last_undegraded":"2023-10-09T04:03:37.223692+0000","last_fullsized":"2023-10-09T04:03:37.223692+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220333+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:03:37.220333+0000","last_peered":"2023-10-09T04:03:37.220333+0000","last_clean":"2023-10-09T04:03:37.220333+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220333+0000","last_undegraded":"2023-10-09T04:03:37.220333+0000","last_fullsized":"2023-10-09T04:03:37.220333+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220097+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:03:37.220097+0000","last_peered":"2023-10-09T04:03:37.220097+0000","last_clean":"2023-10-09T04:03:37.220097+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:03:37.220097+0000","last_undegraded":"2023-10-09T04:03:37.220097+0000","last_fullsized":"2023-10-09T04:03:37.220097+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":77,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208233+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:03:36.208233+0000","last_peered":"2023-10-09T04:03:36.208233+0000","last_clean":"2023-10-09T04:03:36.208233+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:03:36.208233+0000","last_undegraded":"2023-10-09T04:03:36.208233+0000","last_fullsized":"2023-10-09T04:03:36.208233+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201422,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1136082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.56100000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.363,"5min":3.363,"15min":3.363},"last":0.69699999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.746},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.72599999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.58999999999999997,"15min":0.57099999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33000000000000002},"max":{"1min":2.105,"5min":2.105,"15min":3.7709999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.44800000000000001,"5min":0.39400000000000002,"15min":0.33500000000000002},"max":{"1min":0.78500000000000003,"5min":0.82799999999999996,"15min":3.6190000000000002},"last":0.63500000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.5,"15min":0.51500000000000001},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.28599999999999998},"max":{"1min":0.629,"5min":1.0169999999999999,"15min":6.3129999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53200000000000003,"15min":0.52100000000000002},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.27500000000000002},"max":{"1min":0.71699999999999997,"5min":0.81100000000000005,"15min":3.8769999999999998},"last":0.67000000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":0.67100000000000004,"5min":1.4330000000000001,"15min":1.464},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.44400000000000001,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.95599999999999996,"15min":3.1840000000000002},"last":0.65300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.53600000000000003},"min":{"1min":0.40300000000000002,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.92500000000000004,"15min":1.6890000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.46100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.70099999999999996,"5min":0.82199999999999995,"15min":1.9350000000000001},"last":0.58999999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463059,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1135613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.67400000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53700000000000003,"15min":0.53600000000000003},"min":{"1min":0.371,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.40400000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.44600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.46500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56799999999999995,"15min":0.54800000000000004},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":0.79700000000000004,"5min":0.81299999999999994,"15min":1.1120000000000001},"last":0.501},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.34300000000000003},"max":{"1min":0.69399999999999995,"5min":0.77200000000000002,"15min":1.3839999999999999},"last":0.42799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46899999999999997,"15min":0.45000000000000001},"min":{"1min":0.33800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70099999999999996,"5min":0.74099999999999999,"15min":0.76800000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.505,"15min":0.48599999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":0.73199999999999998,"5min":0.84099999999999997,"15min":1.0569999999999999},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.52000000000000002,"15min":0.499},"min":{"1min":0.36799999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.71899999999999997,"5min":0.84699999999999998,"15min":0.96499999999999997},"last":0.495},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58799999999999997,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":1.034},"last":0.52400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.42499999999999999,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.93999999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.63,"15min":0.59899999999999998},"min":{"1min":0.46200000000000002,"5min":0.42099999999999999,"15min":0.34100000000000003},"max":{"1min":0.78900000000000003,"5min":0.86299999999999999,"15min":1.077},"last":0.48099999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089368,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28484,"kb_used_data":1020,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716668,"statfs":{"total":95995035648,"available":95965868032,"internally_reserved":0,"allocated":1044480,"data_stored":549351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.33500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.30499999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.552,"5min":2.552,"15min":2.552},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.6379999999999999,"5min":2.6379999999999999,"15min":2.6379999999999999},"last":0.57299999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.504,"5min":2.504,"15min":2.504},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":2.6859999999999999,"5min":2.6859999999999999,"15min":2.6859999999999999},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.5339999999999998,"5min":2.5339999999999998,"15min":2.5339999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.601,"5min":2.601,"15min":2.601},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.6659999999999999,"5min":2.6659999999999999,"15min":2.6659999999999999},"last":0.68300000000000005}]}]},{"osd":0,"up_from":84,"seq":360777252894,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29136,"kb_used_data":1608,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716016,"statfs":{"total":95995035648,"available":95965200384,"internally_reserved":0,"allocated":1646592,"data_stored":1138279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.224},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.312}]},{"osd":4,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61399999999999999,"15min":0.60999999999999999},"min":{"1min":0.52200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.628,"15min":0.625},"min":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.443,"5min":0.44,"15min":0.44},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.46600000000000003,"5min":0.435,"15min":0.435},"max":{"1min":0.92900000000000005,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.64900000000000002,"15min":0.64700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.65500000000000003,"15min":0.65300000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.67200000000000004,"15min":0.66900000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.53800000000000003}]}]},{"osd":2,"up_from":94,"seq":403726925841,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28604,"kb_used_data":1012,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716548,"statfs":{"total":95995035648,"available":95965745152,"internally_reserved":0,"allocated":1036288,"data_stored":547077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.248}]},{"osd":1,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.33200000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.375}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.50900000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762315,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28464,"kb_used_data":1000,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716688,"statfs":{"total":95995035648,"available":95965888512,"internally_reserved":0,"allocated":1024000,"data_stored":545297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39000000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34200000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35799999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986331,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302380,"kb_used_data":996,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442772,"statfs":{"total":95995035648,"available":95685398528,"internally_reserved":0,"allocated":1019904,"data_stored":545379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.68600000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59199999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.44600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:02:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58799999999999997,"15min":0.55900000000000005},"min":{"1min":0.46300000000000002,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":0.74299999999999999,"5min":1.117,"15min":1.117},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.61499999999999999,"15min":0.57899999999999996},"min":{"1min":0.46800000000000003,"5min":0.378,"15min":0.33100000000000002},"max":{"1min":0.76300000000000001,"5min":1.046,"15min":1.581},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:03:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48599999999999999,"15min":0.442},"min":{"1min":0.34899999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.73799999999999999,"5min":1.038,"15min":1.038},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.52800000000000002,"15min":0.47399999999999998},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.24399999999999999},"max":{"1min":0.67200000000000004,"5min":0.79500000000000004,"15min":0.85499999999999998},"last":0.32000000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56399999999999995,"15min":0.52200000000000002},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.27900000000000003},"max":{"1min":0.75900000000000001,"5min":0.84299999999999997,"15min":0.92700000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57399999999999995,"15min":0.52500000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32800000000000001},"max":{"1min":0.70899999999999996,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.34599999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57899999999999996,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.33900000000000002,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60899999999999999,"15min":0.59999999999999998},"min":{"1min":0.438,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.875,"15min":1.079},"last":0.377}]}]},{"osd":5,"up_from":39,"seq":167503724695,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306460,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438692,"statfs":{"total":95995035648,"available":95681220608,"internally_reserved":0,"allocated":1003520,"data_stored":541065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.57799999999999996,"15min":0.59099999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.74099999999999999,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70299999999999996,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54800000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.42799999999999999,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":0.79300000000000004,"5min":0.83299999999999996,"15min":2.1850000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.622,"15min":0.60199999999999998},"min":{"1min":0.47099999999999997,"5min":0.42699999999999999,"15min":0.32400000000000001},"max":{"1min":0.76100000000000001,"5min":0.85999999999999999,"15min":2.2189999999999999},"last":0.53200000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.34499999999999997,"5min":0.32000000000000001,"15min":0.253},"max":{"1min":0.74199999999999999,"5min":0.81200000000000006,"15min":0.85399999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":0.80200000000000005,"5min":0.82599999999999996,"15min":0.86299999999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51800000000000002,"15min":0.51300000000000001},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.27400000000000002},"max":{"1min":0.72399999999999998,"5min":0.81200000000000006,"15min":0.91000000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.55800000000000005,"15min":0.54200000000000004},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.312},"max":{"1min":0.80500000000000005,"5min":0.86099999999999999,"15min":0.89800000000000002},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59199999999999997,"15min":0.58299999999999996},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":0.81899999999999995,"5min":0.84099999999999997,"15min":0.94799999999999995},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.628,"15min":0.61399999999999999},"min":{"1min":0.47099999999999997,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.96399999999999997},"last":0.66200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:04:24.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:04:24.518 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:24 smithi163 ceph-mon[221067]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:24 smithi163 ceph-mon[213841]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:24 smithi181 ceph-mon[196979]: pgmap v220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:25.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:25 smithi181 ceph-mon[196979]: from='client.34686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:25.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:25 smithi163 ceph-mon[221067]: from='client.34686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:25.784 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:25 smithi163 ceph-mon[213841]: from='client.34686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:04:26.178 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:04:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:26 smithi163 ceph-mon[213841]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:26 smithi163 ceph-mon[221067]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:26 smithi181 ceph-mon[196979]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:26.806 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2023-10-09T04:04:25.264443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333544,"kb_used_data":9768,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627672,"statfs":{"total":767960285184,"available":766594736128,"internally_reserved":0,"allocated":10002432,"data_stored":6138143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002975"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206546+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:03:36.206546+0000","last_peered":"2023-10-09T04:03:36.206546+0000","last_clean":"2023-10-09T04:03:36.206546+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:03:36.206546+0000","last_undegraded":"2023-10-09T04:03:36.206546+0000","last_fullsized":"2023-10-09T04:03:36.206546+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207470+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:03:37.207470+0000","last_peered":"2023-10-09T04:03:37.207470+0000","last_clean":"2023-10-09T04:03:37.207470+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:03:37.207470+0000","last_undegraded":"2023-10-09T04:03:37.207470+0000","last_fullsized":"2023-10-09T04:03:37.207470+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215833+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:03:37.215833+0000","last_peered":"2023-10-09T04:03:37.215833+0000","last_clean":"2023-10-09T04:03:37.215833+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:03:37.215833+0000","last_undegraded":"2023-10-09T04:03:37.215833+0000","last_fullsized":"2023-10-09T04:03:37.215833+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221959+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:03:37.221959+0000","last_peered":"2023-10-09T04:03:37.221959+0000","last_clean":"2023-10-09T04:03:37.221959+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:03:37.221959+0000","last_undegraded":"2023-10-09T04:03:37.221959+0000","last_fullsized":"2023-10-09T04:03:37.221959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.224382+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:03:37.224382+0000","last_peered":"2023-10-09T04:03:37.224382+0000","last_clean":"2023-10-09T04:03:37.224382+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:03:37.224382+0000","last_undegraded":"2023-10-09T04:03:37.224382+0000","last_fullsized":"2023-10-09T04:03:37.224382+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213576+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:03:36.213576+0000","last_peered":"2023-10-09T04:03:36.213576+0000","last_clean":"2023-10-09T04:03:36.213576+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:03:36.213576+0000","last_undegraded":"2023-10-09T04:03:36.213576+0000","last_fullsized":"2023-10-09T04:03:36.213576+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225542+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:03:37.225542+0000","last_peered":"2023-10-09T04:03:37.225542+0000","last_clean":"2023-10-09T04:03:37.225542+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:03:37.225542+0000","last_undegraded":"2023-10-09T04:03:37.225542+0000","last_fullsized":"2023-10-09T04:03:37.225542+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212908+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:03:36.212908+0000","last_peered":"2023-10-09T04:03:36.212908+0000","last_clean":"2023-10-09T04:03:36.212908+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:03:36.212908+0000","last_undegraded":"2023-10-09T04:03:36.212908+0000","last_fullsized":"2023-10-09T04:03:36.212908+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215366+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:03:37.215366+0000","last_peered":"2023-10-09T04:03:37.215366+0000","last_clean":"2023-10-09T04:03:37.215366+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:03:37.215366+0000","last_undegraded":"2023-10-09T04:03:37.215366+0000","last_fullsized":"2023-10-09T04:03:37.215366+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211445+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:03:36.211445+0000","last_peered":"2023-10-09T04:03:36.211445+0000","last_clean":"2023-10-09T04:03:36.211445+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:03:36.211445+0000","last_undegraded":"2023-10-09T04:03:36.211445+0000","last_fullsized":"2023-10-09T04:03:36.211445+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206640+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:03:36.206640+0000","last_peered":"2023-10-09T04:03:36.206640+0000","last_clean":"2023-10-09T04:03:36.206640+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:03:36.206640+0000","last_undegraded":"2023-10-09T04:03:36.206640+0000","last_fullsized":"2023-10-09T04:03:36.206640+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208063+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:03:37.208063+0000","last_peered":"2023-10-09T04:03:37.208063+0000","last_clean":"2023-10-09T04:03:37.208063+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:03:37.208063+0000","last_undegraded":"2023-10-09T04:03:37.208063+0000","last_fullsized":"2023-10-09T04:03:37.208063+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208745+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:03:36.208745+0000","last_peered":"2023-10-09T04:03:36.208745+0000","last_clean":"2023-10-09T04:03:36.208745+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:03:36.208745+0000","last_undegraded":"2023-10-09T04:03:36.208745+0000","last_fullsized":"2023-10-09T04:03:36.208745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221060+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:03:37.221060+0000","last_peered":"2023-10-09T04:03:37.221060+0000","last_clean":"2023-10-09T04:03:37.221060+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:03:37.221060+0000","last_undegraded":"2023-10-09T04:03:37.221060+0000","last_fullsized":"2023-10-09T04:03:37.221060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211820+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:03:36.211820+0000","last_peered":"2023-10-09T04:03:36.211820+0000","last_clean":"2023-10-09T04:03:36.211820+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:03:36.211820+0000","last_undegraded":"2023-10-09T04:03:36.211820+0000","last_fullsized":"2023-10-09T04:03:36.211820+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217253+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:03:37.217253+0000","last_peered":"2023-10-09T04:03:37.217253+0000","last_clean":"2023-10-09T04:03:37.217253+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:03:37.217253+0000","last_undegraded":"2023-10-09T04:03:37.217253+0000","last_fullsized":"2023-10-09T04:03:37.217253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207368+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:03:37.207368+0000","last_peered":"2023-10-09T04:03:37.207368+0000","last_clean":"2023-10-09T04:03:37.207368+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:03:37.207368+0000","last_undegraded":"2023-10-09T04:03:37.207368+0000","last_fullsized":"2023-10-09T04:03:37.207368+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211705+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:03:36.211705+0000","last_peered":"2023-10-09T04:03:36.211705+0000","last_clean":"2023-10-09T04:03:36.211705+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:03:36.211705+0000","last_undegraded":"2023-10-09T04:03:36.211705+0000","last_fullsized":"2023-10-09T04:03:36.211705+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214698+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:03:36.214698+0000","last_peered":"2023-10-09T04:03:36.214698+0000","last_clean":"2023-10-09T04:03:36.214698+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:03:36.214698+0000","last_undegraded":"2023-10-09T04:03:36.214698+0000","last_fullsized":"2023-10-09T04:03:36.214698+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208137+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:03:37.208137+0000","last_peered":"2023-10-09T04:03:37.208137+0000","last_clean":"2023-10-09T04:03:37.208137+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:03:37.208137+0000","last_undegraded":"2023-10-09T04:03:37.208137+0000","last_fullsized":"2023-10-09T04:03:37.208137+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220143+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:03:37.220143+0000","last_peered":"2023-10-09T04:03:37.220143+0000","last_clean":"2023-10-09T04:03:37.220143+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:03:37.220143+0000","last_undegraded":"2023-10-09T04:03:37.220143+0000","last_fullsized":"2023-10-09T04:03:37.220143+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212510+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:03:36.212510+0000","last_peered":"2023-10-09T04:03:36.212510+0000","last_clean":"2023-10-09T04:03:36.212510+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:03:36.212510+0000","last_undegraded":"2023-10-09T04:03:36.212510+0000","last_fullsized":"2023-10-09T04:03:36.212510+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.209067+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:03:36.209067+0000","last_peered":"2023-10-09T04:03:36.209067+0000","last_clean":"2023-10-09T04:03:36.209067+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:03:36.209067+0000","last_undegraded":"2023-10-09T04:03:36.209067+0000","last_fullsized":"2023-10-09T04:03:36.209067+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221180+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:03:37.221180+0000","last_peered":"2023-10-09T04:03:37.221180+0000","last_clean":"2023-10-09T04:03:37.221180+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:03:37.221180+0000","last_undegraded":"2023-10-09T04:03:37.221180+0000","last_fullsized":"2023-10-09T04:03:37.221180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":440,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212413+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:03:36.212413+0000","last_peered":"2023-10-09T04:03:36.212413+0000","last_clean":"2023-10-09T04:03:36.212413+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:03:36.212413+0000","last_undegraded":"2023-10-09T04:03:36.212413+0000","last_fullsized":"2023-10-09T04:03:36.212413+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211739+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:03:36.211739+0000","last_peered":"2023-10-09T04:03:36.211739+0000","last_clean":"2023-10-09T04:03:36.211739+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:03:36.211739+0000","last_undegraded":"2023-10-09T04:03:36.211739+0000","last_fullsized":"2023-10-09T04:03:36.211739+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.218572+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:03:36.218572+0000","last_peered":"2023-10-09T04:03:36.218572+0000","last_clean":"2023-10-09T04:03:36.218572+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:03:36.218572+0000","last_undegraded":"2023-10-09T04:03:36.218572+0000","last_fullsized":"2023-10-09T04:03:36.218572+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213218+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:03:36.213218+0000","last_peered":"2023-10-09T04:03:36.213218+0000","last_clean":"2023-10-09T04:03:36.213218+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:03:36.213218+0000","last_undegraded":"2023-10-09T04:03:36.213218+0000","last_fullsized":"2023-10-09T04:03:36.213218+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215356+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:03:37.215356+0000","last_peered":"2023-10-09T04:03:37.215356+0000","last_clean":"2023-10-09T04:03:37.215356+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:03:37.215356+0000","last_undegraded":"2023-10-09T04:03:37.215356+0000","last_fullsized":"2023-10-09T04:03:37.215356+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.218452+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:03:37.218452+0000","last_peered":"2023-10-09T04:03:37.218452+0000","last_clean":"2023-10-09T04:03:37.218452+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:03:37.218452+0000","last_undegraded":"2023-10-09T04:03:37.218452+0000","last_fullsized":"2023-10-09T04:03:37.218452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207441+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:03:36.207441+0000","last_peered":"2023-10-09T04:03:36.207441+0000","last_clean":"2023-10-09T04:03:36.207441+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:03:36.207441+0000","last_undegraded":"2023-10-09T04:03:36.207441+0000","last_fullsized":"2023-10-09T04:03:36.207441+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214621+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:03:36.214621+0000","last_peered":"2023-10-09T04:03:36.214621+0000","last_clean":"2023-10-09T04:03:36.214621+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:03:36.214621+0000","last_undegraded":"2023-10-09T04:03:36.214621+0000","last_fullsized":"2023-10-09T04:03:36.214621+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211952+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:03:36.211952+0000","last_peered":"2023-10-09T04:03:36.211952+0000","last_clean":"2023-10-09T04:03:36.211952+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:03:36.211952+0000","last_undegraded":"2023-10-09T04:03:36.211952+0000","last_fullsized":"2023-10-09T04:03:36.211952+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220381+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:03:37.220381+0000","last_peered":"2023-10-09T04:03:37.220381+0000","last_clean":"2023-10-09T04:03:37.220381+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220381+0000","last_undegraded":"2023-10-09T04:03:37.220381+0000","last_fullsized":"2023-10-09T04:03:37.220381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":71,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211615+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:03:36.211615+0000","last_peered":"2023-10-09T04:03:36.211615+0000","last_clean":"2023-10-09T04:03:36.211615+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:03:36.211615+0000","last_undegraded":"2023-10-09T04:03:36.211615+0000","last_fullsized":"2023-10-09T04:03:36.211615+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.219140+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:03:37.219140+0000","last_peered":"2023-10-09T04:03:37.219140+0000","last_clean":"2023-10-09T04:03:37.219140+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:03:37.219140+0000","last_undegraded":"2023-10-09T04:03:37.219140+0000","last_fullsized":"2023-10-09T04:03:37.219140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221254+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:03:37.221254+0000","last_peered":"2023-10-09T04:03:37.221254+0000","last_clean":"2023-10-09T04:03:37.221254+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:03:37.221254+0000","last_undegraded":"2023-10-09T04:03:37.221254+0000","last_fullsized":"2023-10-09T04:03:37.221254+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206997+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:03:36.206997+0000","last_peered":"2023-10-09T04:03:36.206997+0000","last_clean":"2023-10-09T04:03:36.206997+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:03:36.206997+0000","last_undegraded":"2023-10-09T04:03:36.206997+0000","last_fullsized":"2023-10-09T04:03:36.206997+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206551+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:03:36.206551+0000","last_peered":"2023-10-09T04:03:36.206551+0000","last_clean":"2023-10-09T04:03:36.206551+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:03:36.206551+0000","last_undegraded":"2023-10-09T04:03:36.206551+0000","last_fullsized":"2023-10-09T04:03:36.206551+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208191+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:03:36.208191+0000","last_peered":"2023-10-09T04:03:36.208191+0000","last_clean":"2023-10-09T04:03:36.208191+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:03:36.208191+0000","last_undegraded":"2023-10-09T04:03:36.208191+0000","last_fullsized":"2023-10-09T04:03:36.208191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206838+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:03:36.206838+0000","last_peered":"2023-10-09T04:03:36.206838+0000","last_clean":"2023-10-09T04:03:36.206838+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:03:36.206838+0000","last_undegraded":"2023-10-09T04:03:36.206838+0000","last_fullsized":"2023-10-09T04:03:36.206838+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225364+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:03:37.225364+0000","last_peered":"2023-10-09T04:03:37.225364+0000","last_clean":"2023-10-09T04:03:37.225364+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:03:37.225364+0000","last_undegraded":"2023-10-09T04:03:37.225364+0000","last_fullsized":"2023-10-09T04:03:37.225364+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220457+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:03:37.220457+0000","last_peered":"2023-10-09T04:03:37.220457+0000","last_clean":"2023-10-09T04:03:37.220457+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:03:37.220457+0000","last_undegraded":"2023-10-09T04:03:37.220457+0000","last_fullsized":"2023-10-09T04:03:37.220457+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207539+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:03:37.207539+0000","last_peered":"2023-10-09T04:03:37.207539+0000","last_clean":"2023-10-09T04:03:37.207539+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:03:37.207539+0000","last_undegraded":"2023-10-09T04:03:37.207539+0000","last_fullsized":"2023-10-09T04:03:37.207539+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221861+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:03:37.221861+0000","last_peered":"2023-10-09T04:03:37.221861+0000","last_clean":"2023-10-09T04:03:37.221861+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:03:37.221861+0000","last_undegraded":"2023-10-09T04:03:37.221861+0000","last_fullsized":"2023-10-09T04:03:37.221861+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206686+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:03:36.206686+0000","last_peered":"2023-10-09T04:03:36.206686+0000","last_clean":"2023-10-09T04:03:36.206686+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:03:36.206686+0000","last_undegraded":"2023-10-09T04:03:36.206686+0000","last_fullsized":"2023-10-09T04:03:36.206686+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214808+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:03:36.214808+0000","last_peered":"2023-10-09T04:03:36.214808+0000","last_clean":"2023-10-09T04:03:36.214808+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:03:36.214808+0000","last_undegraded":"2023-10-09T04:03:36.214808+0000","last_fullsized":"2023-10-09T04:03:36.214808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225348+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:03:37.225348+0000","last_peered":"2023-10-09T04:03:37.225348+0000","last_clean":"2023-10-09T04:03:37.225348+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:03:37.225348+0000","last_undegraded":"2023-10-09T04:03:37.225348+0000","last_fullsized":"2023-10-09T04:03:37.225348+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211069+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:03:36.211069+0000","last_peered":"2023-10-09T04:03:36.211069+0000","last_clean":"2023-10-09T04:03:36.211069+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:03:36.211069+0000","last_undegraded":"2023-10-09T04:03:36.211069+0000","last_fullsized":"2023-10-09T04:03:36.211069+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211887+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:03:36.211887+0000","last_peered":"2023-10-09T04:03:36.211887+0000","last_clean":"2023-10-09T04:03:36.211887+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:03:36.211887+0000","last_undegraded":"2023-10-09T04:03:36.211887+0000","last_fullsized":"2023-10-09T04:03:36.211887+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213608+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:03:36.213608+0000","last_peered":"2023-10-09T04:03:36.213608+0000","last_clean":"2023-10-09T04:03:36.213608+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:03:36.213608+0000","last_undegraded":"2023-10-09T04:03:36.213608+0000","last_fullsized":"2023-10-09T04:03:36.213608+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217150+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:03:37.217150+0000","last_peered":"2023-10-09T04:03:37.217150+0000","last_clean":"2023-10-09T04:03:37.217150+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:03:37.217150+0000","last_undegraded":"2023-10-09T04:03:37.217150+0000","last_fullsized":"2023-10-09T04:03:37.217150+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206089+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:03:36.206089+0000","last_peered":"2023-10-09T04:03:36.206089+0000","last_clean":"2023-10-09T04:03:36.206089+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:03:36.206089+0000","last_undegraded":"2023-10-09T04:03:36.206089+0000","last_fullsized":"2023-10-09T04:03:36.206089+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223252+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:03:37.223252+0000","last_peered":"2023-10-09T04:03:37.223252+0000","last_clean":"2023-10-09T04:03:37.223252+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:03:37.223252+0000","last_undegraded":"2023-10-09T04:03:37.223252+0000","last_fullsized":"2023-10-09T04:03:37.223252+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.210575+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:03:36.210575+0000","last_peered":"2023-10-09T04:03:36.210575+0000","last_clean":"2023-10-09T04:03:36.210575+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:03:36.210575+0000","last_undegraded":"2023-10-09T04:03:36.210575+0000","last_fullsized":"2023-10-09T04:03:36.210575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220488+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:03:37.220488+0000","last_peered":"2023-10-09T04:03:37.220488+0000","last_clean":"2023-10-09T04:03:37.220488+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:03:37.220488+0000","last_undegraded":"2023-10-09T04:03:37.220488+0000","last_fullsized":"2023-10-09T04:03:37.220488+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220100+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:03:37.220100+0000","last_peered":"2023-10-09T04:03:37.220100+0000","last_clean":"2023-10-09T04:03:37.220100+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:03:37.220100+0000","last_undegraded":"2023-10-09T04:03:37.220100+0000","last_fullsized":"2023-10-09T04:03:37.220100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.222047+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:03:37.222047+0000","last_peered":"2023-10-09T04:03:37.222047+0000","last_clean":"2023-10-09T04:03:37.222047+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:03:37.222047+0000","last_undegraded":"2023-10-09T04:03:37.222047+0000","last_fullsized":"2023-10-09T04:03:37.222047+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211450+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:03:36.211450+0000","last_peered":"2023-10-09T04:03:36.211450+0000","last_clean":"2023-10-09T04:03:36.211450+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:03:36.211450+0000","last_undegraded":"2023-10-09T04:03:36.211450+0000","last_fullsized":"2023-10-09T04:03:36.211450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207346+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:03:36.207346+0000","last_peered":"2023-10-09T04:03:36.207346+0000","last_clean":"2023-10-09T04:03:36.207346+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:03:36.207346+0000","last_undegraded":"2023-10-09T04:03:36.207346+0000","last_fullsized":"2023-10-09T04:03:36.207346+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208541+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:03:36.208541+0000","last_peered":"2023-10-09T04:03:36.208541+0000","last_clean":"2023-10-09T04:03:36.208541+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:03:36.208541+0000","last_undegraded":"2023-10-09T04:03:36.208541+0000","last_fullsized":"2023-10-09T04:03:36.208541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211626+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:03:36.211626+0000","last_peered":"2023-10-09T04:03:36.211626+0000","last_clean":"2023-10-09T04:03:36.211626+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:03:36.211626+0000","last_undegraded":"2023-10-09T04:03:36.211626+0000","last_fullsized":"2023-10-09T04:03:36.211626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220454+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:03:37.220454+0000","last_peered":"2023-10-09T04:03:37.220454+0000","last_clean":"2023-10-09T04:03:37.220454+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:03:37.220454+0000","last_undegraded":"2023-10-09T04:03:37.220454+0000","last_fullsized":"2023-10-09T04:03:37.220454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221468+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:03:37.221468+0000","last_peered":"2023-10-09T04:03:37.221468+0000","last_clean":"2023-10-09T04:03:37.221468+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:03:37.221468+0000","last_undegraded":"2023-10-09T04:03:37.221468+0000","last_fullsized":"2023-10-09T04:03:37.221468+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212749+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:03:36.212749+0000","last_peered":"2023-10-09T04:03:36.212749+0000","last_clean":"2023-10-09T04:03:36.212749+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:03:36.212749+0000","last_undegraded":"2023-10-09T04:03:36.212749+0000","last_fullsized":"2023-10-09T04:03:36.212749+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221476+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:03:37.221476+0000","last_peered":"2023-10-09T04:03:37.221476+0000","last_clean":"2023-10-09T04:03:37.221476+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:03:37.221476+0000","last_undegraded":"2023-10-09T04:03:37.221476+0000","last_fullsized":"2023-10-09T04:03:37.221476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217414+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:03:37.217414+0000","last_peered":"2023-10-09T04:03:37.217414+0000","last_clean":"2023-10-09T04:03:37.217414+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:03:37.217414+0000","last_undegraded":"2023-10-09T04:03:37.217414+0000","last_fullsized":"2023-10-09T04:03:37.217414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208807+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:03:36.208807+0000","last_peered":"2023-10-09T04:03:36.208807+0000","last_clean":"2023-10-09T04:03:36.208807+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:03:36.208807+0000","last_undegraded":"2023-10-09T04:03:36.208807+0000","last_fullsized":"2023-10-09T04:03:36.208807+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.216503+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:03:37.216503+0000","last_peered":"2023-10-09T04:03:37.216503+0000","last_clean":"2023-10-09T04:03:37.216503+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:03:37.216503+0000","last_undegraded":"2023-10-09T04:03:37.216503+0000","last_fullsized":"2023-10-09T04:03:37.216503+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223692+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:03:37.223692+0000","last_peered":"2023-10-09T04:03:37.223692+0000","last_clean":"2023-10-09T04:03:37.223692+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:03:37.223692+0000","last_undegraded":"2023-10-09T04:03:37.223692+0000","last_fullsized":"2023-10-09T04:03:37.223692+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220333+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:03:37.220333+0000","last_peered":"2023-10-09T04:03:37.220333+0000","last_clean":"2023-10-09T04:03:37.220333+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220333+0000","last_undegraded":"2023-10-09T04:03:37.220333+0000","last_fullsized":"2023-10-09T04:03:37.220333+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220097+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:03:37.220097+0000","last_peered":"2023-10-09T04:03:37.220097+0000","last_clean":"2023-10-09T04:03:37.220097+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:03:37.220097+0000","last_undegraded":"2023-10-09T04:03:37.220097+0000","last_fullsized":"2023-10-09T04:03:37.220097+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":77,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208233+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:03:36.208233+0000","last_peered":"2023-10-09T04:03:36.208233+0000","last_clean":"2023-10-09T04:03:36.208233+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:03:36.208233+0000","last_undegraded":"2023-10-09T04:03:36.208233+0000","last_fullsized":"2023-10-09T04:03:36.208233+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201423,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1136082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.627}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.363,"5min":3.363,"15min":3.363},"last":0.36399999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.46700000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.58999999999999997,"15min":0.57099999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33000000000000002},"max":{"1min":2.105,"5min":2.105,"15min":3.7709999999999999},"last":0.502},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.44800000000000001,"5min":0.39400000000000002,"15min":0.33500000000000002},"max":{"1min":0.78500000000000003,"5min":0.82799999999999996,"15min":3.6190000000000002},"last":0.52200000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.5,"15min":0.51500000000000001},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.28599999999999998},"max":{"1min":0.629,"5min":1.0169999999999999,"15min":6.3129999999999997},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53200000000000003,"15min":0.52100000000000002},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.27500000000000002},"max":{"1min":0.71699999999999997,"5min":0.81100000000000005,"15min":3.8769999999999998},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":0.67100000000000004,"5min":1.4330000000000001,"15min":1.464},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.44400000000000001,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.95599999999999996,"15min":3.1840000000000002},"last":0.64100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.53600000000000003},"min":{"1min":0.40300000000000002,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.92500000000000004,"15min":1.6890000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.46100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.70099999999999996,"5min":0.82199999999999995,"15min":1.9350000000000001},"last":0.60099999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463060,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1135613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.67400000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53700000000000003,"15min":0.53600000000000003},"min":{"1min":0.371,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.69999999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.622},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.67600000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.75}]},{"osd":3,"last update":"Mon Oct 9 04:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56799999999999995,"15min":0.54800000000000004},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":0.79700000000000004,"5min":0.81299999999999994,"15min":1.1120000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.34300000000000003},"max":{"1min":0.69399999999999995,"5min":0.77200000000000002,"15min":1.3839999999999999},"last":0.71099999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46899999999999997,"15min":0.45000000000000001},"min":{"1min":0.33800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70099999999999996,"5min":0.74099999999999999,"15min":0.76800000000000002},"last":0.504},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.505,"15min":0.48599999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":0.73199999999999998,"5min":0.84099999999999997,"15min":1.0569999999999999},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.52000000000000002,"15min":0.499},"min":{"1min":0.36799999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.71899999999999997,"5min":0.84699999999999998,"15min":0.96499999999999997},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58799999999999997,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":1.034},"last":0.58099999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.42499999999999999,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.93999999999999995},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.63,"15min":0.59899999999999998},"min":{"1min":0.46200000000000002,"5min":0.42099999999999999,"15min":0.34100000000000003},"max":{"1min":0.78900000000000003,"5min":0.86299999999999999,"15min":1.077},"last":0.65600000000000003}]}]},{"osd":1,"up_from":89,"seq":382252089368,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28484,"kb_used_data":1020,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716668,"statfs":{"total":95995035648,"available":95965868032,"internally_reserved":0,"allocated":1044480,"data_stored":549351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.33500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.30499999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.552,"5min":2.552,"15min":2.552},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.6379999999999999,"5min":2.6379999999999999,"15min":2.6379999999999999},"last":0.57299999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.504,"5min":2.504,"15min":2.504},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":2.6859999999999999,"5min":2.6859999999999999,"15min":2.6859999999999999},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.5339999999999998,"5min":2.5339999999999998,"15min":2.5339999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.601,"5min":2.601,"15min":2.601},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.6659999999999999,"5min":2.6659999999999999,"15min":2.6659999999999999},"last":0.68300000000000005}]}]},{"osd":0,"up_from":84,"seq":360777252894,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29136,"kb_used_data":1608,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716016,"statfs":{"total":95995035648,"available":95965200384,"internally_reserved":0,"allocated":1646592,"data_stored":1138279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.224},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.312}]},{"osd":4,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61399999999999999,"15min":0.60999999999999999},"min":{"1min":0.52200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.628,"15min":0.625},"min":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.443,"5min":0.44,"15min":0.44},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.46600000000000003,"5min":0.435,"15min":0.435},"max":{"1min":0.92900000000000005,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.64900000000000002,"15min":0.64700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.65500000000000003,"15min":0.65300000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.67200000000000004,"15min":0.66900000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.53800000000000003}]}]},{"osd":2,"up_from":94,"seq":403726925842,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28604,"kb_used_data":1012,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716548,"statfs":{"total":95995035648,"available":95965745152,"internally_reserved":0,"allocated":1036288,"data_stored":547077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.34100000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.379},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.29499999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36099999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.85299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.79400000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.80800000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.82699999999999996}]}]},{"osd":3,"up_from":99,"seq":425201762316,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28464,"kb_used_data":1000,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716688,"statfs":{"total":95995035648,"available":95965888512,"internally_reserved":0,"allocated":1024000,"data_stored":545297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36799999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33200000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67500000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986332,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302380,"kb_used_data":996,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442772,"statfs":{"total":95995035648,"available":95685398528,"internally_reserved":0,"allocated":1019904,"data_stored":545379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.68600000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.441}]},{"osd":1,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.46600000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:02:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58799999999999997,"15min":0.55900000000000005},"min":{"1min":0.46300000000000002,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":0.74299999999999999,"5min":1.117,"15min":1.117},"last":0.375},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.61499999999999999,"15min":0.57899999999999996},"min":{"1min":0.46800000000000003,"5min":0.378,"15min":0.33100000000000002},"max":{"1min":0.76300000000000001,"5min":1.046,"15min":1.581},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48199999999999998,"15min":0.44900000000000001},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.625,"5min":0.753,"15min":1.038},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.504,"5min":0.53000000000000003,"15min":0.48099999999999998},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.24399999999999999},"max":{"1min":0.67100000000000004,"5min":0.77200000000000002,"15min":0.85499999999999998},"last":0.48199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56399999999999995,"15min":0.52200000000000002},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.27900000000000003},"max":{"1min":0.75900000000000001,"5min":0.84299999999999997,"15min":0.92700000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57399999999999995,"15min":0.52500000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32800000000000001},"max":{"1min":0.70899999999999996,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57899999999999996,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.33900000000000002,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60899999999999999,"15min":0.59999999999999998},"min":{"1min":0.438,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.875,"15min":1.079},"last":0.56100000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724696,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306460,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438692,"statfs":{"total":95995035648,"available":95681220608,"internally_reserved":0,"allocated":1003520,"data_stored":541065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.57799999999999996,"15min":0.59099999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.74099999999999999,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70299999999999996,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.66200000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.74399999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.68799999999999994}]},{"osd":3,"last update":"Mon Oct 9 04:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.42799999999999999,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":0.79300000000000004,"5min":0.83299999999999996,"15min":2.1850000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.622,"15min":0.60199999999999998},"min":{"1min":0.47099999999999997,"5min":0.42699999999999999,"15min":0.32400000000000001},"max":{"1min":0.76100000000000001,"5min":0.85999999999999999,"15min":2.2189999999999999},"last":0.75600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.34499999999999997,"5min":0.32000000000000001,"15min":0.253},"max":{"1min":0.74199999999999999,"5min":0.81200000000000006,"15min":0.85399999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":0.80200000000000005,"5min":0.82599999999999996,"15min":0.86299999999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51800000000000002,"15min":0.51300000000000001},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.27400000000000002},"max":{"1min":0.72399999999999998,"5min":0.81200000000000006,"15min":0.91000000000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.55800000000000005,"15min":0.54200000000000004},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.312},"max":{"1min":0.80500000000000005,"5min":0.86099999999999999,"15min":0.89800000000000002},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59199999999999997,"15min":0.58299999999999996},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":0.81899999999999995,"5min":0.84099999999999997,"15min":0.94799999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.628,"15min":0.61399999999999999},"min":{"1min":0.47099999999999997,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.96399999999999997},"last":0.70199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:04:26.811 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:04:26.812 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:04:26.812 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:04:26.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:04:27.163 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:27.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:27 smithi163 ceph-mon[221067]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:27.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:27 smithi163 ceph-mon[213841]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:27.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:27 smithi181 ceph-mon[196979]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:28.383 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:28 smithi163 ceph-mon[221067]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:28.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:28 smithi163 ceph-mon[213841]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:28.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:28 smithi181 ceph-mon[196979]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:28.810 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:04:28.810 INFO:teuthology.orchestra.run.smithi163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:04:29.471 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:04:29.471 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:04:29.481 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T04:04:29.481 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T04:04:29.490 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T04:04:29.491 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:04:29.491 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T04:04:29.492 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T04:04:29.493 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T04:04:29.493 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T04:04:29.493 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T04:04:29.493 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:04:29.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:04:29.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/258942186' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:04:29.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/258942186' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:04:29.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/258942186' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:04:29.839 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:30.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:30 smithi163 ceph-mon[213841]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:30.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:30 smithi163 ceph-mon[221067]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:30 smithi181 ceph-mon[196979]: pgmap v223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:31.066 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:04:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:04:30] "GET /metrics HTTP/1.1" 200 34044 "" "Prometheus/2.43.0" 2023-10-09T04:04:31.470 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:04:31.471 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:04:32.130 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":224,"stamp":"2023-10-09T04:04:31.265599+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333544,"kb_used_data":9768,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627672,"statfs":{"total":767960285184,"available":766594736128,"internally_reserved":0,"allocated":10002432,"data_stored":6138143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002799"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206546+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:03:36.206546+0000","last_peered":"2023-10-09T04:03:36.206546+0000","last_clean":"2023-10-09T04:03:36.206546+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:03:36.206546+0000","last_undegraded":"2023-10-09T04:03:36.206546+0000","last_fullsized":"2023-10-09T04:03:36.206546+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207470+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:03:37.207470+0000","last_peered":"2023-10-09T04:03:37.207470+0000","last_clean":"2023-10-09T04:03:37.207470+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:03:37.207470+0000","last_undegraded":"2023-10-09T04:03:37.207470+0000","last_fullsized":"2023-10-09T04:03:37.207470+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215833+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:03:37.215833+0000","last_peered":"2023-10-09T04:03:37.215833+0000","last_clean":"2023-10-09T04:03:37.215833+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:03:37.215833+0000","last_undegraded":"2023-10-09T04:03:37.215833+0000","last_fullsized":"2023-10-09T04:03:37.215833+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221959+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:03:37.221959+0000","last_peered":"2023-10-09T04:03:37.221959+0000","last_clean":"2023-10-09T04:03:37.221959+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:03:37.221959+0000","last_undegraded":"2023-10-09T04:03:37.221959+0000","last_fullsized":"2023-10-09T04:03:37.221959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.224382+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:03:37.224382+0000","last_peered":"2023-10-09T04:03:37.224382+0000","last_clean":"2023-10-09T04:03:37.224382+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:03:37.224382+0000","last_undegraded":"2023-10-09T04:03:37.224382+0000","last_fullsized":"2023-10-09T04:03:37.224382+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213576+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:03:36.213576+0000","last_peered":"2023-10-09T04:03:36.213576+0000","last_clean":"2023-10-09T04:03:36.213576+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:03:36.213576+0000","last_undegraded":"2023-10-09T04:03:36.213576+0000","last_fullsized":"2023-10-09T04:03:36.213576+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225542+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:03:37.225542+0000","last_peered":"2023-10-09T04:03:37.225542+0000","last_clean":"2023-10-09T04:03:37.225542+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:03:37.225542+0000","last_undegraded":"2023-10-09T04:03:37.225542+0000","last_fullsized":"2023-10-09T04:03:37.225542+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212908+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:03:36.212908+0000","last_peered":"2023-10-09T04:03:36.212908+0000","last_clean":"2023-10-09T04:03:36.212908+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:03:36.212908+0000","last_undegraded":"2023-10-09T04:03:36.212908+0000","last_fullsized":"2023-10-09T04:03:36.212908+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215366+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:03:37.215366+0000","last_peered":"2023-10-09T04:03:37.215366+0000","last_clean":"2023-10-09T04:03:37.215366+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:03:37.215366+0000","last_undegraded":"2023-10-09T04:03:37.215366+0000","last_fullsized":"2023-10-09T04:03:37.215366+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211445+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:03:36.211445+0000","last_peered":"2023-10-09T04:03:36.211445+0000","last_clean":"2023-10-09T04:03:36.211445+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:03:36.211445+0000","last_undegraded":"2023-10-09T04:03:36.211445+0000","last_fullsized":"2023-10-09T04:03:36.211445+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206640+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:03:36.206640+0000","last_peered":"2023-10-09T04:03:36.206640+0000","last_clean":"2023-10-09T04:03:36.206640+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:03:36.206640+0000","last_undegraded":"2023-10-09T04:03:36.206640+0000","last_fullsized":"2023-10-09T04:03:36.206640+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208063+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:03:37.208063+0000","last_peered":"2023-10-09T04:03:37.208063+0000","last_clean":"2023-10-09T04:03:37.208063+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:03:37.208063+0000","last_undegraded":"2023-10-09T04:03:37.208063+0000","last_fullsized":"2023-10-09T04:03:37.208063+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208745+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:03:36.208745+0000","last_peered":"2023-10-09T04:03:36.208745+0000","last_clean":"2023-10-09T04:03:36.208745+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:03:36.208745+0000","last_undegraded":"2023-10-09T04:03:36.208745+0000","last_fullsized":"2023-10-09T04:03:36.208745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221060+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:03:37.221060+0000","last_peered":"2023-10-09T04:03:37.221060+0000","last_clean":"2023-10-09T04:03:37.221060+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:03:37.221060+0000","last_undegraded":"2023-10-09T04:03:37.221060+0000","last_fullsized":"2023-10-09T04:03:37.221060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211820+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:03:36.211820+0000","last_peered":"2023-10-09T04:03:36.211820+0000","last_clean":"2023-10-09T04:03:36.211820+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:03:36.211820+0000","last_undegraded":"2023-10-09T04:03:36.211820+0000","last_fullsized":"2023-10-09T04:03:36.211820+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217253+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:03:37.217253+0000","last_peered":"2023-10-09T04:03:37.217253+0000","last_clean":"2023-10-09T04:03:37.217253+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:03:37.217253+0000","last_undegraded":"2023-10-09T04:03:37.217253+0000","last_fullsized":"2023-10-09T04:03:37.217253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207368+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:03:37.207368+0000","last_peered":"2023-10-09T04:03:37.207368+0000","last_clean":"2023-10-09T04:03:37.207368+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:03:37.207368+0000","last_undegraded":"2023-10-09T04:03:37.207368+0000","last_fullsized":"2023-10-09T04:03:37.207368+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211705+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:03:36.211705+0000","last_peered":"2023-10-09T04:03:36.211705+0000","last_clean":"2023-10-09T04:03:36.211705+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:03:36.211705+0000","last_undegraded":"2023-10-09T04:03:36.211705+0000","last_fullsized":"2023-10-09T04:03:36.211705+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214698+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:03:36.214698+0000","last_peered":"2023-10-09T04:03:36.214698+0000","last_clean":"2023-10-09T04:03:36.214698+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:03:36.214698+0000","last_undegraded":"2023-10-09T04:03:36.214698+0000","last_fullsized":"2023-10-09T04:03:36.214698+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208137+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:03:37.208137+0000","last_peered":"2023-10-09T04:03:37.208137+0000","last_clean":"2023-10-09T04:03:37.208137+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:03:37.208137+0000","last_undegraded":"2023-10-09T04:03:37.208137+0000","last_fullsized":"2023-10-09T04:03:37.208137+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220143+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:03:37.220143+0000","last_peered":"2023-10-09T04:03:37.220143+0000","last_clean":"2023-10-09T04:03:37.220143+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:03:37.220143+0000","last_undegraded":"2023-10-09T04:03:37.220143+0000","last_fullsized":"2023-10-09T04:03:37.220143+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212510+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:03:36.212510+0000","last_peered":"2023-10-09T04:03:36.212510+0000","last_clean":"2023-10-09T04:03:36.212510+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:03:36.212510+0000","last_undegraded":"2023-10-09T04:03:36.212510+0000","last_fullsized":"2023-10-09T04:03:36.212510+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.209067+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:03:36.209067+0000","last_peered":"2023-10-09T04:03:36.209067+0000","last_clean":"2023-10-09T04:03:36.209067+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:03:36.209067+0000","last_undegraded":"2023-10-09T04:03:36.209067+0000","last_fullsized":"2023-10-09T04:03:36.209067+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221180+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:03:37.221180+0000","last_peered":"2023-10-09T04:03:37.221180+0000","last_clean":"2023-10-09T04:03:37.221180+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:03:37.221180+0000","last_undegraded":"2023-10-09T04:03:37.221180+0000","last_fullsized":"2023-10-09T04:03:37.221180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":440,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212413+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:03:36.212413+0000","last_peered":"2023-10-09T04:03:36.212413+0000","last_clean":"2023-10-09T04:03:36.212413+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:03:36.212413+0000","last_undegraded":"2023-10-09T04:03:36.212413+0000","last_fullsized":"2023-10-09T04:03:36.212413+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211739+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:03:36.211739+0000","last_peered":"2023-10-09T04:03:36.211739+0000","last_clean":"2023-10-09T04:03:36.211739+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:03:36.211739+0000","last_undegraded":"2023-10-09T04:03:36.211739+0000","last_fullsized":"2023-10-09T04:03:36.211739+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.218572+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:03:36.218572+0000","last_peered":"2023-10-09T04:03:36.218572+0000","last_clean":"2023-10-09T04:03:36.218572+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:03:36.218572+0000","last_undegraded":"2023-10-09T04:03:36.218572+0000","last_fullsized":"2023-10-09T04:03:36.218572+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213218+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:03:36.213218+0000","last_peered":"2023-10-09T04:03:36.213218+0000","last_clean":"2023-10-09T04:03:36.213218+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:03:36.213218+0000","last_undegraded":"2023-10-09T04:03:36.213218+0000","last_fullsized":"2023-10-09T04:03:36.213218+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215356+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:03:37.215356+0000","last_peered":"2023-10-09T04:03:37.215356+0000","last_clean":"2023-10-09T04:03:37.215356+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:03:37.215356+0000","last_undegraded":"2023-10-09T04:03:37.215356+0000","last_fullsized":"2023-10-09T04:03:37.215356+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.218452+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:03:37.218452+0000","last_peered":"2023-10-09T04:03:37.218452+0000","last_clean":"2023-10-09T04:03:37.218452+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:03:37.218452+0000","last_undegraded":"2023-10-09T04:03:37.218452+0000","last_fullsized":"2023-10-09T04:03:37.218452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207441+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:03:36.207441+0000","last_peered":"2023-10-09T04:03:36.207441+0000","last_clean":"2023-10-09T04:03:36.207441+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:03:36.207441+0000","last_undegraded":"2023-10-09T04:03:36.207441+0000","last_fullsized":"2023-10-09T04:03:36.207441+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214621+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:03:36.214621+0000","last_peered":"2023-10-09T04:03:36.214621+0000","last_clean":"2023-10-09T04:03:36.214621+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:03:36.214621+0000","last_undegraded":"2023-10-09T04:03:36.214621+0000","last_fullsized":"2023-10-09T04:03:36.214621+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211952+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:03:36.211952+0000","last_peered":"2023-10-09T04:03:36.211952+0000","last_clean":"2023-10-09T04:03:36.211952+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:03:36.211952+0000","last_undegraded":"2023-10-09T04:03:36.211952+0000","last_fullsized":"2023-10-09T04:03:36.211952+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220381+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:03:37.220381+0000","last_peered":"2023-10-09T04:03:37.220381+0000","last_clean":"2023-10-09T04:03:37.220381+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220381+0000","last_undegraded":"2023-10-09T04:03:37.220381+0000","last_fullsized":"2023-10-09T04:03:37.220381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":71,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211615+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:03:36.211615+0000","last_peered":"2023-10-09T04:03:36.211615+0000","last_clean":"2023-10-09T04:03:36.211615+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:03:36.211615+0000","last_undegraded":"2023-10-09T04:03:36.211615+0000","last_fullsized":"2023-10-09T04:03:36.211615+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.219140+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:03:37.219140+0000","last_peered":"2023-10-09T04:03:37.219140+0000","last_clean":"2023-10-09T04:03:37.219140+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:03:37.219140+0000","last_undegraded":"2023-10-09T04:03:37.219140+0000","last_fullsized":"2023-10-09T04:03:37.219140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221254+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:03:37.221254+0000","last_peered":"2023-10-09T04:03:37.221254+0000","last_clean":"2023-10-09T04:03:37.221254+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:03:37.221254+0000","last_undegraded":"2023-10-09T04:03:37.221254+0000","last_fullsized":"2023-10-09T04:03:37.221254+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206997+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:03:36.206997+0000","last_peered":"2023-10-09T04:03:36.206997+0000","last_clean":"2023-10-09T04:03:36.206997+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:03:36.206997+0000","last_undegraded":"2023-10-09T04:03:36.206997+0000","last_fullsized":"2023-10-09T04:03:36.206997+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206551+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:03:36.206551+0000","last_peered":"2023-10-09T04:03:36.206551+0000","last_clean":"2023-10-09T04:03:36.206551+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:03:36.206551+0000","last_undegraded":"2023-10-09T04:03:36.206551+0000","last_fullsized":"2023-10-09T04:03:36.206551+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208191+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:03:36.208191+0000","last_peered":"2023-10-09T04:03:36.208191+0000","last_clean":"2023-10-09T04:03:36.208191+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:03:36.208191+0000","last_undegraded":"2023-10-09T04:03:36.208191+0000","last_fullsized":"2023-10-09T04:03:36.208191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206838+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:03:36.206838+0000","last_peered":"2023-10-09T04:03:36.206838+0000","last_clean":"2023-10-09T04:03:36.206838+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:03:36.206838+0000","last_undegraded":"2023-10-09T04:03:36.206838+0000","last_fullsized":"2023-10-09T04:03:36.206838+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225364+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:03:37.225364+0000","last_peered":"2023-10-09T04:03:37.225364+0000","last_clean":"2023-10-09T04:03:37.225364+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:03:37.225364+0000","last_undegraded":"2023-10-09T04:03:37.225364+0000","last_fullsized":"2023-10-09T04:03:37.225364+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220457+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:03:37.220457+0000","last_peered":"2023-10-09T04:03:37.220457+0000","last_clean":"2023-10-09T04:03:37.220457+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:03:37.220457+0000","last_undegraded":"2023-10-09T04:03:37.220457+0000","last_fullsized":"2023-10-09T04:03:37.220457+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207539+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:03:37.207539+0000","last_peered":"2023-10-09T04:03:37.207539+0000","last_clean":"2023-10-09T04:03:37.207539+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:03:37.207539+0000","last_undegraded":"2023-10-09T04:03:37.207539+0000","last_fullsized":"2023-10-09T04:03:37.207539+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221861+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:03:37.221861+0000","last_peered":"2023-10-09T04:03:37.221861+0000","last_clean":"2023-10-09T04:03:37.221861+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:03:37.221861+0000","last_undegraded":"2023-10-09T04:03:37.221861+0000","last_fullsized":"2023-10-09T04:03:37.221861+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206686+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:03:36.206686+0000","last_peered":"2023-10-09T04:03:36.206686+0000","last_clean":"2023-10-09T04:03:36.206686+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:03:36.206686+0000","last_undegraded":"2023-10-09T04:03:36.206686+0000","last_fullsized":"2023-10-09T04:03:36.206686+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214808+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:03:36.214808+0000","last_peered":"2023-10-09T04:03:36.214808+0000","last_clean":"2023-10-09T04:03:36.214808+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:03:36.214808+0000","last_undegraded":"2023-10-09T04:03:36.214808+0000","last_fullsized":"2023-10-09T04:03:36.214808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225348+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:03:37.225348+0000","last_peered":"2023-10-09T04:03:37.225348+0000","last_clean":"2023-10-09T04:03:37.225348+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:03:37.225348+0000","last_undegraded":"2023-10-09T04:03:37.225348+0000","last_fullsized":"2023-10-09T04:03:37.225348+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211069+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:03:36.211069+0000","last_peered":"2023-10-09T04:03:36.211069+0000","last_clean":"2023-10-09T04:03:36.211069+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:03:36.211069+0000","last_undegraded":"2023-10-09T04:03:36.211069+0000","last_fullsized":"2023-10-09T04:03:36.211069+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211887+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:03:36.211887+0000","last_peered":"2023-10-09T04:03:36.211887+0000","last_clean":"2023-10-09T04:03:36.211887+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:03:36.211887+0000","last_undegraded":"2023-10-09T04:03:36.211887+0000","last_fullsized":"2023-10-09T04:03:36.211887+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213608+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:03:36.213608+0000","last_peered":"2023-10-09T04:03:36.213608+0000","last_clean":"2023-10-09T04:03:36.213608+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:03:36.213608+0000","last_undegraded":"2023-10-09T04:03:36.213608+0000","last_fullsized":"2023-10-09T04:03:36.213608+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217150+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:03:37.217150+0000","last_peered":"2023-10-09T04:03:37.217150+0000","last_clean":"2023-10-09T04:03:37.217150+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:03:37.217150+0000","last_undegraded":"2023-10-09T04:03:37.217150+0000","last_fullsized":"2023-10-09T04:03:37.217150+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206089+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:03:36.206089+0000","last_peered":"2023-10-09T04:03:36.206089+0000","last_clean":"2023-10-09T04:03:36.206089+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:03:36.206089+0000","last_undegraded":"2023-10-09T04:03:36.206089+0000","last_fullsized":"2023-10-09T04:03:36.206089+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223674+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:03:37.223674+0000","last_peered":"2023-10-09T04:03:37.223674+0000","last_clean":"2023-10-09T04:03:37.223674+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:03:37.223674+0000","last_undegraded":"2023-10-09T04:03:37.223674+0000","last_fullsized":"2023-10-09T04:03:37.223674+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.210575+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:03:36.210575+0000","last_peered":"2023-10-09T04:03:36.210575+0000","last_clean":"2023-10-09T04:03:36.210575+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:03:36.210575+0000","last_undegraded":"2023-10-09T04:03:36.210575+0000","last_fullsized":"2023-10-09T04:03:36.210575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220488+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:03:37.220488+0000","last_peered":"2023-10-09T04:03:37.220488+0000","last_clean":"2023-10-09T04:03:37.220488+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:03:37.220488+0000","last_undegraded":"2023-10-09T04:03:37.220488+0000","last_fullsized":"2023-10-09T04:03:37.220488+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220100+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:03:37.220100+0000","last_peered":"2023-10-09T04:03:37.220100+0000","last_clean":"2023-10-09T04:03:37.220100+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:03:37.220100+0000","last_undegraded":"2023-10-09T04:03:37.220100+0000","last_fullsized":"2023-10-09T04:03:37.220100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.222047+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:03:37.222047+0000","last_peered":"2023-10-09T04:03:37.222047+0000","last_clean":"2023-10-09T04:03:37.222047+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:03:37.222047+0000","last_undegraded":"2023-10-09T04:03:37.222047+0000","last_fullsized":"2023-10-09T04:03:37.222047+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211450+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:03:36.211450+0000","last_peered":"2023-10-09T04:03:36.211450+0000","last_clean":"2023-10-09T04:03:36.211450+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:03:36.211450+0000","last_undegraded":"2023-10-09T04:03:36.211450+0000","last_fullsized":"2023-10-09T04:03:36.211450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207346+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:03:36.207346+0000","last_peered":"2023-10-09T04:03:36.207346+0000","last_clean":"2023-10-09T04:03:36.207346+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:03:36.207346+0000","last_undegraded":"2023-10-09T04:03:36.207346+0000","last_fullsized":"2023-10-09T04:03:36.207346+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208541+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:03:36.208541+0000","last_peered":"2023-10-09T04:03:36.208541+0000","last_clean":"2023-10-09T04:03:36.208541+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:03:36.208541+0000","last_undegraded":"2023-10-09T04:03:36.208541+0000","last_fullsized":"2023-10-09T04:03:36.208541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211626+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:03:36.211626+0000","last_peered":"2023-10-09T04:03:36.211626+0000","last_clean":"2023-10-09T04:03:36.211626+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:03:36.211626+0000","last_undegraded":"2023-10-09T04:03:36.211626+0000","last_fullsized":"2023-10-09T04:03:36.211626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220454+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:03:37.220454+0000","last_peered":"2023-10-09T04:03:37.220454+0000","last_clean":"2023-10-09T04:03:37.220454+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:03:37.220454+0000","last_undegraded":"2023-10-09T04:03:37.220454+0000","last_fullsized":"2023-10-09T04:03:37.220454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221468+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:03:37.221468+0000","last_peered":"2023-10-09T04:03:37.221468+0000","last_clean":"2023-10-09T04:03:37.221468+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:03:37.221468+0000","last_undegraded":"2023-10-09T04:03:37.221468+0000","last_fullsized":"2023-10-09T04:03:37.221468+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212749+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:03:36.212749+0000","last_peered":"2023-10-09T04:03:36.212749+0000","last_clean":"2023-10-09T04:03:36.212749+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:03:36.212749+0000","last_undegraded":"2023-10-09T04:03:36.212749+0000","last_fullsized":"2023-10-09T04:03:36.212749+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221476+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:03:37.221476+0000","last_peered":"2023-10-09T04:03:37.221476+0000","last_clean":"2023-10-09T04:03:37.221476+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:03:37.221476+0000","last_undegraded":"2023-10-09T04:03:37.221476+0000","last_fullsized":"2023-10-09T04:03:37.221476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.219309+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:03:37.219309+0000","last_peered":"2023-10-09T04:03:37.219309+0000","last_clean":"2023-10-09T04:03:37.219309+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:03:37.219309+0000","last_undegraded":"2023-10-09T04:03:37.219309+0000","last_fullsized":"2023-10-09T04:03:37.219309+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208807+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:03:36.208807+0000","last_peered":"2023-10-09T04:03:36.208807+0000","last_clean":"2023-10-09T04:03:36.208807+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:03:36.208807+0000","last_undegraded":"2023-10-09T04:03:36.208807+0000","last_fullsized":"2023-10-09T04:03:36.208807+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.216503+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:03:37.216503+0000","last_peered":"2023-10-09T04:03:37.216503+0000","last_clean":"2023-10-09T04:03:37.216503+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:03:37.216503+0000","last_undegraded":"2023-10-09T04:03:37.216503+0000","last_fullsized":"2023-10-09T04:03:37.216503+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223692+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:03:37.223692+0000","last_peered":"2023-10-09T04:03:37.223692+0000","last_clean":"2023-10-09T04:03:37.223692+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:03:37.223692+0000","last_undegraded":"2023-10-09T04:03:37.223692+0000","last_fullsized":"2023-10-09T04:03:37.223692+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220333+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:03:37.220333+0000","last_peered":"2023-10-09T04:03:37.220333+0000","last_clean":"2023-10-09T04:03:37.220333+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220333+0000","last_undegraded":"2023-10-09T04:03:37.220333+0000","last_fullsized":"2023-10-09T04:03:37.220333+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220097+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:03:37.220097+0000","last_peered":"2023-10-09T04:03:37.220097+0000","last_clean":"2023-10-09T04:03:37.220097+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:03:37.220097+0000","last_undegraded":"2023-10-09T04:03:37.220097+0000","last_fullsized":"2023-10-09T04:03:37.220097+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":77,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208233+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:03:36.208233+0000","last_peered":"2023-10-09T04:03:36.208233+0000","last_clean":"2023-10-09T04:03:36.208233+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:03:36.208233+0000","last_undegraded":"2023-10-09T04:03:36.208233+0000","last_fullsized":"2023-10-09T04:03:36.208233+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201424,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1136082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55100000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.363,"5min":3.363,"15min":3.363},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.58999999999999997,"15min":0.57099999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33000000000000002},"max":{"1min":2.105,"5min":2.105,"15min":3.7709999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.44800000000000001,"5min":0.39400000000000002,"15min":0.33500000000000002},"max":{"1min":0.78500000000000003,"5min":0.82799999999999996,"15min":3.6190000000000002},"last":0.61899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.5,"15min":0.51500000000000001},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.28599999999999998},"max":{"1min":0.629,"5min":1.0169999999999999,"15min":6.3129999999999997},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53200000000000003,"15min":0.52100000000000002},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.27500000000000002},"max":{"1min":0.71699999999999997,"5min":0.81100000000000005,"15min":3.8769999999999998},"last":0.69199999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":0.67100000000000004,"5min":1.4330000000000001,"15min":1.464},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.44400000000000001,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.95599999999999996,"15min":3.1840000000000002},"last":0.73499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.53600000000000003},"min":{"1min":0.40300000000000002,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.92500000000000004,"15min":1.6890000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.46100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.70099999999999996,"5min":0.82199999999999995,"15min":1.9350000000000001},"last":0.76100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463061,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1135613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.67400000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53700000000000003,"15min":0.53600000000000003},"min":{"1min":0.371,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.76300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.73799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.77600000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56799999999999995,"15min":0.54800000000000004},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":0.79700000000000004,"5min":0.81299999999999994,"15min":1.1120000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.34300000000000003},"max":{"1min":0.69399999999999995,"5min":0.77200000000000002,"15min":1.3839999999999999},"last":0.80600000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46899999999999997,"15min":0.45000000000000001},"min":{"1min":0.33800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70099999999999996,"5min":0.74099999999999999,"15min":0.76800000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.505,"15min":0.48599999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":0.73199999999999998,"5min":0.84099999999999997,"15min":1.0569999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.52000000000000002,"15min":0.499},"min":{"1min":0.36799999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.71899999999999997,"5min":0.84699999999999998,"15min":0.96499999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58799999999999997,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":1.034},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 04:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.42499999999999999,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.93999999999999995},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.63,"15min":0.59899999999999998},"min":{"1min":0.46200000000000002,"5min":0.42099999999999999,"15min":0.34100000000000003},"max":{"1min":0.78900000000000003,"5min":0.86299999999999999,"15min":1.077},"last":0.71499999999999997}]}]},{"osd":1,"up_from":89,"seq":382252089369,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28484,"kb_used_data":1020,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716668,"statfs":{"total":95995035648,"available":95965868032,"internally_reserved":0,"allocated":1044480,"data_stored":549351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.251},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.28100000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.552,"5min":2.552,"15min":2.552},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.6379999999999999,"5min":2.6379999999999999,"15min":2.6379999999999999},"last":0.5}]},{"osd":5,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.504,"5min":2.504,"15min":2.504},"last":0.441},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":2.6859999999999999,"5min":2.6859999999999999,"15min":2.6859999999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.5339999999999998,"5min":2.5339999999999998,"15min":2.5339999999999998},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.601,"5min":2.601,"15min":2.601},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.6659999999999999,"5min":2.6659999999999999,"15min":2.6659999999999999},"last":0.58399999999999996}]}]},{"osd":0,"up_from":84,"seq":360777252895,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29136,"kb_used_data":1608,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716016,"statfs":{"total":95995035648,"available":95965200384,"internally_reserved":0,"allocated":1646592,"data_stored":1138279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.441}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.48299999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61399999999999999,"15min":0.60999999999999999},"min":{"1min":0.52200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.79500000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.628,"15min":0.625},"min":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.443,"5min":0.44,"15min":0.44},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.74399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.46600000000000003,"5min":0.435,"15min":0.435},"max":{"1min":0.92900000000000005,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.64900000000000002,"15min":0.64700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.84999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.65500000000000003,"15min":0.65300000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.67200000000000004,"15min":0.66900000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.80600000000000005}]}]},{"osd":2,"up_from":94,"seq":403726925843,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28604,"kb_used_data":1012,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716548,"statfs":{"total":95995035648,"available":95965745152,"internally_reserved":0,"allocated":1036288,"data_stored":547077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.23999999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.311}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.52500000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.627},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.60899999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762317,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28464,"kb_used_data":1000,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716688,"statfs":{"total":95995035648,"available":95965888512,"internally_reserved":0,"allocated":1024000,"data_stored":545297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.371}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56299999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]}]},{"osd":4,"up_from":31,"seq":133143986333,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302380,"kb_used_data":996,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442772,"statfs":{"total":95995035648,"available":95685398528,"internally_reserved":0,"allocated":1019904,"data_stored":545379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.434},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.68600000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.496}]},{"osd":1,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56999999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:02:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58799999999999997,"15min":0.55900000000000005},"min":{"1min":0.46300000000000002,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":0.74299999999999999,"5min":1.117,"15min":1.117},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.61499999999999999,"15min":0.57899999999999996},"min":{"1min":0.46800000000000003,"5min":0.378,"15min":0.33100000000000002},"max":{"1min":0.76300000000000001,"5min":1.046,"15min":1.581},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48199999999999998,"15min":0.44900000000000001},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.625,"5min":0.753,"15min":1.038},"last":0.628},{"interface":"front","average":{"1min":0.504,"5min":0.53000000000000003,"15min":0.48099999999999998},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.24399999999999999},"max":{"1min":0.67100000000000004,"5min":0.77200000000000002,"15min":0.85499999999999998},"last":0.755}]},{"osd":6,"last update":"Mon Oct 9 04:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56399999999999995,"15min":0.52200000000000002},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.27900000000000003},"max":{"1min":0.75900000000000001,"5min":0.84299999999999997,"15min":0.92700000000000005},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57399999999999995,"15min":0.52500000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32800000000000001},"max":{"1min":0.70899999999999996,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57899999999999996,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.33900000000000002,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60899999999999999,"15min":0.59999999999999998},"min":{"1min":0.438,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.875,"15min":1.079},"last":0.78400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724697,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306460,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438692,"statfs":{"total":95995035648,"available":95681220608,"internally_reserved":0,"allocated":1003520,"data_stored":541065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.57799999999999996,"15min":0.59099999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.74099999999999999,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70299999999999996,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.435},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.42799999999999999,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":0.79300000000000004,"5min":0.83299999999999996,"15min":2.1850000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.622,"15min":0.60199999999999998},"min":{"1min":0.47099999999999997,"5min":0.42699999999999999,"15min":0.32400000000000001},"max":{"1min":0.76100000000000001,"5min":0.85999999999999999,"15min":2.2189999999999999},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.34499999999999997,"5min":0.32000000000000001,"15min":0.253},"max":{"1min":0.74199999999999999,"5min":0.81200000000000006,"15min":0.85399999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":0.80200000000000005,"5min":0.82599999999999996,"15min":0.86299999999999999},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51800000000000002,"15min":0.51300000000000001},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.27400000000000002},"max":{"1min":0.72399999999999998,"5min":0.81200000000000006,"15min":0.91000000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.55800000000000005,"15min":0.54200000000000004},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.312},"max":{"1min":0.80500000000000005,"5min":0.86099999999999999,"15min":0.89800000000000002},"last":0.67800000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59199999999999997,"15min":0.58299999999999996},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":0.81899999999999995,"5min":0.84099999999999997,"15min":0.94799999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.628,"15min":0.61399999999999999},"min":{"1min":0.47099999999999997,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.96399999999999997},"last":0.71399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:04:32.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:04:32.487 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:32 smithi163 ceph-mon[221067]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:32.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:32 smithi163 ceph-mon[213841]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:32 smithi181 ceph-mon[196979]: pgmap v224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:33.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:33 smithi163 ceph-mon[221067]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:33.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:33 smithi163 ceph-mon[213841]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:33 smithi181 ceph-mon[196979]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:34.108 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:04:34.109 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:04:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:34 smithi163 ceph-mon[213841]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:34 smithi163 ceph-mon[221067]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:34 smithi181 ceph-mon[196979]: pgmap v225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:34.829 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":225,"stamp":"2023-10-09T04:04:33.266157+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1333544,"kb_used_data":9768,"kb_used_omap":126,"kb_used_meta":1323585,"kb_avail":748627672,"statfs":{"total":767960285184,"available":766594736128,"internally_reserved":0,"allocated":10002432,"data_stored":6138143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1355351675},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002982"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206546+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:03:36.206546+0000","last_peered":"2023-10-09T04:03:36.206546+0000","last_clean":"2023-10-09T04:03:36.206546+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:03:36.206546+0000","last_undegraded":"2023-10-09T04:03:36.206546+0000","last_fullsized":"2023-10-09T04:03:36.206546+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207470+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:03:37.207470+0000","last_peered":"2023-10-09T04:03:37.207470+0000","last_clean":"2023-10-09T04:03:37.207470+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:03:37.207470+0000","last_undegraded":"2023-10-09T04:03:37.207470+0000","last_fullsized":"2023-10-09T04:03:37.207470+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":82,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215833+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:03:37.215833+0000","last_peered":"2023-10-09T04:03:37.215833+0000","last_clean":"2023-10-09T04:03:37.215833+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:03:37.215833+0000","last_undegraded":"2023-10-09T04:03:37.215833+0000","last_fullsized":"2023-10-09T04:03:37.215833+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221959+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:03:37.221959+0000","last_peered":"2023-10-09T04:03:37.221959+0000","last_clean":"2023-10-09T04:03:37.221959+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:03:37.221959+0000","last_undegraded":"2023-10-09T04:03:37.221959+0000","last_fullsized":"2023-10-09T04:03:37.221959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.224382+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:03:37.224382+0000","last_peered":"2023-10-09T04:03:37.224382+0000","last_clean":"2023-10-09T04:03:37.224382+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:03:37.224382+0000","last_undegraded":"2023-10-09T04:03:37.224382+0000","last_fullsized":"2023-10-09T04:03:37.224382+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":44,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213576+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:03:36.213576+0000","last_peered":"2023-10-09T04:03:36.213576+0000","last_clean":"2023-10-09T04:03:36.213576+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:03:36.213576+0000","last_undegraded":"2023-10-09T04:03:36.213576+0000","last_fullsized":"2023-10-09T04:03:36.213576+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225542+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:03:37.225542+0000","last_peered":"2023-10-09T04:03:37.225542+0000","last_clean":"2023-10-09T04:03:37.225542+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:03:37.225542+0000","last_undegraded":"2023-10-09T04:03:37.225542+0000","last_fullsized":"2023-10-09T04:03:37.225542+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212908+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:03:36.212908+0000","last_peered":"2023-10-09T04:03:36.212908+0000","last_clean":"2023-10-09T04:03:36.212908+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:03:36.212908+0000","last_undegraded":"2023-10-09T04:03:36.212908+0000","last_fullsized":"2023-10-09T04:03:36.212908+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215366+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:03:37.215366+0000","last_peered":"2023-10-09T04:03:37.215366+0000","last_clean":"2023-10-09T04:03:37.215366+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:03:37.215366+0000","last_undegraded":"2023-10-09T04:03:37.215366+0000","last_fullsized":"2023-10-09T04:03:37.215366+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":41,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207622+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:03:37.207622+0000","last_peered":"2023-10-09T04:03:37.207622+0000","last_clean":"2023-10-09T04:03:37.207622+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:03:37.207622+0000","last_undegraded":"2023-10-09T04:03:37.207622+0000","last_fullsized":"2023-10-09T04:03:37.207622+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206640+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:03:36.206640+0000","last_peered":"2023-10-09T04:03:36.206640+0000","last_clean":"2023-10-09T04:03:36.206640+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:03:36.206640+0000","last_undegraded":"2023-10-09T04:03:36.206640+0000","last_fullsized":"2023-10-09T04:03:36.206640+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208063+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:03:37.208063+0000","last_peered":"2023-10-09T04:03:37.208063+0000","last_clean":"2023-10-09T04:03:37.208063+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:03:37.208063+0000","last_undegraded":"2023-10-09T04:03:37.208063+0000","last_fullsized":"2023-10-09T04:03:37.208063+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208745+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:03:36.208745+0000","last_peered":"2023-10-09T04:03:36.208745+0000","last_clean":"2023-10-09T04:03:36.208745+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:03:36.208745+0000","last_undegraded":"2023-10-09T04:03:36.208745+0000","last_fullsized":"2023-10-09T04:03:36.208745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221060+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:03:37.221060+0000","last_peered":"2023-10-09T04:03:37.221060+0000","last_clean":"2023-10-09T04:03:37.221060+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:03:37.221060+0000","last_undegraded":"2023-10-09T04:03:37.221060+0000","last_fullsized":"2023-10-09T04:03:37.221060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211820+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:03:36.211820+0000","last_peered":"2023-10-09T04:03:36.211820+0000","last_clean":"2023-10-09T04:03:36.211820+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:03:36.211820+0000","last_undegraded":"2023-10-09T04:03:36.211820+0000","last_fullsized":"2023-10-09T04:03:36.211820+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217253+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:03:37.217253+0000","last_peered":"2023-10-09T04:03:37.217253+0000","last_clean":"2023-10-09T04:03:37.217253+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:03:37.217253+0000","last_undegraded":"2023-10-09T04:03:37.217253+0000","last_fullsized":"2023-10-09T04:03:37.217253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207368+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:03:37.207368+0000","last_peered":"2023-10-09T04:03:37.207368+0000","last_clean":"2023-10-09T04:03:37.207368+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:03:37.207368+0000","last_undegraded":"2023-10-09T04:03:37.207368+0000","last_fullsized":"2023-10-09T04:03:37.207368+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211705+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:03:36.211705+0000","last_peered":"2023-10-09T04:03:36.211705+0000","last_clean":"2023-10-09T04:03:36.211705+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:03:36.211705+0000","last_undegraded":"2023-10-09T04:03:36.211705+0000","last_fullsized":"2023-10-09T04:03:36.211705+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214698+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:03:36.214698+0000","last_peered":"2023-10-09T04:03:36.214698+0000","last_clean":"2023-10-09T04:03:36.214698+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:03:36.214698+0000","last_undegraded":"2023-10-09T04:03:36.214698+0000","last_fullsized":"2023-10-09T04:03:36.214698+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.208137+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:03:37.208137+0000","last_peered":"2023-10-09T04:03:37.208137+0000","last_clean":"2023-10-09T04:03:37.208137+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:03:37.208137+0000","last_undegraded":"2023-10-09T04:03:37.208137+0000","last_fullsized":"2023-10-09T04:03:37.208137+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220143+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:03:37.220143+0000","last_peered":"2023-10-09T04:03:37.220143+0000","last_clean":"2023-10-09T04:03:37.220143+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:03:37.220143+0000","last_undegraded":"2023-10-09T04:03:37.220143+0000","last_fullsized":"2023-10-09T04:03:37.220143+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212510+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:03:36.212510+0000","last_peered":"2023-10-09T04:03:36.212510+0000","last_clean":"2023-10-09T04:03:36.212510+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:03:36.212510+0000","last_undegraded":"2023-10-09T04:03:36.212510+0000","last_fullsized":"2023-10-09T04:03:36.212510+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.209067+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:03:36.209067+0000","last_peered":"2023-10-09T04:03:36.209067+0000","last_clean":"2023-10-09T04:03:36.209067+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:03:36.209067+0000","last_undegraded":"2023-10-09T04:03:36.209067+0000","last_fullsized":"2023-10-09T04:03:36.209067+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221180+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:03:37.221180+0000","last_peered":"2023-10-09T04:03:37.221180+0000","last_clean":"2023-10-09T04:03:37.221180+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:03:37.221180+0000","last_undegraded":"2023-10-09T04:03:37.221180+0000","last_fullsized":"2023-10-09T04:03:37.221180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":440,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212413+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:03:36.212413+0000","last_peered":"2023-10-09T04:03:36.212413+0000","last_clean":"2023-10-09T04:03:36.212413+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:03:36.212413+0000","last_undegraded":"2023-10-09T04:03:36.212413+0000","last_fullsized":"2023-10-09T04:03:36.212413+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211739+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:03:36.211739+0000","last_peered":"2023-10-09T04:03:36.211739+0000","last_clean":"2023-10-09T04:03:36.211739+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:03:36.211739+0000","last_undegraded":"2023-10-09T04:03:36.211739+0000","last_fullsized":"2023-10-09T04:03:36.211739+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":66,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.218572+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:03:36.218572+0000","last_peered":"2023-10-09T04:03:36.218572+0000","last_clean":"2023-10-09T04:03:36.218572+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:03:36.218572+0000","last_undegraded":"2023-10-09T04:03:36.218572+0000","last_fullsized":"2023-10-09T04:03:36.218572+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.210380+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:03:37.210380+0000","last_peered":"2023-10-09T04:03:37.210380+0000","last_clean":"2023-10-09T04:03:37.210380+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:03:37.210380+0000","last_undegraded":"2023-10-09T04:03:37.210380+0000","last_fullsized":"2023-10-09T04:03:37.210380+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":81,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.215356+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:03:37.215356+0000","last_peered":"2023-10-09T04:03:37.215356+0000","last_clean":"2023-10-09T04:03:37.215356+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:03:37.215356+0000","last_undegraded":"2023-10-09T04:03:37.215356+0000","last_fullsized":"2023-10-09T04:03:37.215356+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.218452+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:03:37.218452+0000","last_peered":"2023-10-09T04:03:37.218452+0000","last_clean":"2023-10-09T04:03:37.218452+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:03:37.218452+0000","last_undegraded":"2023-10-09T04:03:37.218452+0000","last_fullsized":"2023-10-09T04:03:37.218452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207441+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:03:36.207441+0000","last_peered":"2023-10-09T04:03:36.207441+0000","last_clean":"2023-10-09T04:03:36.207441+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:03:36.207441+0000","last_undegraded":"2023-10-09T04:03:36.207441+0000","last_fullsized":"2023-10-09T04:03:36.207441+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":63,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214621+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:03:36.214621+0000","last_peered":"2023-10-09T04:03:36.214621+0000","last_clean":"2023-10-09T04:03:36.214621+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:03:36.214621+0000","last_undegraded":"2023-10-09T04:03:36.214621+0000","last_fullsized":"2023-10-09T04:03:36.214621+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211952+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:03:36.211952+0000","last_peered":"2023-10-09T04:03:36.211952+0000","last_clean":"2023-10-09T04:03:36.211952+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:03:36.211952+0000","last_undegraded":"2023-10-09T04:03:36.211952+0000","last_fullsized":"2023-10-09T04:03:36.211952+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220381+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:03:37.220381+0000","last_peered":"2023-10-09T04:03:37.220381+0000","last_clean":"2023-10-09T04:03:37.220381+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220381+0000","last_undegraded":"2023-10-09T04:03:37.220381+0000","last_fullsized":"2023-10-09T04:03:37.220381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":71,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211615+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:03:36.211615+0000","last_peered":"2023-10-09T04:03:36.211615+0000","last_clean":"2023-10-09T04:03:36.211615+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:03:36.211615+0000","last_undegraded":"2023-10-09T04:03:36.211615+0000","last_fullsized":"2023-10-09T04:03:36.211615+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.219140+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:03:37.219140+0000","last_peered":"2023-10-09T04:03:37.219140+0000","last_clean":"2023-10-09T04:03:37.219140+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:03:37.219140+0000","last_undegraded":"2023-10-09T04:03:37.219140+0000","last_fullsized":"2023-10-09T04:03:37.219140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221254+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:03:37.221254+0000","last_peered":"2023-10-09T04:03:37.221254+0000","last_clean":"2023-10-09T04:03:37.221254+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:03:37.221254+0000","last_undegraded":"2023-10-09T04:03:37.221254+0000","last_fullsized":"2023-10-09T04:03:37.221254+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206997+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:03:36.206997+0000","last_peered":"2023-10-09T04:03:36.206997+0000","last_clean":"2023-10-09T04:03:36.206997+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:03:36.206997+0000","last_undegraded":"2023-10-09T04:03:36.206997+0000","last_fullsized":"2023-10-09T04:03:36.206997+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206551+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:03:36.206551+0000","last_peered":"2023-10-09T04:03:36.206551+0000","last_clean":"2023-10-09T04:03:36.206551+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:03:36.206551+0000","last_undegraded":"2023-10-09T04:03:36.206551+0000","last_fullsized":"2023-10-09T04:03:36.206551+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208191+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:03:36.208191+0000","last_peered":"2023-10-09T04:03:36.208191+0000","last_clean":"2023-10-09T04:03:36.208191+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:03:36.208191+0000","last_undegraded":"2023-10-09T04:03:36.208191+0000","last_fullsized":"2023-10-09T04:03:36.208191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206838+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:03:36.206838+0000","last_peered":"2023-10-09T04:03:36.206838+0000","last_clean":"2023-10-09T04:03:36.206838+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:03:36.206838+0000","last_undegraded":"2023-10-09T04:03:36.206838+0000","last_fullsized":"2023-10-09T04:03:36.206838+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225364+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:03:37.225364+0000","last_peered":"2023-10-09T04:03:37.225364+0000","last_clean":"2023-10-09T04:03:37.225364+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:03:37.225364+0000","last_undegraded":"2023-10-09T04:03:37.225364+0000","last_fullsized":"2023-10-09T04:03:37.225364+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220457+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:03:37.220457+0000","last_peered":"2023-10-09T04:03:37.220457+0000","last_clean":"2023-10-09T04:03:37.220457+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:03:37.220457+0000","last_undegraded":"2023-10-09T04:03:37.220457+0000","last_fullsized":"2023-10-09T04:03:37.220457+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.207539+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:03:37.207539+0000","last_peered":"2023-10-09T04:03:37.207539+0000","last_clean":"2023-10-09T04:03:37.207539+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:03:37.207539+0000","last_undegraded":"2023-10-09T04:03:37.207539+0000","last_fullsized":"2023-10-09T04:03:37.207539+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221861+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:03:37.221861+0000","last_peered":"2023-10-09T04:03:37.221861+0000","last_clean":"2023-10-09T04:03:37.221861+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:03:37.221861+0000","last_undegraded":"2023-10-09T04:03:37.221861+0000","last_fullsized":"2023-10-09T04:03:37.221861+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206686+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:03:36.206686+0000","last_peered":"2023-10-09T04:03:36.206686+0000","last_clean":"2023-10-09T04:03:36.206686+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:03:36.206686+0000","last_undegraded":"2023-10-09T04:03:36.206686+0000","last_fullsized":"2023-10-09T04:03:36.206686+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":43,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.214808+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:03:36.214808+0000","last_peered":"2023-10-09T04:03:36.214808+0000","last_clean":"2023-10-09T04:03:36.214808+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:03:36.214808+0000","last_undegraded":"2023-10-09T04:03:36.214808+0000","last_fullsized":"2023-10-09T04:03:36.214808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.225348+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:03:37.225348+0000","last_peered":"2023-10-09T04:03:37.225348+0000","last_clean":"2023-10-09T04:03:37.225348+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:03:37.225348+0000","last_undegraded":"2023-10-09T04:03:37.225348+0000","last_fullsized":"2023-10-09T04:03:37.225348+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211069+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:03:36.211069+0000","last_peered":"2023-10-09T04:03:36.211069+0000","last_clean":"2023-10-09T04:03:36.211069+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:03:36.211069+0000","last_undegraded":"2023-10-09T04:03:36.211069+0000","last_fullsized":"2023-10-09T04:03:36.211069+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211887+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:03:36.211887+0000","last_peered":"2023-10-09T04:03:36.211887+0000","last_clean":"2023-10-09T04:03:36.211887+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:03:36.211887+0000","last_undegraded":"2023-10-09T04:03:36.211887+0000","last_fullsized":"2023-10-09T04:03:36.211887+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":60,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.213608+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:03:36.213608+0000","last_peered":"2023-10-09T04:03:36.213608+0000","last_clean":"2023-10-09T04:03:36.213608+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:03:36.213608+0000","last_undegraded":"2023-10-09T04:03:36.213608+0000","last_fullsized":"2023-10-09T04:03:36.213608+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.217150+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:03:37.217150+0000","last_peered":"2023-10-09T04:03:37.217150+0000","last_clean":"2023-10-09T04:03:37.217150+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:03:37.217150+0000","last_undegraded":"2023-10-09T04:03:37.217150+0000","last_fullsized":"2023-10-09T04:03:37.217150+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":67,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.206089+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:03:36.206089+0000","last_peered":"2023-10-09T04:03:36.206089+0000","last_clean":"2023-10-09T04:03:36.206089+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:03:36.206089+0000","last_undegraded":"2023-10-09T04:03:36.206089+0000","last_fullsized":"2023-10-09T04:03:36.206089+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223674+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:03:37.223674+0000","last_peered":"2023-10-09T04:03:37.223674+0000","last_clean":"2023-10-09T04:03:37.223674+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:03:37.223674+0000","last_undegraded":"2023-10-09T04:03:37.223674+0000","last_fullsized":"2023-10-09T04:03:37.223674+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":42,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.210575+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:03:36.210575+0000","last_peered":"2023-10-09T04:03:36.210575+0000","last_clean":"2023-10-09T04:03:36.210575+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:03:36.210575+0000","last_undegraded":"2023-10-09T04:03:36.210575+0000","last_fullsized":"2023-10-09T04:03:36.210575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220488+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:03:37.220488+0000","last_peered":"2023-10-09T04:03:37.220488+0000","last_clean":"2023-10-09T04:03:37.220488+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:03:37.220488+0000","last_undegraded":"2023-10-09T04:03:37.220488+0000","last_fullsized":"2023-10-09T04:03:37.220488+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220100+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:03:37.220100+0000","last_peered":"2023-10-09T04:03:37.220100+0000","last_clean":"2023-10-09T04:03:37.220100+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:03:37.220100+0000","last_undegraded":"2023-10-09T04:03:37.220100+0000","last_fullsized":"2023-10-09T04:03:37.220100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.222047+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:03:37.222047+0000","last_peered":"2023-10-09T04:03:37.222047+0000","last_clean":"2023-10-09T04:03:37.222047+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:03:37.222047+0000","last_undegraded":"2023-10-09T04:03:37.222047+0000","last_fullsized":"2023-10-09T04:03:37.222047+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":61,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211450+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:03:36.211450+0000","last_peered":"2023-10-09T04:03:36.211450+0000","last_clean":"2023-10-09T04:03:36.211450+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:03:36.211450+0000","last_undegraded":"2023-10-09T04:03:36.211450+0000","last_fullsized":"2023-10-09T04:03:36.211450+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.207346+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:03:36.207346+0000","last_peered":"2023-10-09T04:03:36.207346+0000","last_clean":"2023-10-09T04:03:36.207346+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:03:36.207346+0000","last_undegraded":"2023-10-09T04:03:36.207346+0000","last_fullsized":"2023-10-09T04:03:36.207346+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":64,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208541+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:03:36.208541+0000","last_peered":"2023-10-09T04:03:36.208541+0000","last_clean":"2023-10-09T04:03:36.208541+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:03:36.208541+0000","last_undegraded":"2023-10-09T04:03:36.208541+0000","last_fullsized":"2023-10-09T04:03:36.208541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.211626+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:03:36.211626+0000","last_peered":"2023-10-09T04:03:36.211626+0000","last_clean":"2023-10-09T04:03:36.211626+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:03:36.211626+0000","last_undegraded":"2023-10-09T04:03:36.211626+0000","last_fullsized":"2023-10-09T04:03:36.211626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":62,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220454+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:03:37.220454+0000","last_peered":"2023-10-09T04:03:37.220454+0000","last_clean":"2023-10-09T04:03:37.220454+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:03:37.220454+0000","last_undegraded":"2023-10-09T04:03:37.220454+0000","last_fullsized":"2023-10-09T04:03:37.220454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221468+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:03:37.221468+0000","last_peered":"2023-10-09T04:03:37.221468+0000","last_clean":"2023-10-09T04:03:37.221468+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:03:37.221468+0000","last_undegraded":"2023-10-09T04:03:37.221468+0000","last_fullsized":"2023-10-09T04:03:37.221468+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.212749+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:03:36.212749+0000","last_peered":"2023-10-09T04:03:36.212749+0000","last_clean":"2023-10-09T04:03:36.212749+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:03:36.212749+0000","last_undegraded":"2023-10-09T04:03:36.212749+0000","last_fullsized":"2023-10-09T04:03:36.212749+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.221476+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:03:37.221476+0000","last_peered":"2023-10-09T04:03:37.221476+0000","last_clean":"2023-10-09T04:03:37.221476+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:03:37.221476+0000","last_undegraded":"2023-10-09T04:03:37.221476+0000","last_fullsized":"2023-10-09T04:03:37.221476+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.219309+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:03:37.219309+0000","last_peered":"2023-10-09T04:03:37.219309+0000","last_clean":"2023-10-09T04:03:37.219309+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:03:37.219309+0000","last_undegraded":"2023-10-09T04:03:37.219309+0000","last_fullsized":"2023-10-09T04:03:37.219309+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208807+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:03:36.208807+0000","last_peered":"2023-10-09T04:03:36.208807+0000","last_clean":"2023-10-09T04:03:36.208807+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:03:36.208807+0000","last_undegraded":"2023-10-09T04:03:36.208807+0000","last_fullsized":"2023-10-09T04:03:36.208807+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.216503+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:03:37.216503+0000","last_peered":"2023-10-09T04:03:37.216503+0000","last_clean":"2023-10-09T04:03:37.216503+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:03:37.216503+0000","last_undegraded":"2023-10-09T04:03:37.216503+0000","last_fullsized":"2023-10-09T04:03:37.216503+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.223692+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:03:37.223692+0000","last_peered":"2023-10-09T04:03:37.223692+0000","last_clean":"2023-10-09T04:03:37.223692+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:03:37.223692+0000","last_undegraded":"2023-10-09T04:03:37.223692+0000","last_fullsized":"2023-10-09T04:03:37.223692+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220333+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:03:37.220333+0000","last_peered":"2023-10-09T04:03:37.220333+0000","last_clean":"2023-10-09T04:03:37.220333+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:03:37.220333+0000","last_undegraded":"2023-10-09T04:03:37.220333+0000","last_fullsized":"2023-10-09T04:03:37.220333+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":100,"state":"active+clean","last_fresh":"2023-10-09T04:03:37.220097+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:03:37.220097+0000","last_peered":"2023-10-09T04:03:37.220097+0000","last_clean":"2023-10-09T04:03:37.220097+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:03:37.220097+0000","last_undegraded":"2023-10-09T04:03:37.220097+0000","last_fullsized":"2023-10-09T04:03:37.220097+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":77,"reported_epoch":99,"state":"active+clean","last_fresh":"2023-10-09T04:03:36.208233+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:03:36.208233+0000","last_peered":"2023-10-09T04:03:36.208233+0000","last_clean":"2023-10-09T04:03:36.208233+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:03:36.208233+0000","last_undegraded":"2023-10-09T04:03:36.208233+0000","last_fullsized":"2023-10-09T04:03:36.208233+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201424,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311160,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433992,"statfs":{"total":95995035648,"available":95676407808,"internally_reserved":0,"allocated":1622016,"data_stored":1136082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55100000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.4740000000000002,"5min":2.4740000000000002,"15min":2.4740000000000002},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.363,"5min":3.363,"15min":3.363},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.58999999999999997,"15min":0.57099999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33000000000000002},"max":{"1min":2.105,"5min":2.105,"15min":3.7709999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.44800000000000001,"5min":0.39400000000000002,"15min":0.33500000000000002},"max":{"1min":0.78500000000000003,"5min":0.82799999999999996,"15min":3.6190000000000002},"last":0.61899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.5,"15min":0.51500000000000001},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.28599999999999998},"max":{"1min":0.629,"5min":1.0169999999999999,"15min":6.3129999999999997},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53200000000000003,"15min":0.52100000000000002},"min":{"1min":0.371,"5min":0.29299999999999998,"15min":0.27500000000000002},"max":{"1min":0.71699999999999997,"5min":0.81100000000000005,"15min":3.8769999999999998},"last":0.69199999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":0.67100000000000004,"5min":1.4330000000000001,"15min":1.464},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.44400000000000001,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":0.71599999999999997,"5min":0.95599999999999996,"15min":3.1840000000000002},"last":0.73499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.53600000000000003},"min":{"1min":0.40300000000000002,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.92500000000000004,"15min":1.6890000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.46100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.70099999999999996,"5min":0.82199999999999995,"15min":1.9350000000000001},"last":0.76100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463061,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298856,"kb_used_data":1568,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446296,"statfs":{"total":95995035648,"available":95689007104,"internally_reserved":0,"allocated":1605632,"data_stored":1135613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.67400000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53700000000000003,"15min":0.53600000000000003},"min":{"1min":0.371,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.76300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.73799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.77600000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56799999999999995,"15min":0.54800000000000004},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":0.79700000000000004,"5min":0.81299999999999994,"15min":1.1120000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.34300000000000003},"max":{"1min":0.69399999999999995,"5min":0.77200000000000002,"15min":1.3839999999999999},"last":0.80600000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46899999999999997,"15min":0.45000000000000001},"min":{"1min":0.33800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70099999999999996,"5min":0.74099999999999999,"15min":0.76800000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.505,"15min":0.48599999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":0.73199999999999998,"5min":0.84099999999999997,"15min":1.0569999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.52000000000000002,"15min":0.499},"min":{"1min":0.36799999999999999,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.71899999999999997,"5min":0.84699999999999998,"15min":0.96499999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58799999999999997,"15min":0.55300000000000005},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":1.034},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 04:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58399999999999996,"15min":0.56699999999999995},"min":{"1min":0.42499999999999999,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.93999999999999995},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.63,"15min":0.59899999999999998},"min":{"1min":0.46200000000000002,"5min":0.42099999999999999,"15min":0.34100000000000003},"max":{"1min":0.78900000000000003,"5min":0.86299999999999999,"15min":1.077},"last":0.71499999999999997}]}]},{"osd":1,"up_from":89,"seq":382252089370,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28484,"kb_used_data":1020,"kb_used_omap":9,"kb_used_meta":27446,"kb_avail":93716668,"statfs":{"total":95995035648,"available":95965868032,"internally_reserved":0,"allocated":1044480,"data_stored":549351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":28105705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.222},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.318}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.245}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.552,"5min":2.552,"15min":2.552},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.6379999999999999,"5min":2.6379999999999999,"15min":2.6379999999999999},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.504,"5min":2.504,"15min":2.504},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":2.6859999999999999,"5min":2.6859999999999999,"15min":2.6859999999999999},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":2.5339999999999998,"5min":2.5339999999999998,"15min":2.5339999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.601,"5min":2.601,"15min":2.601},"last":0.52500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"last":0.627},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.6659999999999999,"5min":2.6659999999999999,"15min":2.6659999999999999},"last":0.505}]}]},{"osd":0,"up_from":84,"seq":360777252896,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29136,"kb_used_data":1608,"kb_used_omap":28,"kb_used_meta":27491,"kb_avail":93716016,"statfs":{"total":95995035648,"available":95965200384,"internally_reserved":0,"allocated":1646592,"data_stored":1138279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":28151632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.45100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61399999999999999,"15min":0.60999999999999999},"min":{"1min":0.52200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.69399999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.628,"15min":0.625},"min":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.443,"5min":0.44,"15min":0.44},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.70499999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.46600000000000003,"5min":0.435,"15min":0.435},"max":{"1min":0.92900000000000005,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.622},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.64900000000000002,"15min":0.64700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.71699999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.65500000000000003,"15min":0.65300000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.67200000000000004,"15min":0.66900000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.74399999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925843,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28604,"kb_used_data":1012,"kb_used_omap":81,"kb_used_meta":27502,"kb_avail":93716548,"statfs":{"total":95995035648,"available":95965745152,"internally_reserved":0,"allocated":1036288,"data_stored":547077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":28162266},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.23999999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.311}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.52500000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.63900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.627},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.60899999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762317,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28464,"kb_used_data":1000,"kb_used_omap":7,"kb_used_meta":27448,"kb_avail":93716688,"statfs":{"total":95995035648,"available":95965888512,"internally_reserved":0,"allocated":1024000,"data_stored":545297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":28107368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29399999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23999999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.371}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56299999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]}]},{"osd":4,"up_from":31,"seq":133143986333,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302380,"kb_used_data":996,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93442772,"statfs":{"total":95995035648,"available":95685398528,"internally_reserved":0,"allocated":1019904,"data_stored":545379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.434},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.68600000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.496}]},{"osd":1,"last update":"Mon Oct 9 04:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56999999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:02:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58799999999999997,"15min":0.55900000000000005},"min":{"1min":0.46300000000000002,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":0.74299999999999999,"5min":1.117,"15min":1.117},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.61499999999999999,"15min":0.57899999999999996},"min":{"1min":0.46800000000000003,"5min":0.378,"15min":0.33100000000000002},"max":{"1min":0.76300000000000001,"5min":1.046,"15min":1.581},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48199999999999998,"15min":0.44900000000000001},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.625,"5min":0.753,"15min":1.038},"last":0.628},{"interface":"front","average":{"1min":0.504,"5min":0.53000000000000003,"15min":0.48099999999999998},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.24399999999999999},"max":{"1min":0.67100000000000004,"5min":0.77200000000000002,"15min":0.85499999999999998},"last":0.755}]},{"osd":6,"last update":"Mon Oct 9 04:03:44 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56399999999999995,"15min":0.52200000000000002},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.27900000000000003},"max":{"1min":0.75900000000000001,"5min":0.84299999999999997,"15min":0.92700000000000005},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57399999999999995,"15min":0.52500000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32800000000000001},"max":{"1min":0.70899999999999996,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:04:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57899999999999996,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.33900000000000002,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60899999999999999,"15min":0.59999999999999998},"min":{"1min":0.438,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":0.78900000000000003,"5min":0.875,"15min":1.079},"last":0.78400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724697,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306460,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438692,"statfs":{"total":95995035648,"available":95681220608,"internally_reserved":0,"allocated":1003520,"data_stored":541065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.57799999999999996,"15min":0.59099999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.74099999999999999,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70299999999999996,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58099999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.435},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.42799999999999999,"5min":0.33300000000000002,"15min":0.32200000000000001},"max":{"1min":0.79300000000000004,"5min":0.83299999999999996,"15min":2.1850000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.622,"15min":0.60199999999999998},"min":{"1min":0.47099999999999997,"5min":0.42699999999999999,"15min":0.32400000000000001},"max":{"1min":0.76100000000000001,"5min":0.85999999999999999,"15min":2.2189999999999999},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.34499999999999997,"5min":0.32000000000000001,"15min":0.253},"max":{"1min":0.74199999999999999,"5min":0.81200000000000006,"15min":0.85399999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":0.80200000000000005,"5min":0.82599999999999996,"15min":0.86299999999999999},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51800000000000002,"15min":0.51300000000000001},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.27400000000000002},"max":{"1min":0.72399999999999998,"5min":0.81200000000000006,"15min":0.91000000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.55800000000000005,"15min":0.54200000000000004},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.312},"max":{"1min":0.80500000000000005,"5min":0.86099999999999999,"15min":0.89800000000000002},"last":0.67800000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59199999999999997,"15min":0.58299999999999996},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":0.81899999999999995,"5min":0.84099999999999997,"15min":0.94799999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.628,"15min":0.61399999999999999},"min":{"1min":0.47099999999999997,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.96399999999999997},"last":0.71399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:04:34.834 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:04:34.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T04:04:35.182 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:35.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:35 smithi163 ceph-mon[221067]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:35.730 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:35 smithi163 ceph-mon[213841]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:35.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:35 smithi181 ceph-mon[196979]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:04:36.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:36 smithi163 ceph-mon[221067]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:36.397 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:36 smithi163 ceph-mon[213841]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:36.763 INFO:teuthology.orchestra.run.smithi163.stdout:epoch 100 2023-10-09T04:04:36.763 INFO:teuthology.orchestra.run.smithi163.stdout:fsid 8628d772-6656-11ee-8db5-212e2dc638e7 2023-10-09T04:04:36.763 INFO:teuthology.orchestra.run.smithi163.stdout:created 2023-10-09T03:48:06.379138+0000 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:modified 2023-10-09T04:03:37.200609+0000 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:crush_version 22 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:full_ratio 0.95 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:backfillfull_ratio 0.9 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:nearfull_ratio 0.85 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:require_min_compat_client luminous 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:min_compat_client luminous 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:require_osd_release quincy 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:stretch_mode_enabled false 2023-10-09T04:04:36.764 INFO:teuthology.orchestra.run.smithi163.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 64 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 69 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:max_osd 8 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0 up in weight 1 up_from 84 up_thru 99 down_at 81 last_clean_interval [8,80) [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252] [v2:172.21.15.163:6804/658095252,v1:172.21.15.163:6805/658095252] exists,up 9f39300f-5c82-4020-b75d-28b9705365f3 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 up in weight 1 up_from 89 up_thru 99 down_at 86 last_clean_interval [13,85) [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215] [v2:172.21.15.163:6812/3761898215,v1:172.21.15.163:6813/3761898215] exists,up 07dc6811-c7f1-40ce-83b3-46a33ff57f0a 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2 up in weight 1 up_from 94 up_thru 99 down_at 91 last_clean_interval [18,90) [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669] [v2:172.21.15.163:6820/96613669,v1:172.21.15.163:6821/96613669] exists,up d5a25867-ce16-4bfd-8356-c346de1fcbf6 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 up in weight 1 up_from 99 up_thru 99 down_at 96 last_clean_interval [26,95) [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858] [v2:172.21.15.163:6828/3467616858,v1:172.21.15.163:6829/3467616858] exists,up 6d47d9fe-d7d7-4505-abf8-22ac463ee305 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 up in weight 1 up_from 31 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687] [v2:172.21.15.181:6802/2565848687,v1:172.21.15.181:6803/2565848687] exists,up ad58407a-b12d-4543-8092-f0ba514729f5 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 up in weight 1 up_from 39 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307] [v2:172.21.15.181:6810/2772326307,v1:172.21.15.181:6811/2772326307] exists,up 21ccce9f-28b2-4624-8a70-2f726e7c45d1 2023-10-09T04:04:36.765 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6 up in weight 1 up_from 47 up_thru 96 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149] [v2:172.21.15.181:6818/4236482149,v1:172.21.15.181:6819/4236482149] exists,up 69caed62-5f69-4fcd-bbbd-5369ec7bc6e7 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7 up in weight 1 up_from 55 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668] [v2:172.21.15.181:6826/2762148668,v1:172.21.15.181:6827/2762148668] exists,up 3dc4fa53-2d90-441e-a825-74bd4da41f4a 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:pg_upmap_items 4.e [1,5] 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:pg_upmap_items 4.10 [1,2] 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:pg_upmap_items 4.1f [1,2] 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1229473398 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2134854516 expires 2023-10-10T03:48:55.215367+0000 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/65515623 expires 2023-10-10T03:48:32.847175+0000 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:6834/704384906 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2705338472 expires 2023-10-10T03:53:17.088151+0000 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/65515623 expires 2023-10-10T03:48:32.847175+0000 2023-10-09T04:04:36.766 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/330134886 expires 2023-10-10T03:53:17.088151+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1713944666 expires 2023-10-10T03:53:17.088151+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/3907401791 expires 2023-10-10T03:53:17.088151+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/1808329322 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/330134886 expires 2023-10-10T03:53:17.088151+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1847962187 expires 2023-10-10T03:56:35.412470+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/983322351 expires 2023-10-10T03:48:32.847175+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2266209559 expires 2023-10-10T03:48:55.215367+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/3524034575 expires 2023-10-10T03:56:35.412470+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:6835/704384906 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/814100112 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/2543837887 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.767 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/277872466 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/647401329 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1488239398 expires 2023-10-10T03:48:32.847175+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/3683659572 expires 2023-10-10T03:48:55.215367+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/3987314134 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2462309240 expires 2023-10-10T03:56:35.412470+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/343886015 expires 2023-10-10T03:53:17.088151+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/277872466 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/55760003 expires 2023-10-10T03:56:35.412470+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6800/3683659572 expires 2023-10-10T03:48:55.215367+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:6801/55760003 expires 2023-10-10T03:56:35.412470+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.181:0/1614027396 expires 2023-10-10T03:57:13.005970+0000 2023-10-09T04:04:36.768 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/3405897408 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1361101496 expires 2023-10-10T03:56:35.412470+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2271486409 expires 2023-10-10T03:48:55.215367+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/611815376 expires 2023-10-10T03:48:32.847175+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1275790360 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/4218140698 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/1802556470 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.769 INFO:teuthology.orchestra.run.smithi163.stdout:blocklist 172.21.15.163:0/2380156011 expires 2023-10-10T03:57:49.149190+0000 2023-10-09T04:04:36.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:36 smithi181 ceph-mon[196979]: pgmap v226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:37.407 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 84 up_thru 99 down_at 81 last_clean_interval [8,80) [v2:172.21.15.163:6802/658095252,v1:172.21.15.163:6803/658095252] [v2:172.21.15.163:6804/658095252,v1:172.21.15.163:6805/658095252] exists,up 9f39300f-5c82-4020-b75d-28b9705365f3', 'osd.1 up in weight 1 up_from 89 up_thru 99 down_at 86 last_clean_interval [13,85) [v2:172.21.15.163:6810/3761898215,v1:172.21.15.163:6811/3761898215] [v2:172.21.15.163:6812/3761898215,v1:172.21.15.163:6813/3761898215] exists,up 07dc6811-c7f1-40ce-83b3-46a33ff57f0a', 'osd.2 up in weight 1 up_from 94 up_thru 99 down_at 91 last_clean_interval [18,90) [v2:172.21.15.163:6818/96613669,v1:172.21.15.163:6819/96613669] [v2:172.21.15.163:6820/96613669,v1:172.21.15.163:6821/96613669] exists,up d5a25867-ce16-4bfd-8356-c346de1fcbf6', 'osd.3 up in weight 1 up_from 99 up_thru 99 down_at 96 last_clean_interval [26,95) [v2:172.21.15.163:6826/3467616858,v1:172.21.15.163:6827/3467616858] [v2:172.21.15.163:6828/3467616858,v1:172.21.15.163:6829/3467616858] exists,up 6d47d9fe-d7d7-4505-abf8-22ac463ee305', 'osd.4 up in weight 1 up_from 31 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6800/2565848687,v1:172.21.15.181:6801/2565848687] [v2:172.21.15.181:6802/2565848687,v1:172.21.15.181:6803/2565848687] exists,up ad58407a-b12d-4543-8092-f0ba514729f5', 'osd.5 up in weight 1 up_from 39 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6808/2772326307,v1:172.21.15.181:6809/2772326307] [v2:172.21.15.181:6810/2772326307,v1:172.21.15.181:6811/2772326307] exists,up 21ccce9f-28b2-4624-8a70-2f726e7c45d1', 'osd.6 up in weight 1 up_from 47 up_thru 96 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6816/4236482149,v1:172.21.15.181:6817/4236482149] [v2:172.21.15.181:6818/4236482149,v1:172.21.15.181:6819/4236482149] exists,up 69caed62-5f69-4fcd-bbbd-5369ec7bc6e7', 'osd.7 up in weight 1 up_from 55 up_thru 99 down_at 0 last_clean_interval [0,0) [v2:172.21.15.181:6824/2762148668,v1:172.21.15.181:6825/2762148668] [v2:172.21.15.181:6826/2762148668,v1:172.21.15.181:6827/2762148668] exists,up 3dc4fa53-2d90-441e-a825-74bd4da41f4a'] 2023-10-09T04:04:37.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T04:04:37.751 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3301480443' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:04:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:37 smithi181 ceph-mon[196979]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3301480443' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:04:37.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:37 smithi163 ceph-mon[213841]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:37.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3301480443' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:04:37.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:37 smithi163 ceph-mon[221067]: pgmap v227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "name": "mon.a", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster": "ceph", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "admin_socket_mode": "", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_client_required": "cephx, none", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_cluster_required": "cephx", 2023-10-09T04:04:39.287 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_debug": "true", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_service_required": "cephx", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "auth_supported": "", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio": "true", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_aio_reap_max": "16", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_async_discard": "false", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_block_size": "4096", 2023-10-09T04:04:39.288 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_aio": "false", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_enable_discard": "false", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_flock_retry": "3", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_inject_crash": "0", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_ioring": "false", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_ioring_hipri": "false", 2023-10-09T04:04:39.289 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bdev_type": "", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_allocator": "hybrid", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_buffered_io": "true", 2023-10-09T04:04:39.290 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T04:04:39.291 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_replay_recovery": "false", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluefs_sync_write": "false", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_allocator": "hybrid", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T04:04:39.292 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_create": "true", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_db_create": "false", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_db_path": "", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_db_size": "0", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_path": "", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T04:04:39.293 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_size": "107374182400", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_wal_create": "false", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_wal_path": "", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs": "true", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:04:39.294 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_autotune": "true", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_size": "0", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T04:04:39.295 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cache_type": "2q", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_clone_cow": "true", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T04:04:39.296 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_mode": "none", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_freelist": "false", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T04:04:39.297 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T04:04:39.298 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fail_eio": "false", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T04:04:39.299 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T04:04:39.300 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_blob_size": "0", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T04:04:39.301 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T04:04:39.302 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T04:04:39.302 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T04:04:39.302 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T04:04:39.302 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T04:04:39.302 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T04:04:39.302 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T04:04:39.305 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T04:04:39.305 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T04:04:39.305 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T04:04:39.305 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_spdk_mem": "512", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T04:04:39.306 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_tracing": "false", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T04:04:39.307 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T04:04:39.308 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_cluster_require_version": "2", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_require_signatures": "false", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_require_version": "2", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_service_require_signatures": "false", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_service_require_version": "2", 2023-10-09T04:04:39.309 INFO:teuthology.orchestra.run.smithi163.stdout: "cephx_sign_messages": "true", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "chdir": "", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_acl_type": "", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_asio_thread_count": "2", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_cache_mid": "0.750000", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_cache_size": "16384", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_caps_release_delay": "5", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_check_pool_perm": "true", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_debug_force_sync_read": "false", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_debug_getattr_caps": "false", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T04:04:39.310 INFO:teuthology.orchestra.run.smithi163.stdout: "client_die_on_failed_remount": "false", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_dirsize_rbytes": "true", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_force_lazyio": "false", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_fs": "", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_inject_release_failure": "false", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_max_inline_size": "4096", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mds_namespace": "", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_metadata": "", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mount_gid": "-1", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mount_timeout": "300", 2023-10-09T04:04:39.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mount_uid": "-1", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_mountpoint": "/", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_notify_timeout": "10", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc": "true", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_max_objects": "1000", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_size": "209715200", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_permissions": "true", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_quota": "true", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_quota_df": "true", 2023-10-09T04:04:39.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_readahead_max_bytes": "0", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_readahead_max_periods": "4", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_readahead_min": "131072", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_reconnect_stale": "false", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_shutdown_timeout": "30", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_snapdir": ".snap", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_tick_interval": "1", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_trace": "", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_use_faked_inos": "false", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_use_random_mds": "false", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_graylog": "false", 2023-10-09T04:04:39.313 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_graylog_port": "12201", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_monitors": "default=true", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_syslog": "false", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "clog_to_syslog_level": "info", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster_addr": "-", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster_network": "", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "cluster_network_interface": "", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zlib_isal": "false", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zlib_level": "5", 2023-10-09T04:04:39.314 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "compressor_zstd_level": "1", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crimson_seastar_smp": "1", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crush_location": "", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crush_location_hook": "", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "crush_location_hook_timeout": "10", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "daemonize": "false", 2023-10-09T04:04:39.315 INFO:teuthology.orchestra.run.smithi163.stdout: "daos_pool": "tank", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_alienstore": "0/5", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_asok": "1/5", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_asok_assert_abort": "false", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_auth": "1/5", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_bdev": "1/3", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_bluefs": "1/5", 2023-10-09T04:04:39.316 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_bluestore": "1/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_buffer": "0/1", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_cephsqlite": "0/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_client": "0/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_compressor": "1/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_context": "0/1", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_crush": "1/1", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_crypto": "1/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_cyanstore": "0/5", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T04:04:39.317 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_dpdk": "1/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_eventtrace": "1/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_filer": "0/1", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_filestore": "1/3", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_finisher": "1/1", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_fuse": "1/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_javaclient": "1/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_journal": "1/3", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_journaler": "0/5", 2023-10-09T04:04:39.318 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_kstore": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_leveldb": "4/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_lockdep": "0/1", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mclock": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_balancer": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_locker": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_log": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mds_migrator": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_memstore": "1/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mgr": "2/5", 2023-10-09T04:04:39.319 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mgrc": "1/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_mon": "20/20", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_monc": "0/10", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_ms": "1/1", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_none": "0/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_objclass": "0/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_objectcacher": "0/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_objecter": "0/1", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_optracker": "0/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_osd": "1/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_paxos": "20/20", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_perfcounter": "1/5", 2023-10-09T04:04:39.320 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_prioritycache": "1/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rados": "0/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd": "0/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rbd_replay": "0/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_refs": "0/0", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_reserver": "1/1", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw": "1/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_access": "1/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_flight": "1/5", 2023-10-09T04:04:39.321 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rgw_sync": "1/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_rocksdb": "4/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_backref": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_cache": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_device": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_epm": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_journal": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_lba": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_odata": "0/5", 2023-10-09T04:04:39.322 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_omap": "0/5", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_onode": "0/5", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_t": "0/5", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_seastore_tm": "0/5", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_striper": "0/1", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_test": "0/5", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_throttle": "1/1", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_timer": "0/1", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "debug_tp": "0/5", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "device_failure_prediction_mode": "none", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T04:04:39.323 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_graylog": "false", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_journald": "false", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_stderr": "true", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "err_to_syslog": "false", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event_tracing": "false", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_http_port": "9926", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_prio_limit": "5", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_sort_metrics": "true", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "exporter_stats_period": "5", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "fake_statfs_for_testing": "0", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "fatal_signal_handlers": "true", 2023-10-09T04:04:39.324 INFO:teuthology.orchestra.run.smithi163.stdout: "filer_max_purge_ops": "10", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filer_max_truncate_ops": "128", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_blackhole": "false", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_btrfs_snap": "true", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_caller_concurrency": "10", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_omap_check": "false", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T04:04:39.325 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_debug_verify_split": "false", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_dump_file": "", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fadvise": "true", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fail_eio": "true", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fd_cache_size": "128", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fiemap": "false", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T04:04:39.326 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_inject_stall": "0", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_journal_parallel": "false", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_journal_trailing": "false", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_journal_writeahead": "false", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_kill_at": "0", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T04:04:39.327 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_merge_threshold": "-10", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_odsync_write": "false", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_omap_backend_path": "", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T04:04:39.328 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_op_threads": "2", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_punch_hole": "false", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T04:04:39.329 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_queue_max_ops": "50", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_seek_data_hole": "false", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_sloppy_crc": "false", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_splice": "false", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_split_multiple": "2", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_split_rand_factor": "20", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_update_to": "1000", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T04:04:39.330 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_xfs_extsize": "false", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "filestore_zfs_snap": "false", 2023-10-09T04:04:39.331 INFO:teuthology.orchestra.run.smithi163.stdout: "fio_dir": "/tmp/fio", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fsid": "8628d772-6656-11ee-8db5-212e2dc638e7", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_allow_other": "true", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_big_writes": "true", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_debug": "false", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_default_permissions": "false", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_disable_pagecache": "false", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_max_write": "0", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_multithreaded": "true", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_require_active_mds": "true", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_set_user_groups": "true", 2023-10-09T04:04:39.332 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_splice_move": "true", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_splice_read": "true", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_splice_write": "true", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "gss_target_name": "ceph", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "heartbeat_file": "", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "heartbeat_inject_failure": "0", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "heartbeat_interval": "5", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "host": "smithi163", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T04:04:39.333 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "inject_early_sigterm": "false", 2023-10-09T04:04:39.334 INFO:teuthology.orchestra.run.smithi163.stdout: "jaeger_agent_port": "6799", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "jaeger_tracing_enable": "false", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_aio": "true", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_align_min_size": "65536", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_block_align": "true", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_block_size": "4096", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_dio": "true", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_discard": "false", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_force_aio": "false", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_ignore_corruption": "false", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_max_write_entries": "100", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_replay_from": "0", 2023-10-09T04:04:39.335 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_write_header_frequency": "0", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journal_zero_on_create": "false", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journaler_prefetch_periods": "10", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journaler_prezero_periods": "5", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "journaler_write_head_interval": "15", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "key": "", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "keyfile": "", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T04:04:39.336 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_backend": "rocksdb", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_max_bytes": "67108864", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_max_ops": "512", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_onode_map_size": "1024", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "kstore_sync_transaction": "false", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_block_size": "65536", 2023-10-09T04:04:39.337 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_bloom_size": "0", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_cache_size": "536870912", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_compression": "false", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_log": "", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_max_open_files": "0", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_paranoid": "false", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "librados_thread_count": "2", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "lockdep": "false", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "lockdep_force_backtrace": "false", 2023-10-09T04:04:39.338 INFO:teuthology.orchestra.run.smithi163.stdout: "log_coarse_timestamps": "true", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_flush_on_exit": "false", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_graylog_port": "12201", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_max_new": "1000", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_max_recent": "10000", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_stderr_prefix": "", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_file": "true", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_graylog": "false", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_journald": "false", 2023-10-09T04:04:39.339 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_stderr": "false", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "log_to_syslog": "false", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_action_on_write_error": "1", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_alternate_name_max": "8192", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_asio_thread_count": "2", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_export_pin": "true", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T04:04:39.340 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_interval": "10", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_max": "-1", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_max_until": "-1", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_merge_size": "50", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_mode": "0", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T04:04:39.341 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_bits": "3", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_size": "10000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T04:04:39.342 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_mid": "0.700000", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_trim_interval": "1", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T04:04:39.343 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_auth_pins": "false", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_frag": "false", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_scatterstat": "false", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_debug_subtrees": "false", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_default_dir_hash": "2", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_defer_session_stale": "true", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T04:04:39.344 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_max_entries": "0", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dir_prefetch": "true", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_early_reply": "true", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_enable_op_tracker": "true", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_enforce_unique_name": "true", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T04:04:39.345 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_health_dummy": "false", 2023-10-09T04:04:39.346 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_join_fs": "", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_journal_format": "1", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_create_at": "0", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_export_at": "0", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_import_at": "0", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_journal_at": "0", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T04:04:39.347 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_link_at": "0", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_openc_at": "0", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_rename_at": "0", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_max_events": "-1", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_max_segments": "128", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_pause": "false", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_segment_size": "0", 2023-10-09T04:04:39.348 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_completed_requests": "100000", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_export_size": "20971520", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_file_recover": "32", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_purge_files": "64", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_purge_ops": "8192", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T04:04:39.349 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_metrics_update_interval": "2", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_min_caps_per_client": "100", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_numa_node": "-1", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_history_duration": "600", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_history_size": "20", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_op_log_threshold": "5", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_ping_grace": "15", 2023-10-09T04:04:39.350 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_ping_interval": "5", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_max_caps": "30000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_replay_interval": "1.000000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T04:04:39.351 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_root_ino_gid": "0", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_root_ino_uid": "0", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_shutdown_check": "0", 2023-10-09T04:04:39.352 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_skip_ino": "0", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_snap_min_uid": "0", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_snap_rstat": "false", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_symlink_recovery": "true", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_thrash_exports": "0", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_thrash_fragments": "0", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_tick_interval": "5.000000", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_valgrind_exit": "false", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_verify_backtrace": "1", 2023-10-09T04:04:39.353 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_verify_scatter": "false", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mds_wipe_sessions": "false", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mempool_debug": "false", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_page_set": "false", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "memstore_page_size": "65536", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_client_bytes": "134217728", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_client_messages": "512", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T04:04:39.354 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_disabled_modules": "", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mds_messages": "128", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_mon_messages": "128", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_osd_messages": "8192", 2023-10-09T04:04:39.355 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_pool": "true", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_standby_modules": "true", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_stats_period": "5", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_stats_threshold": "5", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_tick_period": "2", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_allow_pool_delete": "true", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T04:04:39.356 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_bytes": "104857600", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T04:04:39.357 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T04:04:39.358 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_compact_on_start": "false", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_compact_on_trim": "true", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_cpu_threads": "4", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data_avail_crit": "5", 2023-10-09T04:04:39.359 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data_avail_warn": "5", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_dump_json": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_extra_checks": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T04:04:39.360 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_election_default_strategy": "3", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_election_timeout": "5.000000", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_enable_op_tracker": "true", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_fake_pool_delete": "false", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_force_quorum_join": "false", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T04:04:39.361 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_log_update_period": "5", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_max_detail": "50", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_to_clog": "true", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_host": "[v2:172.21.15.163:3300/0,v1:172.21.15.163:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] [v2:172.21.15.163:3301/0,v1:172.21.15.163:6790/0]", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_host_override": "", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_initial_members": "", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T04:04:39.362 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_lease": "5.000000", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_log_full_interval": "50", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_log_max": "10000", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_log_max_summary": "50", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_log_epochs": "500", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T04:04:39.363 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_osd": "10000", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_memory_autotune": "true", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_memory_target": "2147483648", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_digest_period": "5", 2023-10-09T04:04:39.364 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_complaint_time": "30", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_duration": "600", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_size": "20", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_op_log_threshold": "5", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T04:04:39.365 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_cache_size": "500", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T04:04:39.366 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T04:04:39.367 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_report_timeout": "900", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T04:04:39.368 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_max_osds": "4", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T04:04:39.369 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_interval": "86400", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_max_keys": "100", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_scrub_timeout": "300", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_session_timeout": "300", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_smart_report_timeout": "5", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_pool_size": "4", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T04:04:39.370 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_debug": "false", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_tick_interval": "5", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T04:04:39.371 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_older_version": "true", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T04:04:39.372 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "monmap": "", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T04:04:39.373 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "motr_tracing_enabled": "false", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_op_threads": "3", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_cm": "false", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_device_name": "", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T04:04:39.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_sl": "3", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_rdma_type": "ib", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_async_reap_threshold": "5", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_before_connect": "false", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_ipv4": "true", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_ipv6": "false", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_msgr1": "true", 2023-10-09T04:04:39.375 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_msgr2": "true", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_port_max": "7568", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_port_min": "6800", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_retry_count": "3", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_bind_retry_delay": "5", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_client": "false", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_mds": "false", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_mgr": "false", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_mon": "false", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_blackhole_osd": "false", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_client_mode": "crc secure", 2023-10-09T04:04:39.376 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_cluster_type": "", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_compress_secure": "false", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_crc_data": "true", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_crc_header": "true", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_bug": "true", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_old_message": "true", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T04:04:39.377 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_hugepages": "", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_lro": "true", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T04:04:39.378 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_pmd": "", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_port_id": "0", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_dump_on_send": "false", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_delay_type": "", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_network_congestion": "0", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_inject_socket_failures": "0", 2023-10-09T04:04:39.379 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_max_accept_failures": "4", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_max_backoff": "15.000000", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_osd_compress_mode": "none", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_pq_min_cost": "65536", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_public_type": "", 2023-10-09T04:04:39.380 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_service_mode": "crc secure", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_nodelay": "true", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "ms_type": "async+posix", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "no_config_file": "false", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_inflight_ops": "1024", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T04:04:39.381 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "objecter_timeout": "10.000000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "objectstore_blackhole": "false", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "openssl_engine_opts": "", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_max_ops": "4", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_agent_slop": "0.020000", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T04:04:39.382 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_asio_thread_count": "2", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backfill_scan_max": "512", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backfill_scan_min": "64", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backoff_on_peering": "false", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_beacon_report_interval": "300", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_duration": "30", 2023-10-09T04:04:39.383 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_blkin_trace_all": "false", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T04:04:39.384 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_class_update_on_start": "true", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_message_cap": "256", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_op_priority": "63", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_client_watch_timeout": "30", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_command_max_records": "256", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_compact_on_start": "false", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_update_on_start": "true", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T04:04:39.385 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_op_order": "false", 2023-10-09T04:04:39.386 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_shutdown": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T04:04:39.387 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_default_notify_timeout": "30", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T04:04:39.388 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_enable_op_tracker": "true", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_info": "true", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_shutdown": "true", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T04:04:39.389 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_function_tracing": "false", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_grace": "20", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_interval": "6", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_stale": "600", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T04:04:39.390 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_journal_size": "5120", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_kill_backfill_at": "0", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_cache_size": "50", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_dedup": "true", 2023-10-09T04:04:39.391 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_message_max": "40", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_attr_name_len": "100", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_attr_size": "0", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_backfills": "1", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_markdown_count": "5", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_markdown_period": "600", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_object_name_len": "2048", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_object_size": "134217728", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T04:04:39.392 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_pgls": "1024", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_push_cost": "8388608", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_push_objects": "10", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_scrubs": "1", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_max_write_size": "90", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T04:04:39.393 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_profile": "balanced", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T04:04:39.394 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_base": "805306368", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_target": "4294967296", 2023-10-09T04:04:39.395 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_target_autotune": "false", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_min_recovery_priority": "0", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_report_interval": "5", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_num_cache_shards": "32", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_numa_node": "-1", 2023-10-09T04:04:39.396 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objecter_finishers": "1", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objectstore": "bluestore", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objectstore_fuse": "false", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_objectstore_tracing": "false", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_duration": "600", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_size": "20", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_log_threshold": "5", 2023-10-09T04:04:39.397 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_shards": "0", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_op_thread_timeout": "15", 2023-10-09T04:04:39.398 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_open_classes_on_start": "true", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_os_flags": "0", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_peering_op_priority": "255", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_delete_priority": "5", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T04:04:39.399 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_crimson": "false", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T04:04:39.400 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_flags": "0", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_min_size": "0", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_size": "3", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_default_type": "replicated", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T04:04:39.401 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_cost": "20971520", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_active": "0", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T04:04:39.402 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_op_priority": "3", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_priority": "5", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_repair_during_recovery": "false", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T04:04:39.403 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_cost": "52428800", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_end_hour": "0", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T04:04:39.404 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_priority": "5", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T04:04:39.405 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_skip_data_digest": "false", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_smart_report_timeout": "5", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_priority": "5", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T04:04:39.406 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_target_transaction_size": "30", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T04:04:39.407 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_tracing": "false", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_use_stale_snap": "false", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_kill_at": "0", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_max_join_drift": "10", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_min": "500", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_min_wait": "0.050000", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_service_trim_max": "500", 2023-10-09T04:04:39.408 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_service_trim_min": "250", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_stash_full_interval": "25", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_trim_max": "500", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "paxos_trim_min": "250", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "perf": "true", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "pid_file": "", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "public_addr": "-", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "public_addrv": "", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "public_bind_addr": "-", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "public_network": "172.21.0.0/20", 2023-10-09T04:04:39.409 INFO:teuthology.orchestra.run.smithi163.stdout: "public_network_interface": "", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "qat_compressor_enabled": "false", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rados_mon_op_timeout": "0", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rados_osd_op_timeout": "0", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rados_tracing": "false", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_atime_update_interval": "60", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T04:04:39.410 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache": "true", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_policy": "writearound", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_size": "33554432", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_compression_hint": "none", 2023-10-09T04:04:39.411 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_clone_format": "auto", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_data_pool": "", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_features": "61", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_format": "2", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_map_options": "", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_order": "22", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_pool": "rbd", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_stripe_count": "0", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T04:04:39.412 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_io_scheduler": "simple", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T04:04:39.413 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_order": "24", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_pool": "", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_journal_splay_width": "4", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T04:04:39.414 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T04:04:39.415 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T04:04:39.416 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_op_threads": "1", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_plugins": "", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T04:04:39.417 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T04:04:39.418 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T04:04:39.419 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_tracing": "false", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_validate_names": "true", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rbd_validate_pool": "true", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "restapi_base_url": "", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "restapi_log_level": "", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_admin_entry": "admin", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_backend_store": "rados", 2023-10-09T04:04:39.420 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_barbican_url": "", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_beast_enable_async": "true", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cache_enabled": "true", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T04:04:39.421 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_config_store": "rados", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_content_length_compat": "false", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_cross_domain_policy": "", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T04:04:39.422 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T04:04:39.423 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T04:04:39.424 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T04:04:39.425 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_log_window": "30", 2023-10-09T04:04:39.426 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T04:04:39.427 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dns_name": "", 2023-10-09T04:04:39.428 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dns_s3website_name": "", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_ops_log": "false", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_static_website": "false", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enable_usage_log": "false", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_expose_bucket": "false", 2023-10-09T04:04:39.429 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_extended_http_attrs": "", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_filter": "none", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_objs": "32", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T04:04:39.430 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_init_timeout": "300", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_password": "", 2023-10-09T04:04:39.431 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_project": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_token": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_admin_user": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_api_version": "2", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T04:04:39.432 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_url": "", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_objs": "32", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T04:04:39.433 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_worker": "3", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T04:04:39.434 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_http_headers": "", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_attr_size": "0", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T04:04:39.435 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_listing_results": "1000", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_notify_retries": "10", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T04:04:39.436 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T04:04:39.437 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_num_control_oids": "8", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_numa_node": "-1", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T04:04:39.438 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_op_tracing": "false", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_opa_token": "", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_opa_url": "", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_rados": "false", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T04:04:39.439 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_print_continue": "true", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T04:04:39.440 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rados_tracing": "false", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_realm": "", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_realm_id": "", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_region": "", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_request_uri": "", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T04:04:39.441 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_resolve_cname": "false", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_run_sync_thread": "true", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T04:04:39.442 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_script_uri": "", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_service_provider_name": "", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_client_id": "", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_client_secret": "", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_entry": "sts", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_key": "sts", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T04:04:39.443 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_auth_url": "", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_custom_header": "", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_need_stats": "true", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_tenant_name": "", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_url": "", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T04:04:39.444 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_lease_period": "120", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T04:04:39.445 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_thread_pool_size": "512", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_comment": "", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_createby": "", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_encoding": "", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_flag": "false", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_origin": "", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_torrent_tracker": "", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T04:04:39.446 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_max_shards": "32", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_use_opa_authz": "false", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_user_unique_email": "true", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_verify_ssl": "true", 2023-10-09T04:04:39.447 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zone": "", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zone_id": "", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zonegroup": "", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zonegroup_id": "", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_block_size": "4096", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T04:04:39.448 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T04:04:39.449 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_partition_filters": "false", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_perf": "false", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "run_dir": "/var/run/ceph", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_block_create": "true", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T04:04:39.450 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_device_size": "53687091200", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_main_device_type": "SSD", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T04:04:39.451 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "seastore_segment_size": "67108864", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "set_keepcaps": "false", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "setgroup": "ceph", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "setuser": "ceph", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "setuser_match_path": "", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "thp": "false", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "threadpool_default_timeout": "60", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout: "throttler_perf_counter": "true" 2023-10-09T04:04:39.452 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:04:39.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T04:04:40.294 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:40.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:40 smithi163 ceph-mon[221067]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:40.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:40 smithi163 ceph-mon[213841]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:40 smithi181 ceph-mon[196979]: pgmap v228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:41.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:04:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:04:40] "GET /metrics HTTP/1.1" 200 34045 "" "Prometheus/2.43.0" 2023-10-09T04:04:41.691 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a: {} 2023-10-09T04:04:41.701 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b: {} 2023-10-09T04:04:41.710 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c: {} 2023-10-09T04:04:41.711 INFO:teuthology.orchestra.run.smithi163.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:04:41.712 INFO:teuthology.orchestra.run.smithi163.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:04:41.712 INFO:teuthology.orchestra.run.smithi163.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:04:42.328 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T04:04:42.328 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T04:04:42.329 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:04:42.331 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:04:42.331 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T04:04:42.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T04:04:42.331 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T04:04:42.332 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:04:42.332 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:04:42.332 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T04:04:42.332 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:04:42.332 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:04:42.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T04:04:42.333 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T04:04:42.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:04:42.333 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T04:04:42.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:04:42.334 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T04:04:42.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:04:42.334 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T04:04:42.334 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:04:42.384 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:42.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:04:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:42.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:04:42.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:42 smithi163 ceph-mon[213841]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/362083351' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:04:42.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:42 smithi163 ceph-mon[221067]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:42.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/362083351' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:04:42.583 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:42.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:42.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:42.759 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:04:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:42 smithi181 ceph-mon[196979]: pgmap v229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/362083351' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:04:42.781 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:42.796 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:42.826 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:42.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:04:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:42.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:04:42.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:04:42.981 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:43.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:04:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:43.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:04:43.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:43.202 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:43.222 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:43.244 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:43.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:43.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:43.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:43.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:04:43.385 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:43.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:04:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:43.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:04:43.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:43.669 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:43.714 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:43.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:43.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:04:43.848 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:43.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:04:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:43.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:04:44.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:04:44.026 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:44.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:04:44.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:44.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:04:44.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:44.205 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:44.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:44 smithi163 ceph-mon[213841]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:44.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:44 smithi163 ceph-mon[221067]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:44 smithi181 ceph-mon[196979]: pgmap v230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:45.656 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:45.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:45.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:45.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:04:45.821 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:46.419 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:04:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:46.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:04:46.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:46.555 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:46 smithi163 ceph-mon[221067]: pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:46 smithi163 ceph-mon[213841]: pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:46.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:46.608 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:46 smithi181 ceph-mon[196979]: pgmap v231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:46.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:46.710 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:46.740 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:46.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:46.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:46.834 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:04:46.850 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:04:46.868 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:04:46.888 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:04:46.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:04:46.906 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:04:46.924 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:46.927 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:04:46.949 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:04:46.969 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:04:46.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.971 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:04:46.971 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:04:47.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:04:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:47.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:04:47.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:47.306 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:47.342 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:04:47.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:47.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:47.366 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:04:47.366 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:04:47.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:47.487 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:47.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1168759866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:04:47.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:04:47.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2237704874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:04:47.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1168759866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:04:47.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:04:47.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2237704874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:04:47.530 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:47.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:47.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:47.665 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1168759866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:04:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:04:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2237704874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:04:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[213841]: pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:04:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[213841]: failed to encode map e101 with expected crc 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[213841]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[221067]: pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[221067]: failed to encode map e101 with expected crc 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:04:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:48 smithi163 ceph-mon[221067]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T04:04:48.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:04:48.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:48 smithi181 ceph-mon[196979]: pgmap v232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:48 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:04:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:48 smithi181 ceph-mon[196979]: failed to encode map e101 with expected crc 2023-10-09T04:04:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:04:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:48 smithi181 ceph-mon[196979]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T04:04:48.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:48.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:48.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:04:48.945 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:49.219 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:04:49.219 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:04:49.219 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:04:49.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:04:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:49.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:04:49.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:49.594 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:49.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:49.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:49.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:49.777 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:49.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:49.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:49.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:04:49.956 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:50.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:04:50.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:50.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:04:50.083 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:50.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:50.182 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:50.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:50.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:50.269 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_0' created 2023-10-09T04:04:50.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:50.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:50.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[213841]: pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:50.277 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2629465027' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:04:50.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:04:50.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:50.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:50.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[221067]: pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:50.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2629465027' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:04:50.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:04:50.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:50.356 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:50.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:50.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:50.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:04:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:04:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:50 smithi181 ceph-mon[196979]: pgmap v234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2629465027' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:04:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:04:50.529 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:50.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:04:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:50.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:04:50.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:04:50.707 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:50.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:04:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:50.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:04:50.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:04:50.919 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:04:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:04:50] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2023-10-09T04:04:50.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:50.982 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:51.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:51.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:51.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:51.131 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:51.158 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:51.201 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:51.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:51.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:51.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:51.334 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:51.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:51.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:51.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T04:04:51.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:04:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T04:04:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:51 smithi181 ceph-mon[196979]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.545 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[221067]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:04:51.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T04:04:51.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:51 smithi163 ceph-mon[213841]: failed to encode map e102 with expected crc 2023-10-09T04:04:51.549 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:51.596 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:51.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:51.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:51.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:04:51.739 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:51.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:04:51.936 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:04:51.936 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:04:51.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:04:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:51.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:04:52.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:52.073 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:52.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:52.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:52.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:52.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:52.255 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:52.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[221067]: pgmap v237: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:04:52.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:52 smithi163 ceph-mon[213841]: pgmap v237: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: failed to encode map e103 with expected crc 2023-10-09T04:04:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:52 smithi181 ceph-mon[196979]: pgmap v237: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:52.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:04:52.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:52.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:52.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:53.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:53.064 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:53.111 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:53.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:53.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:04:53.243 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:53.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:04:53.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:53.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:04:53.354 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:53.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:53.472 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:53.472 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:53.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:53 smithi181 ceph-mon[196979]: failed to encode map e104 with expected crc 2023-10-09T04:04:53.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:53 smithi181 ceph-mon[196979]: failed to encode map e104 with expected crc 2023-10-09T04:04:53.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:53 smithi181 ceph-mon[196979]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:04:53.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:53 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:53.480 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:53.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:53.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[221067]: failed to encode map e104 with expected crc 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[221067]: failed to encode map e104 with expected crc 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[221067]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[213841]: failed to encode map e104 with expected crc 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[213841]: failed to encode map e104 with expected crc 2023-10-09T04:04:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[213841]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:04:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:53 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:53.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:53.658 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:53.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:53.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:53.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:53.841 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:53.937 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:53.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:54.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:54.075 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:54.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:54.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:54.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:04:54.308 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:54.332 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:04:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[213841]: pgmap v239: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[213841]: failed to encode map e104 with expected crc 2023-10-09T04:04:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2974917245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:04:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:04:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[221067]: pgmap v239: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[221067]: failed to encode map e104 with expected crc 2023-10-09T04:04:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2974917245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:04:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:04:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:54 smithi181 ceph-mon[196979]: pgmap v239: 89 pgs: 12 unknown, 77 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:54 smithi181 ceph-mon[196979]: failed to encode map e104 with expected crc 2023-10-09T04:04:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2974917245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:04:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:04:55.016 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:04:55.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:04:55.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:04:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:55.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:04:55.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:55.490 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:55.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:55.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:55.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:04:55.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:04:55.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[213841]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[213841]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:04:55.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[213841]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[213841]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[221067]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[221067]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[221067]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:04:55.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:55 smithi163 ceph-mon[221067]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.666 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:55 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:04:55.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:55 smithi181 ceph-mon[196979]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:55 smithi181 ceph-mon[196979]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:55 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:04:55.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:55 smithi181 ceph-mon[196979]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:04:55.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:55 smithi181 ceph-mon[196979]: failed to encode map e105 with expected crc 2023-10-09T04:04:55.669 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:55.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:04:55.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:55.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:04:55.866 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:55.892 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:55.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:55.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:55.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:56.042 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:56.067 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:56.109 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:56.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:56.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:04:56.247 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:56 smithi163 ceph-mon[221067]: pgmap v241: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:56 smithi163 ceph-mon[213841]: pgmap v241: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:56.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:56 smithi181 ceph-mon[196979]: pgmap v241: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:04:56.897 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:04:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:56.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:04:57.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:04:57.033 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:57.089 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:04:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:57.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:04:57.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:04:57.223 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:57.223 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:57.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:04:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:57.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:04:57.324 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:04:57.324 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:04:57.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:57.401 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:57.442 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:57.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:57.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:04:57.577 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:57.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:04:58.033 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:04:58.052 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:04:58.071 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:04:58.091 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:04:58.111 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:04:58.115 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T04:04:58.131 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:04:58.150 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:04:58.170 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.172 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:04:58.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:58 smithi163 ceph-mon[221067]: pgmap v242: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:58.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4088722090' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:04:58.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/654160461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:04:58.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:58 smithi163 ceph-mon[213841]: pgmap v242: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:58.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4088722090' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:04:58.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/654160461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:04:58.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:04:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:58.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:04:58.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:04:58.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:58 smithi181 ceph-mon[196979]: pgmap v242: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:04:58.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4088722090' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:04:58.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/654160461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:04:58.511 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:58.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:04:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:58.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:04:58.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:04:58.697 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:58.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:04:58.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:04:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:58.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:04:58.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:04:58.873 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:59.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:04:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:59.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:04:59.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:04:59.153 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[221067]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4088722090' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:04:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[221067]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:04:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[221067]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[221067]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[213841]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4088722090' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:04:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[213841]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:04:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[213841]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:04:59 smithi163 ceph-mon[213841]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:59 smithi181 ceph-mon[196979]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4088722090' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:04:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:59 smithi181 ceph-mon[196979]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:04:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:59 smithi181 ceph-mon[196979]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:04:59 smithi181 ceph-mon[196979]: failed to encode map e106 with expected crc 2023-10-09T04:04:59.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:04:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:59.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:04:59.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:04:59.894 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:04:59.926 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T04:04:59.929 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:04:59.929 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:04:59.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:04:59.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:04:59.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:04:59.974 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:59.979 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T04:04:59.979 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:04:59.979 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_287034 2023-10-09T04:04:59.979 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:04:59.979 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:04:59.998 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:00.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:00.088 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:00.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:00.153 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:00 smithi181 ceph-mon[196979]: pgmap v244: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:05:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:00.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:00.256 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:00.296 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:00.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:00.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:00.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:00 smithi163 ceph-mon[213841]: pgmap v244: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:05:00.448 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:00 smithi163 ceph-mon[221067]: pgmap v244: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T04:05:00.487 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:00.560 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:00.704 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:05:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:05:00] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2023-10-09T04:05:00.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:00.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:00.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:00.890 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:00.980 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2824 2808 175.49 175.5 0.00475854 0.00567994 2023-10-09T04:05:01.152 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:05:01.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/410733649' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:05:01.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/410733649' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:05:01.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/410733649' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:05:01.650 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:01.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:01.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:05:01.755 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:01.819 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:01.982 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 6090 6074 189.8 204.125 0.00481572 0.0052538 2023-10-09T04:05:02.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:02 smithi181 ceph-mon[196979]: failed to encode map e107 with expected crc 2023-10-09T04:05:02.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/410733649' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:05:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:02 smithi181 ceph-mon[196979]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:05:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:02 smithi181 ceph-mon[196979]: failed to encode map e107 with expected crc 2023-10-09T04:05:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:02 smithi181 ceph-mon[196979]: pgmap v246: 89 pgs: 89 active+clean; 31 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2023-10-09T04:05:02.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[213841]: failed to encode map e107 with expected crc 2023-10-09T04:05:02.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/410733649' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:05:02.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[213841]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:05:02.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[213841]: failed to encode map e107 with expected crc 2023-10-09T04:05:02.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[213841]: pgmap v246: 89 pgs: 89 active+clean; 31 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2023-10-09T04:05:02.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[221067]: failed to encode map e107 with expected crc 2023-10-09T04:05:02.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/410733649' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:05:02.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[221067]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:05:02.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[221067]: failed to encode map e107 with expected crc 2023-10-09T04:05:02.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:02 smithi163 ceph-mon[221067]: pgmap v246: 89 pgs: 89 active+clean; 31 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2023-10-09T04:05:02.980 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 9418 9402 195.86 208 0.00286515 0.00509624 2023-10-09T04:05:03.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:03.983 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 11843 11827 184.622 151.562 0.00682011 0.00540996 2023-10-09T04:05:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:03.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:04.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:04.151 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:04.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:04.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:04.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:04.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:04.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:04 smithi163 ceph-mon[213841]: pgmap v247: 89 pgs: 89 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s wr, 294 op/s 2023-10-09T04:05:04.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:04.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:04 smithi163 ceph-mon[221067]: pgmap v247: 89 pgs: 89 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s wr, 294 op/s 2023-10-09T04:05:04.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:04.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:05:04.678 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:04 smithi181 ceph-mon[196979]: pgmap v247: 89 pgs: 89 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s wr, 294 op/s 2023-10-09T04:05:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:04.983 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 14182 14166 176.936 146.188 0.00876535 0.00564467 2023-10-09T04:05:05.019 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:05.176 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:05:05.176 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:05:05.177 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:05:05.177 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:05:05.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:05.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:05.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:05.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:05.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:05.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:05.820 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:05.984 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 16453 16437 171.097 141.938 0.00934557 0.00583642 2023-10-09T04:05:06.376 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:05:06.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:06 smithi181 ceph-mon[196979]: pgmap v248: 89 pgs: 89 active+clean; 800 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T04:05:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1790393808' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:06 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:06 smithi163 ceph-mon[221067]: pgmap v248: 89 pgs: 89 active+clean; 800 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T04:05:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1790393808' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:06 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:06.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:06 smithi163 ceph-mon[213841]: pgmap v248: 89 pgs: 89 active+clean; 800 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T04:05:06.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1790393808' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:06.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:06 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:06.984 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 18968 18952 169.108 157.188 0.00732271 0.00590748 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:07 smithi181 ceph-mon[196979]: pgmap v250: 89 pgs: 89 active+clean; 898 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T04:05:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[213841]: pgmap v250: 89 pgs: 89 active+clean; 898 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: failed to encode map e108 with expected crc 2023-10-09T04:05:07.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:07 smithi163 ceph-mon[221067]: pgmap v250: 89 pgs: 89 active+clean; 898 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T04:05:07.984 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 20910 20894 163.13 121.375 0.00329841 0.0061236 2023-10-09T04:05:08.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:08.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:08.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:08.227 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:08.312 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:08.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:08.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:08.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:05:08.457 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:08.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:08.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:08.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:08.634 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:08.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:08.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:08.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:08.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:08.833 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:08.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:08.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:08.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:08.989 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 22745 22729 157.672 114.688 0.0102666 0.00633684 2023-10-09T04:05:09.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:09.080 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:09.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:09.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:09.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:09.472 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:09.494 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:09.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:09.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:09.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:09.757 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:09.934 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:09.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:09.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:09.989 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 24533 24517 153.082 111.75 0.00903547 0.00652569 2023-10-09T04:05:10.010 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:05:10.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:05:10.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:10.098 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:05:10.137 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:05:10.142 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:10.169 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:05:10.227 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:05:10.264 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:05:10.304 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:10.305 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:05:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:10.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:05:10.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: pgmap v251: 89 pgs: 89 active+clean; 985 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2322399449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:10.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: pgmap v251: 89 pgs: 89 active+clean; 985 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2322399449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:10.461 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.466 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:10.565 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:10.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:10.671 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:10.721 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:05:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:05:10] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T04:05:10.727 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:10.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: pgmap v251: 89 pgs: 89 active+clean; 985 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:05:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2322399449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:10.989 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 26554 26538 150.65 126.312 0.00484035 0.00663173 2023-10-09T04:05:11.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:05:11.989 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 28378 28362 147.596 114 0.00350268 0.0067321 2023-10-09T04:05:12.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:12.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:12.599 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:12 smithi163 ceph-mon[221067]: pgmap v252: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T04:05:12.600 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:12 smithi163 ceph-mon[213841]: pgmap v252: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T04:05:12.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:12.702 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:12 smithi181 ceph-mon[196979]: pgmap v252: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-10-09T04:05:12.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:12.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:12.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:12.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:12.976 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:12.989 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 30016 30000 144.117 102.375 0.00699521 0.00693186 2023-10-09T04:05:13.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:13.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:13.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:13.283 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:13.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:13.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:13.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:13.516 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:13.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:13.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:13.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:13.734 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:13.779 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:13.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:13.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:13.930 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:13.990 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 31653 31637 141.132 102.312 0.00943327 0.00707993 2023-10-09T04:05:14.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:14.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:14.114 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:14.116 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:14.206 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:14.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:14.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:14.385 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:05:14.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:14.497 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:14.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:14.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:14.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:05:14.651 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:14 smithi163 ceph-mon[213841]: pgmap v253: 89 pgs: 89 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-10-09T04:05:14.651 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4168524997' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:05:14.651 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:05:14.652 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:14 smithi163 ceph-mon[221067]: pgmap v253: 89 pgs: 89 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-10-09T04:05:14.652 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4168524997' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:05:14.652 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:05:14.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:14.763 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:14 smithi181 ceph-mon[196979]: pgmap v253: 89 pgs: 89 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-10-09T04:05:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4168524997' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:05:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:05:14.991 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 33300 33284 138.585 102.938 0.00630579 0.00721058 2023-10-09T04:05:15.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:15.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:05:15.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:05:15.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[221067]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:15 smithi163 ceph-mon[213841]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:15.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:15.749 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:05:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:05:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:15 smithi181 ceph-mon[196979]: failed to encode map e109 with expected crc 2023-10-09T04:05:15.982 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:05:15.982 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:05:15.990 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 34745 34729 135.569 90.3125 0.00884545 0.0073713 2023-10-09T04:05:16.270 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:16.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:16.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:16.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:05:16.446 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:16 smithi163 ceph-mon[213841]: pgmap v255: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T04:05:16.592 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:16 smithi163 ceph-mon[221067]: pgmap v255: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T04:05:16.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:16.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:16.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:16.739 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:16 smithi181 ceph-mon[196979]: pgmap v255: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T04:05:16.747 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:16.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:16.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:16.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:16.966 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:16.990 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 15 36120 36105 132.654 86 0.00704481 0.00753427 2023-10-09T04:05:17.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:17.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:17.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:17.223 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:17.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:17.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:17.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:05:17.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:17.611 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:17.680 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:17.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:17.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:17.788 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:17.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:17.942 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:17.990 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 37578 37562 130.344 91.0625 0.0133328 0.00766489 2023-10-09T04:05:18.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:18.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:18.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:18.224 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:18.272 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:18 smithi163 ceph-mon[221067]: pgmap v256: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:05:18.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:18 smithi163 ceph-mon[213841]: pgmap v256: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:05:18.279 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:18.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:18.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:18.460 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:18.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:18 smithi181 ceph-mon[196979]: pgmap v256: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:05:18.991 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 39152 39136 128.662 98.375 0.0052813 0.00776808 2023-10-09T04:05:19.163 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:05:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/111238658' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/111238658' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/111238658' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:19.991 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:05:19.991409+0000 min lat: 0.00118809 max lat: 0.205811 avg lat: 0.00788286 2023-10-09T04:05:19.991 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:05:19.991 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 40610 40594 126.785 91.125 0.00632777 0.00788286 2023-10-09T04:05:20.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:20.518 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[221067]: pgmap v258: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:20.519 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:20 smithi163 ceph-mon[213841]: pgmap v258: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: failed to encode map e110 with expected crc 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:20.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:20 smithi181 ceph-mon[196979]: pgmap v258: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:05:20.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:05:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:05:20] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.43.0" 2023-10-09T04:05:20.991 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 42186 42170 125.438 98.5 0.0027941 0.00796556 2023-10-09T04:05:21.086 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:21.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:21.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:21.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:21.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:21.325 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:21.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:21.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:21.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:21.803 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:21.948 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:05:21.995 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 43555 43539 123.605 85.5625 0.0111732 0.00808467 2023-10-09T04:05:22.010 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:05:22.048 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:05:22.088 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:05:22.121 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:22.130 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:05:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:22.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:22.171 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:05:22.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:05:22.232 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:22.254 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.269 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:05:22.270 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:22.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:22.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:22.357 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:05:22.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:22.497 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:22 smithi181 ceph-mon[196979]: pgmap v259: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:05:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3639885515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/511334724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:05:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:22 smithi163 ceph-mon[221067]: pgmap v259: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:05:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3639885515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:22.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/511334724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:05:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:22 smithi163 ceph-mon[213841]: pgmap v259: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:05:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3639885515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:22.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/511334724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:05:22.605 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:22.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:22.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:22.754 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:22.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:22.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:22.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:22.930 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:22.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:22.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:22.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:22.995 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 44881 44865 121.835 82.875 0.00235916 0.00817615 2023-10-09T04:05:23.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:23.119 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:23.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:05:23.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:23.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:23.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:23.410 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:23 smithi181 ceph-mon[196979]: failed to encode map e111 with expected crc 2023-10-09T04:05:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/511334724' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:05:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:23 smithi181 ceph-mon[196979]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:05:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:23 smithi181 ceph-mon[196979]: failed to encode map e111 with expected crc 2023-10-09T04:05:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[213841]: failed to encode map e111 with expected crc 2023-10-09T04:05:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/511334724' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:05:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[213841]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:05:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[213841]: failed to encode map e111 with expected crc 2023-10-09T04:05:23.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[221067]: failed to encode map e111 with expected crc 2023-10-09T04:05:23.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/511334724' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:05:23.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[221067]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:05:23.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:23 smithi163 ceph-mon[221067]: failed to encode map e111 with expected crc 2023-10-09T04:05:23.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:23.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:23.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:23.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:23.976 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:23.995 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 46272 46256 120.382 86.9375 0.0104853 0.00830202 2023-10-09T04:05:24.442 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:24 smithi163 ceph-mon[221067]: pgmap v261: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T04:05:24.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:05:24.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:24 smithi163 ceph-mon[213841]: pgmap v261: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T04:05:24.584 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:24.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:24.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:24.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:24 smithi181 ceph-mon[196979]: pgmap v261: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T04:05:24.787 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:24.860 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:24.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:24.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:24.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:24.995 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 47490 47474 118.612 76.125 0.0133381 0.00842573 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:11420 6.1 6:8ac83889:::benchmark_data_smithi163_287034_object11419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.625099+0000", 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.390821320000001, 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13146813600000001, 2023-10-09T04:05:25.019 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 11420 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625099+0000", 2023-10-09T04:05:25.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625099+0000", 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625090+0000", 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999909 2023-10-09T04:05:25.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625121+0000", 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1019999999999998e-05 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625134+0000", 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3254999999999999e-05 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625142+0000", 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625197+0000", 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4926999999999998e-05 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625317+0000", 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119986 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625471+0000", 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015390600000000001 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626524+0000", 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001053472 2023-10-09T04:05:25.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.628523+0000", 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001998133 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756469+0000", 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12794636700000001 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756526+0000", 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7522999999999998e-05 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756567+0000", 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0825000000000003e-05 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11431 6.d e107/102 6:b775af05:::benchmark_data_smithi163_287034_object11430:head v 107'696, mlcod=107'694)", 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.630968+0000", 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.384952548000001, 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112370646, 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630968+0000", 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630968+0000", 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630967+0000", 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.631012+0000", 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4505999999999997e-05 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.631024+0000", 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2177e-05 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.631030+0000", 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1229999999999997e-06 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.631071+0000", 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0438999999999997e-05 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.631088+0000", 2023-10-09T04:05:25.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.751e-05 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.743303+0000", 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11221450199999999 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.743339+0000", 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5987e-05 2023-10-09T04:05:25.031 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:20537 6.1 6:8cc5adb7:::benchmark_data_smithi163_287034_object20536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.560971+0000", 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.454949760000002, 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14633850700000001, 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.032 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 20537 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.560971+0000", 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.560971+0000", 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.560970+0000", 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.560991+0000", 2023-10-09T04:05:25.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1708999999999998e-05 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561001+0000", 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5880000000000004e-06 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561008+0000", 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7519999999999999e-06 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561203+0000", 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019542299999999999 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561332+0000", 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012917000000000001 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561501+0000", 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016888800000000001 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.565771+0000", 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042701190000000002 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.565794+0000", 2023-10-09T04:05:25.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2354e-05 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707214+0000", 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14142068499999999 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707265+0000", 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0571000000000001e-05 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707309+0000", 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4341000000000001e-05 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:20539 6.1 6:891c1f57:::benchmark_data_smithi163_287034_object20538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.561023+0000", 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.45489753, 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14643716200000001, 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 20539 2023-10-09T04:05:25.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561023+0000", 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561023+0000", 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561023+0000", 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561047+0000", 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4015e-05 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561054+0000", 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6279999999999996e-06 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561060+0000", 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9649999999999996e-06 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.562386+0000", 2023-10-09T04:05:25.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001325716 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.562505+0000", 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0001195 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.562658+0000", 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015292800000000001 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.565797+0000", 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031383589999999999 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.565992+0000", 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019535599999999999 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707391+0000", 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.141398575 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707426+0000", 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5858000000000001e-05 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707460+0000", 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3732000000000002e-05 2023-10-09T04:05:25.046 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:20540 6.1 6:85c459cd:::benchmark_data_smithi163_287034_object20539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.561066+0000", 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.454854977, 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14787307899999999, 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.047 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 20540 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561066+0000", 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561066+0000", 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561065+0000", 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561086+0000", 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0798999999999999e-05 2023-10-09T04:05:25.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561093+0000", 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8000000000000001e-06 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.561109+0000", 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6787999999999999e-05 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563391+0000", 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022810899999999999 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563528+0000", 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013750500000000001 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563684+0000", 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000155864 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.565799+0000", 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021154199999999998 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707497+0000", 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14169789899999999 2023-10-09T04:05:25.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708860+0000", 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013629969999999999 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708903+0000", 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2658999999999999e-05 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708939+0000", 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5692000000000003e-05 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:20545 6.1 6:83d9fec8:::benchmark_data_smithi163_287034_object20544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.563182+0000", 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.452738498999999, 2023-10-09T04:05:25.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14587748, 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 20545 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563182+0000", 2023-10-09T04:05:25.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563182+0000", 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563180+0000", 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563212+0000", 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1501999999999998e-05 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563224+0000", 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2685e-05 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.563230+0000", 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8050000000000003e-06 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.564792+0000", 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015617820000000001 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.564931+0000", 2023-10-09T04:05:25.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013941100000000001 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.565063+0000", 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013168300000000001 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566013+0000", 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00095011 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707622+0000", 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14160894299999999 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708997+0000", 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013744899999999999 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.709031+0000", 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4607999999999998e-05 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.709059+0000", 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8354e-05 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.061 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:20558 6.4 6:2a29e352:::benchmark_data_smithi163_287034_object20557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.566434+0000", 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.449486863000001, 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.146183962, 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 20558 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566434+0000", 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566434+0000", 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566432+0000", 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566456+0000", 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4553e-05 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566469+0000", 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2095e-05 2023-10-09T04:05:25.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566476+0000", 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7549999999999994e-06 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566526+0000", 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9744999999999999e-05 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566644+0000", 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118151 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707258+0000", 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.140613338 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.711362+0000", 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041040349999999998 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.711539+0000", 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017732499999999999 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.712530+0000", 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00099056899999999995 2023-10-09T04:05:25.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.712578+0000", 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8717999999999997e-05 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.712618+0000", 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9385999999999999e-05 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.068 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:20537 6.1 e108/102 ondisk, result = 0)", 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.566672+0000", 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.449248864000001, 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14064942999999999, 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566672+0000", 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566672+0000", 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566671+0000", 2023-10-09T04:05:25.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566676+0000", 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3639999999999998e-06 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566686+0000", 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0139999999999999e-05 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566693+0000", 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6560000000000003e-06 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566734+0000", 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0447000000000001e-05 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707199+0000", 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.140465861 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707321+0000", 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121691 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.073 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:20539 6.1 e108/102 ondisk, result = 0)", 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.566710+0000", 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.449210676, 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14075616799999999, 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566710+0000", 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566710+0000", 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566710+0000", 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566712+0000", 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.734e-06 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566718+0000", 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8749999999999997e-06 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566723+0000", 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7890000000000002e-06 2023-10-09T04:05:25.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707378+0000", 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14065523699999999 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707385+0000", 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3339999999999996e-06 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707466+0000", 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0517000000000002e-05 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:20540 6.1 e108/102 ondisk, result = 0)", 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.566786+0000", 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.449134772000001, 2023-10-09T04:05:25.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.142158652, 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566786+0000", 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566786+0000", 2023-10-09T04:05:25.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566785+0000", 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566789+0000", 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8060000000000001e-06 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707186+0000", 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14039702000000001 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707213+0000", 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7061000000000001e-05 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708847+0000", 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0016340860000000001 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708854+0000", 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1799999999999999e-06 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.708944+0000", 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9988000000000006e-05 2023-10-09T04:05:25.082 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:20554 6.b e108/102 6:d54477a5:::benchmark_data_smithi163_287034_object20553:head v 108'1278, mlcod=108'1276)", 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.566822+0000", 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.449098806999999, 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.143100647, 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.083 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566822+0000", 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566822+0000", 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566821+0000", 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566845+0000", 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3302000000000002e-05 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707195+0000", 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14035006799999999 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707206+0000", 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1399e-05 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707252+0000", 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6037e-05 2023-10-09T04:05:25.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707278+0000", 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5981000000000001e-05 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.709888+0000", 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026098240000000002 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.709922+0000", 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.451e-05 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:20551 6.e e108/102 6:767e41f5:::benchmark_data_smithi163_287034_object20550:head v 108'1310, mlcod=108'1306)", 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.566912+0000", 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.449008351, 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.142333652, 2023-10-09T04:05:25.088 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566912+0000", 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566912+0000", 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566912+0000", 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.566940+0000", 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8710000000000001e-05 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707211+0000", 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.140270533 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707219+0000", 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4989999999999994e-06 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707309+0000", 2023-10-09T04:05:25.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9579000000000004e-05 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.707338+0000", 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8938999999999999e-05 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.709203+0000", 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018652880000000001 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.709246+0000", 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2694000000000002e-05 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:20841 6.1 6:86d2adc4:::benchmark_data_smithi163_287034_object20840:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.858571+0000", 2023-10-09T04:05:25.093 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.157349611000001, 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.116981193, 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 20841 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.094 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858571+0000", 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858571+0000", 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858570+0000", 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:25.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858593+0000", 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2898999999999999e-05 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858603+0000", 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0304e-05 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858610+0000", 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1149999999999996e-06 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858669+0000", 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9237999999999998e-05 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.858864+0000", 2023-10-09T04:05:25.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019435700000000001 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859022+0000", 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015854000000000001 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.860065+0000", 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001042421 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861518+0000", 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014534470000000001 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861914+0000", 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00039628500000000002 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861953+0000", 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8080000000000001e-05 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.975552+0000", 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11359957499999999 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.100 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:20844 6.b e108/102 6:d40a6f4c:::benchmark_data_smithi163_287034_object20843:head v 108'1300, mlcod=108'1298)", 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.859204+0000", 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.156716685999999, 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.116863673, 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859204+0000", 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859204+0000", 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859203+0000", 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859227+0000", 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4088999999999999e-05 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859238+0000", 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0618999999999999e-05 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859244+0000", 2023-10-09T04:05:25.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3180000000000003e-06 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859287+0000", 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2583999999999998e-05 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.859306+0000", 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8896999999999999e-05 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.976031+0000", 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11672545099999999 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.976067+0000", 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6353999999999998e-05 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:20848 6.b e108/102 6:dbf16df9:::benchmark_data_smithi163_287034_object20847:head v 108'1301, mlcod=108'1298)", 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.861567+0000", 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.154353767, 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.115336119, 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861567+0000", 2023-10-09T04:05:25.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861567+0000", 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861566+0000", 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861591+0000", 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4712e-05 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861602+0000", 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1391e-05 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861609+0000", 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.126e-06 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861651+0000", 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1721999999999998e-05 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861675+0000", 2023-10-09T04:05:25.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3612e-05 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.976878+0000", 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.115203503 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.976903+0000", 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4797e-05 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.110 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:20841 6.1 e108/102 ondisk, result = 0)", 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.861826+0000", 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.15409447, 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11373404600000001, 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861826+0000", 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861826+0000", 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861825+0000", 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861830+0000", 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8099999999999997e-06 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861840+0000", 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2790000000000004e-06 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861846+0000", 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3770000000000001e-06 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861902+0000", 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6378000000000001e-05 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.861909+0000", 2023-10-09T04:05:25.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.939e-06 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.975560+0000", 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.113650814 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:20852 6.b e108/102 6:d1b2645e:::benchmark_data_smithi163_287034_object20851:head v 108'1302, mlcod=108'1298)", 2023-10-09T04:05:25.115 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:07.862598+0000", 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.153322183, 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11550508900000001, 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.862598+0000", 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.862598+0000", 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.862598+0000", 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.862622+0000", 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4309000000000001e-05 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.975570+0000", 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112948436 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.975577+0000", 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9940000000000003e-06 2023-10-09T04:05:25.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.976098+0000", 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00052039899999999995 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.976115+0000", 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7255999999999999e-05 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.978067+0000", 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019517740000000001 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:07.978103+0000", 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6686000000000001e-05 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:22259 6.4 6:2908492a:::benchmark_data_smithi163_287034_object22258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:08.671421+0000", 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 16.344499233000001, 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13447429399999999, 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 22259 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.121 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671421+0000", 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671421+0000", 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671420+0000", 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671445+0000", 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4859999999999999e-05 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671460+0000", 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5500000000000001e-05 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671468+0000", 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.32e-06 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671520+0000", 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1826000000000002e-05 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671665+0000", 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000144726 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.671786+0000", 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120677 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.672905+0000", 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011194519999999999 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.673369+0000", 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046381100000000001 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.805801+0000", 2023-10-09T04:05:25.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13243227399999999 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.805855+0000", 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3879999999999999e-05 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.805896+0000", 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0509999999999997e-05 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.127 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:22271 6.4 6:22eb0523:::benchmark_data_smithi163_287034_object22270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:08.678585+0000", 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 16.337335122999999, 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12833886999999999, 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.128 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 22271 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678585+0000", 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678585+0000", 2023-10-09T04:05:25.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678584+0000", 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678604+0000", 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9777000000000001e-05 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678617+0000", 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2483000000000001e-05 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678625+0000", 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9650000000000008e-06 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678666+0000", 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1539000000000003e-05 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678771+0000", 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000104922 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.678870+0000", 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9154999999999995e-05 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.679643+0000", 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077264100000000002 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.680040+0000", 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00039662900000000001 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.806843+0000", 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12680335400000001 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.806885+0000", 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2227000000000001e-05 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:08.806924+0000", 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9099e-05 2023-10-09T04:05:25.134 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:46254 6.1 6:8c5bcecf:::benchmark_data_smithi163_287034_object46253:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:23.984764+0000", 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.0311569599999999, 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112762575, 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:25.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 46254 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.984764+0000", 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.984764+0000", 2023-10-09T04:05:25.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.984763+0000", 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.984784+0000", 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1169999999999999e-05 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.984792+0000", 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0539999999999995e-06 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.984798+0000", 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8390000000000003e-06 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:25.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.986491+0000", 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001693389 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.986607+0000", 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011531699999999999 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.986719+0000", 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112274 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.989069+0000", 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023497879999999998 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:23.993361+0000", 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042925300000000001 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:25.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:24.097440+0000", 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10407909999999999 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:24.097484+0000", 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3584000000000001e-05 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:24.097526+0000", 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2089999999999999e-05 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.141 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.142 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.142 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:25.142 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:25.142 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:05:25.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:25.181 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:25.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:25.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:25.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:25.403 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:25.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:25.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:25.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:25.645 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:25.995 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 48881 48865 117.392 86.9375 0.0119501 0.00851396 2023-10-09T04:05:26.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:26.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:26.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:05:26.274 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:26.317 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:26.344 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:26.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:26.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:26.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:26.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:26 smithi163 ceph-mon[213841]: pgmap v262: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:05:26.486 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:26 smithi163 ceph-mon[221067]: pgmap v262: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:05:26.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:26.529 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:26.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:05:26.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:26.592 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:26 smithi181 ceph-mon[196979]: pgmap v262: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:05:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:26.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:26.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:26.721 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:26.997 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 49990 49974 115.612 69.3125 0.0119383 0.00864333 2023-10-09T04:05:27.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:27.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:27.277 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:27.313 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:27.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:27.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:27.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:27.516 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:27.581 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:27.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:27.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:27.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:27.724 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:27.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:27.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:27.893 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:27.947 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:27.996 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 51305 51289 114.419 82.1875 0.00243228 0.00871671 2023-10-09T04:05:28.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:28 smithi181 ceph-mon[196979]: pgmap v263: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T04:05:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:28 smithi163 ceph-mon[221067]: pgmap v263: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T04:05:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:28 smithi163 ceph-mon[213841]: pgmap v263: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T04:05:28.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:28.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:28.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:28.753 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:28.783 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:28.858 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:28.869 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:28.997 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 52465 52449 112.968 72.5 0.0186454 0.0088467 2023-10-09T04:05:29.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:29.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:29.178 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:05:29.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:29.283 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:29.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1021815705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:05:29.327 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1021815705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:05:29.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1021815705' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:05:29.998 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 53790 53774 111.963 82.8125 0.00445922 0.00892443 2023-10-09T04:05:30.177 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:05:30.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:30.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1021815705' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:05:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:05:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: pgmap v265: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3078693427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1021815705' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: pgmap v265: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:05:30.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3078693427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:05:30.440 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:05:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1021815705' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: failed to encode map e112 with expected crc 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: pgmap v265: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3078693427' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:05:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:05:30.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:30.584 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:30.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:30.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:30.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:30.739 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:05:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:05:30] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.43.0" 2023-10-09T04:05:30.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:30.808 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:30.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:30.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:30.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:30.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:30.994 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:30.999 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 54994 54978 110.778 75.25 0.0172818 0.00902225 2023-10-09T04:05:31.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:31.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:31.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:31.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:31 smithi181 ceph-mon[196979]: failed to encode map e113 with expected crc 2023-10-09T04:05:31.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:05:31.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:31 smithi181 ceph-mon[196979]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:05:31.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:31 smithi181 ceph-mon[196979]: failed to encode map e113 with expected crc 2023-10-09T04:05:31.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:31.238 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[221067]: failed to encode map e113 with expected crc 2023-10-09T04:05:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:05:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[221067]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:05:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[221067]: failed to encode map e113 with expected crc 2023-10-09T04:05:31.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[213841]: failed to encode map e113 with expected crc 2023-10-09T04:05:31.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:05:31.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[213841]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:05:31.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:31 smithi163 ceph-mon[213841]: failed to encode map e113 with expected crc 2023-10-09T04:05:31.999 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 56244 56228 109.752 78.125 0.00900947 0.00910675 2023-10-09T04:05:32.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:32 smithi163 ceph-mon[221067]: pgmap v267: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T04:05:32.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:32 smithi163 ceph-mon[213841]: pgmap v267: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T04:05:32.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:32 smithi181 ceph-mon[196979]: pgmap v267: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T04:05:32.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:32.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:32.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:05:32.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:32.785 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:32.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:32.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:32.965 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:32.996 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:32.999 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 57477 57461 108.762 77.0625 0.0149061 0.00918804 2023-10-09T04:05:33.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:33.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:33.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:33.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:33.191 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:33.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:33.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:33.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:33.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:33.412 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:33.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:05:33.449 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:57990 6.2 e113/102 6:442bc1e7:::benchmark_data_smithi163_287034_object57989:head v 113'3612, mlcod=113'3610)", 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:33.450713+0000", 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0031728080000000001, 2023-10-09T04:05:33.461 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003204087, 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.450713+0000", 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:33.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.450713+0000", 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.450712+0000", 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.450842+0000", 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013034 2023-10-09T04:05:33.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.450857+0000", 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4969e-05 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.450865+0000", 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1300000000000001e-06 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.453008+0000", 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002142325 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.453028+0000", 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9845000000000001e-05 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:57982 6.0 e113/102 6:0999f10f:::benchmark_data_smithi163_287034_object57981:head v 113'3611, mlcod=113'3609)", 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:33.451145+0000", 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0027400929999999999, 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028444299999999998, 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451145+0000", 2023-10-09T04:05:33.466 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451145+0000", 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451145+0000", 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451177+0000", 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2591e-05 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451189+0000", 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.169e-05 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:33.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451196+0000", 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.25e-06 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451242+0000", 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5482000000000003e-05 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451261+0000", 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9148999999999999e-05 2023-10-09T04:05:33.469 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:57986 6.3 e113/102 6:cd61f76f:::benchmark_data_smithi163_287034_object57985:head v 113'3649, mlcod=113'3647)", 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:33.451224+0000", 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0026615010000000001, 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002829065, 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:33.470 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451224+0000", 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451224+0000", 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451223+0000", 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451273+0000", 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9496999999999997e-05 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:33.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451281+0000", 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4549999999999995e-06 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451287+0000", 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.507e-06 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451440+0000", 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015348799999999999 2023-10-09T04:05:33.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:33.451461+0000", 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1109999999999999e-05 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 3 2023-10-09T04:05:33.474 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:05:33.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:33.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:33.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:33.490 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:05:33.546 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:05:33.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:33.589 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:05:33.622 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:33.624 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:05:33.660 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:05:33.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:33.696 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:05:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:33.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:33.711 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.711 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:05:33.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:33.840 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:33.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:33.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:33.999 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 58708 58692 107.826 76.9375 0.0115898 0.00926937 2023-10-09T04:05:34.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:34.073 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:34.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:05:34.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:34.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:34.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:34.320 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:34.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:34.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:34.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:34.557 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[213841]: pgmap v268: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-09T04:05:34.557 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4111740308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:34.557 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:34.557 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:34.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[221067]: pgmap v268: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-09T04:05:34.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4111740308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:34.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:34.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:34.560 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:34.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:34 smithi181 ceph-mon[196979]: pgmap v268: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-09T04:05:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4111740308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:05:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:34.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:34.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:34.895 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:35.000 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 59661 59645 106.448 59.5625 0.00411111 0.00935148 2023-10-09T04:05:35.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:05:35.220 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:35.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:35.317 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:35.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:35.446 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:35.460 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:35.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:35.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:35.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:35.728 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:35.808 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:36.000 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 60820 60804 105.503 72.4375 0.00616788 0.00947245 2023-10-09T04:05:36.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:36 smithi181 ceph-mon[196979]: pgmap v269: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:05:36.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:36 smithi163 ceph-mon[221067]: pgmap v269: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:05:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:36 smithi163 ceph-mon[213841]: pgmap v269: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:05:37.000 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 61890 61874 104.46 66.875 0.00856952 0.00956801 2023-10-09T04:05:37.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:37.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:37.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:37.204 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:37.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:37.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:37.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:37.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:05:37.375 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:37.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:37.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:37.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:37.554 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:37.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:37.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:37.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:37.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:37.758 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:37.789 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:37.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:37.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:37.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:38.000 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 62741 62725 103.11 53.1875 0.00937382 0.00969419 2023-10-09T04:05:38.007 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:38.136 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:38 smithi181 ceph-mon[196979]: pgmap v270: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:05:38.217 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:38.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:38.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:38.410 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:38 smithi163 ceph-mon[221067]: pgmap v270: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:05:38.411 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:38 smithi163 ceph-mon[213841]: pgmap v270: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:05:38.418 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:38.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:38.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:38.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:38.703 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:38.737 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:38.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:38.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:38.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:39.003 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 63768 63752 102.111 64.1875 0.0165858 0.00978872 2023-10-09T04:05:39.003 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:39.165 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:05:39.167 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:05:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1697887368' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:05:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:05:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3033589349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:39.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1697887368' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:05:39.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:05:39.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3033589349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:39.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1697887368' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:05:39.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:05:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3033589349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:05:39.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:39.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:39.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:39.890 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:39.931 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:39.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:39.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:40.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:05:40.001454+0000 min lat: 0.00118809 max lat: 0.296599 avg lat: 0.00987422 2023-10-09T04:05:40.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:05:40.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 64835 64819 101.226 66.6875 0.017211 0.00987422 2023-10-09T04:05:40.037 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:40.097 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:40.347 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3033589349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.348 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[221067]: pgmap v272: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:05:40.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3033589349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:40 smithi163 ceph-mon[213841]: pgmap v272: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:05:40.371 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:40.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:40.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:40.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:40.499 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:40.499 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:05:40.499 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3033589349' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:05:40.499 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T04:05:40.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: failed to encode map e114 with expected crc 2023-10-09T04:05:40.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:40 smithi181 ceph-mon[196979]: pgmap v272: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:05:40.501 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:40.543 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:40.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:40.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:40.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:40.713 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:40.962 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:40.966 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:05:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:05:40] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T04:05:40.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:40.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:41.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 65918 65902 100.408 67.6875 0.0161034 0.00995306 2023-10-09T04:05:41.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:41.120 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:41.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:41.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:41.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:41.343 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:41.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:41.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:41.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:41.544 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:41.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:41.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:41.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:41.676 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:05:41.676 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:05:41.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:41.754 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:41.766 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:41.830 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:41.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:41.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:41.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:41.994 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:42.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 67139 67123 99.8345 76.3125 0.00341983 0.0100107 2023-10-09T04:05:42.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:42.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:42.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:42.212 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:42.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:42 smithi181 ceph-mon[196979]: pgmap v273: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:05:42.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:42 smithi163 ceph-mon[221067]: pgmap v273: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:05:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:42 smithi163 ceph-mon[213841]: pgmap v273: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:05:42.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:05:42.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:42.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:43.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 68163 68147 99.0013 64 0.0161509 0.0100951 2023-10-09T04:05:43.032 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:43.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:05:43.170 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:43.680 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:05:43.718 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:05:43.727 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:43.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:43.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:43.762 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:05:43.800 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:05:43.840 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:43.841 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:05:43.879 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:43.880 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:05:43.933 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:05:43.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:43.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:43.975 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:05:43.990 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:05:44.001 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 69185 69169 98.2031 63.875 0.0139452 0.0101781 2023-10-09T04:05:44.057 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:44.099 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:44.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:44.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:44.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:44.369 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:44 smithi181 ceph-mon[196979]: pgmap v274: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:05:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3857770182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:44 smithi163 ceph-mon[221067]: pgmap v274: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:05:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3857770182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:44.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:44 smithi163 ceph-mon[213841]: pgmap v274: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:05:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3857770182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:45.002 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 70271 70255 97.5276 67.875 0.0171143 0.0102486 2023-10-09T04:05:45.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:45.060 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:45.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:45.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:45.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:05:45.216 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:45.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:05:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:45.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:05:45.398 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:45.436 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:45.488 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:45.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:45.546 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:45.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:45.666 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:45.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:45.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:46.003 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 71237 71221 96.717 60.375 0.0151614 0.0103336 2023-10-09T04:05:46.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:46.124 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:46.180 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:46.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:46.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:46.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:46.376 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:46.437 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:05:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:46 smithi181 ceph-mon[196979]: pgmap v275: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:05:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/984194525' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:05:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:46 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:05:46.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:46 smithi163 ceph-mon[221067]: pgmap v275: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:05:46.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/984194525' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:05:46.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:46 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:05:46.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:46 smithi163 ceph-mon[213841]: pgmap v275: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:05:46.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/984194525' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:05:46.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:46 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:05:47.004 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 72239 72223 95.9918 62.625 0.00432861 0.0104126 2023-10-09T04:05:47.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:05:47.121 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:47.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:47.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:47.263 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:47.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[213841]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[213841]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:05:47.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[213841]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T04:05:47.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[213841]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[213841]: pgmap v277: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:05:47.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[221067]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[221067]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:05:47.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[221067]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T04:05:47.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[221067]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:47 smithi163 ceph-mon[221067]: pgmap v277: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:05:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:47 smithi181 ceph-mon[196979]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:47 smithi181 ceph-mon[196979]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:05:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:47 smithi181 ceph-mon[196979]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T04:05:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:47 smithi181 ceph-mon[196979]: failed to encode map e115 with expected crc 2023-10-09T04:05:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:47 smithi181 ceph-mon[196979]: pgmap v277: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:05:48.004 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 73196 73180 95.2382 59.8125 0.00986073 0.0104958 2023-10-09T04:05:48.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:48.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:48.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:05:48.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:48.372 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:48.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:48.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:48.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:48.921 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:48.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:48.961 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11413 6.0 e107/102 6:0ec8682d:::benchmark_data_smithi163_287034_object11412:head v 107'686, mlcod=107'682)", 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.625484+0000", 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.316667414000001, 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13187043400000001, 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.962 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625484+0000", 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625484+0000", 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625483+0000", 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625522+0000", 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8216000000000001e-05 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625531+0000", 2023-10-09T04:05:48.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6029999999999994e-06 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625538+0000", 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7689999999999999e-06 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627005+0000", 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00146689 2023-10-09T04:05:48.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627024+0000", 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9385000000000002e-05 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757307+0000", 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.130282324 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757355+0000", 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8055e-05 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11414 6.0 e107/102 6:04f10604:::benchmark_data_smithi163_287034_object11413:head v 107'687, mlcod=107'682)", 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.625553+0000", 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.316599050999997, 2023-10-09T04:05:48.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.131829534, 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625553+0000", 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625553+0000", 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625552+0000", 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625581+0000", 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8218999999999999e-05 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625591+0000", 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0804e-05 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625597+0000", 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9170000000000003e-06 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755690+0000", 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.130093123 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755716+0000", 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.529e-05 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757359+0000", 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0016433260000000001 2023-10-09T04:05:48.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757382+0000", 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3198999999999999e-05 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11417 6.9 e107/102 6:9a562bdc:::benchmark_data_smithi163_287034_object11416:head v 107'709, mlcod=107'707)", 2023-10-09T04:05:48.972 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.625602+0000", 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.316549422000001, 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.130230126, 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625602+0000", 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625602+0000", 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625602+0000", 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625620+0000", 2023-10-09T04:05:48.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8138000000000002e-05 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625626+0000", 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1879999999999999e-06 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625631+0000", 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.211e-06 2023-10-09T04:05:48.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625802+0000", 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000171528 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625821+0000", 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9232000000000002e-05 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755793+0000", 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12997156400000001 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755832+0000", 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9536999999999998e-05 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:48.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11421 6.0 e107/102 6:07048b72:::benchmark_data_smithi163_287034_object11420:head v 107'688, mlcod=107'682)", 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.625655+0000", 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.316496239999999, 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.131745998, 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625655+0000", 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625655+0000", 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625655+0000", 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625694+0000", 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8979000000000001e-05 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625709+0000", 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4524e-05 2023-10-09T04:05:48.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.625713+0000", 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3420000000000001e-06 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756244+0000", 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13053071299999999 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756257+0000", 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3369e-05 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757384+0000", 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011266679999999999 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757401+0000", 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7666e-05 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:11422 6.f 6:fee46628:::benchmark_data_smithi163_287034_object11421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.626176+0000", 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.315975473999998, 2023-10-09T04:05:48.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13118369099999999, 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 11422 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626176+0000", 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626176+0000", 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626174+0000", 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626209+0000", 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5009000000000003e-05 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626223+0000", 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3511e-05 2023-10-09T04:05:48.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626231+0000", 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7409999999999992e-06 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626280+0000", 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9518999999999998e-05 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626415+0000", 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013537 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626566+0000", 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015042300000000001 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756428+0000", 2023-10-09T04:05:48.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12986191499999999 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756463+0000", 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4900000000000001e-05 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757267+0000", 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080385899999999998 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757319+0000", 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2052999999999998e-05 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757360+0000", 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1303000000000001e-05 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11420 6.1 e107/102 6:8ac83889:::benchmark_data_smithi163_287034_object11419:head v 107'708, mlcod=107'706)", 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.626263+0000", 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.315889179000003, 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13005814199999999, 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626263+0000", 2023-10-09T04:05:48.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626263+0000", 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626262+0000", 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626285+0000", 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.357e-05 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626294+0000", 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2479999999999996e-06 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626300+0000", 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1859999999999997e-06 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626367+0000", 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6863e-05 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.626385+0000", 2023-10-09T04:05:48.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.88e-05 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756285+0000", 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.129899085 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756321+0000", 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6127000000000002e-05 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.995 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:11423 6.f 6:fff7accd:::benchmark_data_smithi163_287034_object11422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.627302+0000", 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.314849674999998, 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13175321200000001, 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:48.996 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 11423 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627302+0000", 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627302+0000", 2023-10-09T04:05:48.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627301+0000", 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627335+0000", 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4031e-05 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627347+0000", 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.221e-05 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627355+0000", 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3309999999999998e-06 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:48.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627398+0000", 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3398e-05 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755722+0000", 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12832399999999999 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755875+0000", 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015316400000000001 2023-10-09T04:05:49.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757251+0000", 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013763359999999999 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757370+0000", 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118953 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.758970+0000", 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015996859999999999 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.759017+0000", 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7321000000000003e-05 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.759055+0000", 2023-10-09T04:05:49.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7840999999999999e-05 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:11422 6.f e107/102 ondisk, result = 0)", 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.627466+0000", 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.314685392000001, 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12896848899999999, 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.003 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627466+0000", 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627466+0000", 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627466+0000", 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.627470+0000", 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2590000000000002e-06 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755646+0000", 2023-10-09T04:05:49.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12817571999999999 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755657+0000", 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1443999999999999e-05 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756415+0000", 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00075822800000000001 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756422+0000", 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7580000000000003e-06 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756435+0000", 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2639999999999999e-05 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:11429 6.0 e107/102 6:0fe902e9:::benchmark_data_smithi163_287034_object11428:head v 107'689, mlcod=107'684)", 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:03.630266+0000", 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.311885879999998, 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.127672797, 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630266+0000", 2023-10-09T04:05:49.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630266+0000", 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630265+0000", 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.630292+0000", 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7574999999999999e-05 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755642+0000", 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12534983 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.755657+0000", 2023-10-09T04:05:49.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5007e-05 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756655+0000", 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00099746699999999993 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.756673+0000", 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8051999999999999e-05 2023-10-09T04:05:49.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757902+0000", 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012293130000000001 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:03.757939+0000", 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6607999999999999e-05 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49830 6.f 6:fdf7a709:::benchmark_data_smithi163_287034_object49829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.696408+0000", 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 22.245743350000001, 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21095125200000001, 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49830 2023-10-09T04:05:49.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696408+0000", 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696408+0000", 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696407+0000", 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696429+0000", 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2234000000000001e-05 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696438+0000", 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4269999999999997e-06 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696444+0000", 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4190000000000001e-06 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696487+0000", 2023-10-09T04:05:49.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2809999999999998e-05 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696633+0000", 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014677200000000001 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696811+0000", 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000178133 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.700571+0000", 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037594880000000001 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.700618+0000", 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7077999999999999e-05 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907264+0000", 2023-10-09T04:05:49.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.206646471 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907312+0000", 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7716999999999997e-05 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907360+0000", 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7426000000000003e-05 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.019 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49834 6.f 6:f21ebc5a:::benchmark_data_smithi163_287034_object49833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.696766+0000", 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 22.245385789, 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.210726, 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:49.020 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49834 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696766+0000", 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696766+0000", 2023-10-09T04:05:49.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696765+0000", 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696787+0000", 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1902000000000001e-05 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696806+0000", 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8995999999999999e-05 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696813+0000", 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4939999999999997e-06 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.698613+0000", 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017995929999999999 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.698771+0000", 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015784499999999999 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.698925+0000", 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000154439 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.700710+0000", 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001785247 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.702366+0000", 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001655414 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907411+0000", 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20504536600000001 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907456+0000", 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4634e-05 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907492+0000", 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6100000000000003e-05 2023-10-09T04:05:49.026 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:51282 6.0 e111/102 6:00843b8c:::benchmark_data_smithi163_287034_object51281:head v 111'3175, mlcod=111'3173)", 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:27.924806+0000", 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.017345589000001, 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11476432, 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.924806+0000", 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.924806+0000", 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.924806+0000", 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.924829+0000", 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3609e-05 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.924837+0000", 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1920000000000005e-06 2023-10-09T04:05:49.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.924843+0000", 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6659999999999996e-06 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.927398+0000", 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025554179999999998 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.927418+0000", 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9806999999999999e-05 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.039542+0000", 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112124243 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.039570+0000", 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7971e-05 2023-10-09T04:05:49.031 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:51286 6.f 6:f806ff70:::benchmark_data_smithi163_287034_object51285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:27.929449+0000", 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.012703004999999, 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.116472836, 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:49.032 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 51286 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929449+0000", 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929449+0000", 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929447+0000", 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929475+0000", 2023-10-09T04:05:49.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7864999999999998e-05 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929489+0000", 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3759e-05 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929497+0000", 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6750000000000008e-06 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929563+0000", 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5698999999999996e-05 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:27.929706+0000", 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014305599999999999 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:49.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.039469+0000", 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10976261 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.041806+0000", 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023369509999999999 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.043908+0000", 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002102432 2023-10-09T04:05:49.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.045841+0000", 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001932979 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.045884+0000", 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3362000000000003e-05 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.045922+0000", 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7057000000000001e-05 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:51579 6.9 e111/102 6:95e8e5d6:::benchmark_data_smithi163_287034_object51578:head v 111'3221, mlcod=111'3219)", 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:28.211992+0000", 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 20.730159293, 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14061759900000001, 2023-10-09T04:05:49.039 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.211992+0000", 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.211992+0000", 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.211991+0000", 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.212015+0000", 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3193e-05 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.212026+0000", 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1406999999999999e-05 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.212034+0000", 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6350000000000006e-06 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.212110+0000", 2023-10-09T04:05:49.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6273000000000006e-05 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.212129+0000", 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9425000000000001e-05 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352578+0000", 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14044852499999999 2023-10-09T04:05:49.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352610+0000", 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2131000000000001e-05 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:51575 6.0 e111/102 6:000ba16a:::benchmark_data_smithi163_287034_object51574:head v 111'3200, mlcod=111'3198)", 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:28.213943+0000", 2023-10-09T04:05:49.044 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 20.728208433999999, 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14347678999999999, 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.213943+0000", 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.213943+0000", 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.213943+0000", 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.213968+0000", 2023-10-09T04:05:49.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5633999999999999e-05 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.213980+0000", 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1936000000000001e-05 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.213988+0000", 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6599999999999995e-06 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214052+0000", 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4478000000000002e-05 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214074+0000", 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.177e-05 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.357377+0000", 2023-10-09T04:05:49.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.143303393 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.357420+0000", 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2669000000000001e-05 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:51587 6.2 e111/102 6:4f72754b:::benchmark_data_smithi163_287034_object51586:head v 111'3194, mlcod=111'3191)", 2023-10-09T04:05:49.049 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:28.214084+0000", 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 20.728067915, 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14484447, 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214084+0000", 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214084+0000", 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214083+0000", 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214118+0000", 2023-10-09T04:05:49.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5413000000000001e-05 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214128+0000", 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5840000000000001e-06 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214135+0000", 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4219999999999997e-06 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214226+0000", 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1126999999999997e-05 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214243+0000", 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7564e-05 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.358888+0000", 2023-10-09T04:05:49.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.144645262 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.358928+0000", 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9784999999999999e-05 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.054 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:51585 6.8 e111/102 6:11deac0d:::benchmark_data_smithi163_287034_object51584:head v 111'3151, mlcod=111'3149)", 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:28.214460+0000", 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 20.727691239999999, 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14298855999999999, 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214460+0000", 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214460+0000", 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214460+0000", 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.214555+0000", 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4901000000000005e-05 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352483+0000", 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.137928369 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352494+0000", 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1394e-05 2023-10-09T04:05:49.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352642+0000", 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000147286 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352662+0000", 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9896000000000001e-05 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.357425+0000", 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047637690000000002 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.357449+0000", 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3621e-05 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:51588 6.2 e111/102 6:45b1a934:::benchmark_data_smithi163_287034_object51587:head v 111'3195, mlcod=111'3191)", 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:28.215892+0000", 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 20.726259515999999, 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14306281000000001, 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.215892+0000", 2023-10-09T04:05:49.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.215892+0000", 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.215891+0000", 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.215995+0000", 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010323300000000001 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352520+0000", 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.136525588 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.352529+0000", 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7190000000000007e-06 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.354120+0000", 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015905159999999999 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.354140+0000", 2023-10-09T04:05:49.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0026000000000001e-05 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.358933+0000", 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047938080000000001 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:28.358955+0000", 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1627000000000001e-05 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.064 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:55621 6.f 6:f4f12206:::benchmark_data_smithi163_287034_object55620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:31.424491+0000", 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 17.517660321000001, 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.118727385, 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:49.065 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 55621 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424491+0000", 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424491+0000", 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424490+0000", 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424514+0000", 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4501000000000001e-05 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424526+0000", 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1871999999999999e-05 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424534+0000", 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8760000000000005e-06 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424578+0000", 2023-10-09T04:05:49.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4078e-05 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424737+0000", 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015917899999999999 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.424906+0000", 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016824400000000001 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.426675+0000", 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001769025 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.428544+0000", 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001869386 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.543126+0000", 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.114581982 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.543174+0000", 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7784999999999997e-05 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:31.543219+0000", 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4910000000000002e-05 2023-10-09T04:05:49.071 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:72562 6.f 6:f5a2f68b:::benchmark_data_smithi163_287034_object72561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.338738+0000", 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.6034132400000001, 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.186594601, 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:05:49.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 72562 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338738+0000", 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:05:49.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338738+0000", 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338737+0000", 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338767+0000", 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0165999999999999e-05 2023-10-09T04:05:49.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338780+0000", 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3084999999999999e-05 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338788+0000", 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0099999999999995e-06 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.338846+0000", 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7748999999999999e-05 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.339000+0000", 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015382699999999999 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:05:49.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.339151+0000", 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000150903 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.341535+0000", 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002384246 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.343382+0000", 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001847429 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.525235+0000", 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.181852661 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.525285+0000", 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0219000000000001e-05 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:05:49.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.525333+0000", 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7917999999999998e-05 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:05:49.079 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:05:49.080 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 74148 74132 94.5083 59.5 0.0250255 0.0105753 2023-10-09T04:05:49.080 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:49.081 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:49.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:49.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:49.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:49.285 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:49.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:05:50.004 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 75056 75040 93.7534 56.75 0.00569569 0.0106621 2023-10-09T04:05:50.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:50.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:50.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:50.301 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:50.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:05:50.332 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:50.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:50.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:50.405 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:05:50.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:50 smithi163 ceph-mon[221067]: pgmap v278: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:05:50.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1581067573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:50.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:50.466 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:50 smithi163 ceph-mon[213841]: pgmap v278: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:05:50.466 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1581067573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:50.466 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:50.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:50.531 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:50.594 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:50.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:50 smithi181 ceph-mon[196979]: pgmap v278: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:05:50.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1581067573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:50.599 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:05:50.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:50.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:50.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:50.742 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:50.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:50.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:05:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:05:50] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2023-10-09T04:05:50.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:50.993 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:51.004 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 76125 76109 93.2253 66.8125 0.00814795 0.0107218 2023-10-09T04:05:51.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:51.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[213841]: failed to encode map e116 with expected crc 2023-10-09T04:05:51.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[213841]: failed to encode map e116 with expected crc 2023-10-09T04:05:51.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:05:51.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[213841]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T04:05:51.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:51.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[221067]: failed to encode map e116 with expected crc 2023-10-09T04:05:51.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[221067]: failed to encode map e116 with expected crc 2023-10-09T04:05:51.700 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:05:51.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:51 smithi163 ceph-mon[221067]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T04:05:51.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:51 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:05:51.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:51 smithi181 ceph-mon[196979]: failed to encode map e116 with expected crc 2023-10-09T04:05:51.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:51 smithi181 ceph-mon[196979]: failed to encode map e116 with expected crc 2023-10-09T04:05:51.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:05:51.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:51 smithi181 ceph-mon[196979]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T04:05:52.004 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 76890 76874 92.3522 47.8125 0.0250525 0.0108192 2023-10-09T04:05:52.542 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:52.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:52.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:52.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:52 smithi181 ceph-mon[196979]: pgmap v280: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:05:52.689 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:52.756 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:52.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:52.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:52 smithi163 ceph-mon[221067]: pgmap v280: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:05:52.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:52 smithi163 ceph-mon[213841]: pgmap v280: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:05:52.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:52.905 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:52.971 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:52.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:53.006 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 77870 77854 91.7632 61.25 0.0148363 0.0108935 2023-10-09T04:05:53.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:53.199 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:53.325 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:53.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:53.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:53.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:53.536 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:05:53.575 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:53.587 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:05:53.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:05:53.658 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:05:53.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:53.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:53.715 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:05:53.755 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:05:53.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:53.792 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:05:53.834 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:53.835 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:05:53.837 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:05:53.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:53.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:54.006 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 78788 78772 91.1266 57.375 0.0105549 0.0109687 2023-10-09T04:05:54.015 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:54.094 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:54.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:54 smithi163 ceph-mon[213841]: pgmap v281: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s 2023-10-09T04:05:54.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4191290761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:54.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3278294356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:05:54.399 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:05:54.423 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:54.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:54.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:05:54.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:05:54.577 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:54.652 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:05:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:54.659 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:54 smithi181 ceph-mon[196979]: pgmap v281: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s 2023-10-09T04:05:54.659 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4191290761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:54.659 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3278294356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:05:54.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:05:54.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:54.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:54 smithi163 ceph-mon[221067]: pgmap v281: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s 2023-10-09T04:05:54.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4191290761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:05:54.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3278294356' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:05:54.838 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:55.006 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 79803 79787 90.6231 63.4375 0.00573611 0.0110236 2023-10-09T04:05:55.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:55 smithi163 ceph-mon[213841]: failed to encode map e117 with expected crc 2023-10-09T04:05:55.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3278294356' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:05:55.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:55 smithi163 ceph-mon[213841]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T04:05:55.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:55 smithi163 ceph-mon[221067]: failed to encode map e117 with expected crc 2023-10-09T04:05:55.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3278294356' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:05:55.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:55 smithi163 ceph-mon[221067]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T04:05:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:55 smithi181 ceph-mon[196979]: failed to encode map e117 with expected crc 2023-10-09T04:05:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3278294356' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:05:55.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:55 smithi181 ceph-mon[196979]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T04:05:56.008 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 80593 80577 89.8844 49.375 0.00749092 0.011121 2023-10-09T04:05:56.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:05:56.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:56.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:56.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:56.216 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:56.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:56.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:56.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:56.462 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:56 smithi181 ceph-mon[196979]: pgmap v283: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T04:05:56.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:56.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:05:56.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:56 smithi163 ceph-mon[221067]: pgmap v283: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T04:05:56.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:56 smithi163 ceph-mon[213841]: pgmap v283: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T04:05:56.660 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:05:56.661 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:05:56.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:05:56.691 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:56.712 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:56.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:05:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:56.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:05:56.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:56.958 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:57.009 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 81522 81506 89.3241 58.0625 0.0100155 0.01119 2023-10-09T04:05:57.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:57.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:57.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:57.712 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:05:57.777 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:05:57.802 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:57.868 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:57.947 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:05:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:57.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:05:58.009 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 82545 82529 88.8865 63.9375 0.0149924 0.0112457 2023-10-09T04:05:58.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:05:58.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:05:58.169 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:58.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:05:58 smithi181 ceph-mon[196979]: pgmap v284: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:05:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:05:58 smithi163 ceph-mon[221067]: pgmap v284: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:05:58.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:05:58 smithi163 ceph-mon[213841]: pgmap v284: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:05:59.013 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 83398 83382 88.2787 53.3125 0.00579053 0.011323 2023-10-09T04:05:59.103 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:05:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:59.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:05:59.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:59.256 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:59.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:59.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:59.426 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:05:59.457 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:59.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:05:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:59.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:05:59.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:05:59.668 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:59.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:05:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:59.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:05:59.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:05:59.898 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:05:59.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:05:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:05:59.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:00.013 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:06:00.013529+0000 min lat: 0.00118809 max lat: 0.296599 avg lat: 0.0114013 2023-10-09T04:06:00.013 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:06:00.013 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 84233 84217 87.6774 52.1875 0.00621661 0.0114013 2023-10-09T04:06:00.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:00.132 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:00.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:00.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:00.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:00.331 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:00.426 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:00.427 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:00.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:00 smithi181 ceph-mon[196979]: pgmap v285: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s 2023-10-09T04:06:00.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:00 smithi163 ceph-mon[221067]: pgmap v285: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s 2023-10-09T04:06:00.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:00 smithi163 ceph-mon[213841]: pgmap v285: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s 2023-10-09T04:06:00.595 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:00.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:00.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:00.787 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:00.866 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:06:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:06:00] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2023-10-09T04:06:00.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:00.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:01.013 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 85179 85163 87.2094 59.125 0.0168736 0.0114619 2023-10-09T04:06:01.090 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:01.129 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:01.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:01.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:01.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:01.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2909311837' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:01.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:01 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:01.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2909311837' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:01.379 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:01 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:01.398 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:01.418 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:06:01.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2909311837' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:01.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:01 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:02.015 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 86083 86067 86.7121 56.5 0.0125346 0.0115279 2023-10-09T04:06:02.399 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:06:02.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:02.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:02.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:02.566 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:02.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:02.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: pgmap v286: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T04:06:02.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:06:02.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:06:02.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:02.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T04:06:02.639 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.639 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.639 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.639 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3707771248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:06:02.639 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:02 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:06:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:02.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:02.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: pgmap v286: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3707771248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:06:02.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:06:02.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: pgmap v286: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T04:06:02.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:06:02.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:06:02.780 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:02.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T04:06:02.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: failed to encode map e118 with expected crc 2023-10-09T04:06:02.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3707771248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:06:02.781 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:02 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:06:02.815 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:03.014 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 87122 87106 86.3665 64.9375 0.00523029 0.011569 2023-10-09T04:06:03.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[213841]: failed to encode map e118 with expected crc 2023-10-09T04:06:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[213841]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:06:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[213841]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T04:06:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[213841]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[213841]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[221067]: failed to encode map e118 with expected crc 2023-10-09T04:06:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[221067]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:06:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[221067]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T04:06:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[221067]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:03 smithi163 ceph-mon[221067]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:03 smithi181 ceph-mon[196979]: failed to encode map e118 with expected crc 2023-10-09T04:06:03.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:03 smithi181 ceph-mon[196979]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:03 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:06:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:03 smithi181 ceph-mon[196979]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T04:06:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:03 smithi181 ceph-mon[196979]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:03 smithi181 ceph-mon[196979]: failed to encode map e119 with expected crc 2023-10-09T04:06:03.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:03.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:04.014 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 87964 87948 85.8395 52.625 0.0170735 0.0116452 2023-10-09T04:06:04.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:04.154 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:04.389 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[213841]: failed to encode map e119 with expected crc 2023-10-09T04:06:04.389 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[213841]: pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-10-09T04:06:04.389 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:04.389 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:04.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[221067]: failed to encode map e119 with expected crc 2023-10-09T04:06:04.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[221067]: pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-10-09T04:06:04.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:04.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:04 smithi181 ceph-mon[196979]: failed to encode map e119 with expected crc 2023-10-09T04:06:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:04 smithi181 ceph-mon[196979]: pgmap v289: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-10-09T04:06:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:05.019 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 88864 88848 85.3794 56.25 0.00991932 0.0117072 2023-10-09T04:06:05.079 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:05.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:05.097 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:06:05.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:06:05.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:05.260 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:05.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:05.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:05.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:05.453 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:05.518 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:05.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:05.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:05.705 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:05.747 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:05.747 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:05.811 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:06:05.858 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:06:05.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:06:05.897 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:06:05.932 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:06:05.982 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:06:06.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 89559 89543 84.743 43.4375 0.0255819 0.011794 2023-10-09T04:06:06.041 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:06:06.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:06.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:06.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:06.075 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:06:06.107 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.113 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:06:06.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:06.190 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:06.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:06.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:06.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:06.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:06.398 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:06.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:06.455 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:06 smithi181 ceph-mon[196979]: pgmap v290: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:06:06.455 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1688557972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:06.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:06.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:06.608 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:06.682 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:06.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:06.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:06.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:06 smithi163 ceph-mon[221067]: pgmap v290: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:06:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1688557972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:06 smithi163 ceph-mon[213841]: pgmap v290: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:06:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1688557972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:06.895 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:07.022 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 90368 90352 84.233 50.5625 0.00335093 0.0118645 2023-10-09T04:06:07.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:06:07.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:07.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:07.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:07.521 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:07.557 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:07.574 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:07.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:07.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:07.688 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:07.800 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:08.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 91040 91024 83.6122 42 0.0193703 0.0119556 2023-10-09T04:06:08.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:08.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:08.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:08.252 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:08.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:08.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:08.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:08.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:08 smithi163 ceph-mon[221067]: pgmap v291: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:06:08.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:08 smithi163 ceph-mon[213841]: pgmap v291: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:06:08.497 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:08 smithi181 ceph-mon[196979]: pgmap v291: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:06:08.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:09.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:09.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 91915 91899 83.1932 54.6875 0.0228207 0.0120147 2023-10-09T04:06:09.102 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:09.178 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:09.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:06:09.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:09.820 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:09.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:09.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:09.972 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:10.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 92737 92721 82.7388 51.375 0.00888257 0.0120807 2023-10-09T04:06:10.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:10.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:10.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:10.187 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:10.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:10.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:10.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:10.403 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:10 smithi163 ceph-mon[221067]: pgmap v292: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 694 op/s 2023-10-09T04:06:10.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:10 smithi163 ceph-mon[213841]: pgmap v292: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 694 op/s 2023-10-09T04:06:10.480 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:10 smithi181 ceph-mon[196979]: pgmap v292: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 694 op/s 2023-10-09T04:06:10.839 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:06:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:06:10] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2023-10-09T04:06:11.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 93588 93572 82.3227 53.1875 0.00710768 0.0121426 2023-10-09T04:06:11.195 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:06:11.222 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:11.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:11.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:11.271 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:06:11.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:11.393 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:11.405 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:11.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:11.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:11.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:11.677 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:11.683 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:11.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:11.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:11.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:12.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 94446 94430 81.9243 53.625 0.00732377 0.0121929 2023-10-09T04:06:12.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:12.048 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:12.181 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:12.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:12.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:12.444 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:06:12.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:12.512 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":292,"stamp":"2023-10-09T04:06:09.300169+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5856040864,"num_objects":89371,"num_object_clones":0,"num_object_copies":268113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89371,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":89647,"num_write_kb":5720653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89563,"ondisk_log_size":89563,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18583080,"kb_used_data":17083816,"kb_used_omap":126,"kb_used_meta":1499073,"kb_avail":731378136,"statfs":{"total":767960285184,"available":748931211264,"internally_reserved":0,"allocated":17493827584,"data_stored":17489828107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1535051387},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":364314624,"num_objects":5559,"num_object_clones":0,"num_object_copies":16677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5559,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002464"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":96,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.405705+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:06:01.405705+0000","last_peered":"2023-10-09T04:06:01.405705+0000","last_clean":"2023-10-09T04:06:01.405705+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:06:01.405705+0000","last_undegraded":"2023-10-09T04:06:01.405705+0000","last_fullsized":"2023-10-09T04:06:01.405705+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.379175+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:06:02.379175+0000","last_peered":"2023-10-09T04:06:02.379175+0000","last_clean":"2023-10-09T04:06:02.379175+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:06:02.379175+0000","last_undegraded":"2023-10-09T04:06:02.379175+0000","last_fullsized":"2023-10-09T04:06:02.379175+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413452+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:06:01.413452+0000","last_peered":"2023-10-09T04:06:01.413452+0000","last_clean":"2023-10-09T04:06:01.413452+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:01.413452+0000","last_undegraded":"2023-10-09T04:06:01.413452+0000","last_fullsized":"2023-10-09T04:06:01.413452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":72,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.696642+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:06:01.696642+0000","last_peered":"2023-10-09T04:06:01.696642+0000","last_clean":"2023-10-09T04:06:01.696642+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:06:01.696642+0000","last_undegraded":"2023-10-09T04:06:01.696642+0000","last_fullsized":"2023-10-09T04:06:01.696642+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":99,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397825+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:06:01.397825+0000","last_peered":"2023-10-09T04:06:01.397825+0000","last_clean":"2023-10-09T04:06:01.397825+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:06:01.397825+0000","last_undegraded":"2023-10-09T04:06:01.397825+0000","last_fullsized":"2023-10-09T04:06:01.397825+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.391256+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:06:01.391256+0000","last_peered":"2023-10-09T04:06:01.391256+0000","last_clean":"2023-10-09T04:06:01.391256+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:06:01.391256+0000","last_undegraded":"2023-10-09T04:06:01.391256+0000","last_fullsized":"2023-10-09T04:06:01.391256+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390954+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:06:01.390954+0000","last_peered":"2023-10-09T04:06:01.390954+0000","last_clean":"2023-10-09T04:06:01.390954+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:06:01.390954+0000","last_undegraded":"2023-10-09T04:06:01.390954+0000","last_fullsized":"2023-10-09T04:06:01.390954+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":95,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397747+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:06:01.397747+0000","last_peered":"2023-10-09T04:06:01.397747+0000","last_clean":"2023-10-09T04:06:01.397747+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:06:01.397747+0000","last_undegraded":"2023-10-09T04:06:01.397747+0000","last_fullsized":"2023-10-09T04:06:01.397747+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397057+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:06:01.397057+0000","last_peered":"2023-10-09T04:06:01.397057+0000","last_clean":"2023-10-09T04:06:01.397057+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:06:01.397057+0000","last_undegraded":"2023-10-09T04:06:01.397057+0000","last_fullsized":"2023-10-09T04:06:01.397057+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":42,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.396999+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:06:01.396999+0000","last_peered":"2023-10-09T04:06:01.396999+0000","last_clean":"2023-10-09T04:06:01.396999+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:06:01.396999+0000","last_undegraded":"2023-10-09T04:06:01.396999+0000","last_fullsized":"2023-10-09T04:06:01.396999+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":81,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395463+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:06:01.395463+0000","last_peered":"2023-10-09T04:06:01.395463+0000","last_clean":"2023-10-09T04:06:01.395463+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:06:01.395463+0000","last_undegraded":"2023-10-09T04:06:01.395463+0000","last_fullsized":"2023-10-09T04:06:01.395463+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413186+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:06:01.413186+0000","last_peered":"2023-10-09T04:06:01.413186+0000","last_clean":"2023-10-09T04:06:01.413186+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:06:01.413186+0000","last_undegraded":"2023-10-09T04:06:01.413186+0000","last_fullsized":"2023-10-09T04:06:01.413186+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":83,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.405322+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:06:01.405322+0000","last_peered":"2023-10-09T04:06:01.405322+0000","last_clean":"2023-10-09T04:06:01.405322+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:06:01.405322+0000","last_undegraded":"2023-10-09T04:06:01.405322+0000","last_fullsized":"2023-10-09T04:06:01.405322+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.404877+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:06:01.404877+0000","last_peered":"2023-10-09T04:06:01.404877+0000","last_clean":"2023-10-09T04:06:01.404877+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:06:01.404877+0000","last_undegraded":"2023-10-09T04:06:01.404877+0000","last_fullsized":"2023-10-09T04:06:01.404877+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":80,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395663+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:06:01.395663+0000","last_peered":"2023-10-09T04:06:01.395663+0000","last_clean":"2023-10-09T04:06:01.395663+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:06:01.395663+0000","last_undegraded":"2023-10-09T04:06:01.395663+0000","last_fullsized":"2023-10-09T04:06:01.395663+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":81,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397491+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:06:01.397491+0000","last_peered":"2023-10-09T04:06:01.397491+0000","last_clean":"2023-10-09T04:06:01.397491+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:06:01.397491+0000","last_undegraded":"2023-10-09T04:06:01.397491+0000","last_fullsized":"2023-10-09T04:06:01.397491+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390446+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:06:01.390446+0000","last_peered":"2023-10-09T04:06:01.390446+0000","last_clean":"2023-10-09T04:06:01.390446+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:06:01.390446+0000","last_undegraded":"2023-10-09T04:06:01.390446+0000","last_fullsized":"2023-10-09T04:06:01.390446+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397011+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:06:01.397011+0000","last_peered":"2023-10-09T04:06:01.397011+0000","last_clean":"2023-10-09T04:06:01.397011+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:06:01.397011+0000","last_undegraded":"2023-10-09T04:06:01.397011+0000","last_fullsized":"2023-10-09T04:06:01.397011+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395303+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:06:01.395303+0000","last_peered":"2023-10-09T04:06:01.395303+0000","last_clean":"2023-10-09T04:06:01.395303+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:06:01.395303+0000","last_undegraded":"2023-10-09T04:06:01.395303+0000","last_fullsized":"2023-10-09T04:06:01.395303+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":71,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390749+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:06:01.390749+0000","last_peered":"2023-10-09T04:06:01.390749+0000","last_clean":"2023-10-09T04:06:01.390749+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:06:01.390749+0000","last_undegraded":"2023-10-09T04:06:01.390749+0000","last_fullsized":"2023-10-09T04:06:01.390749+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":86,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390689+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:06:01.390689+0000","last_peered":"2023-10-09T04:06:01.390689+0000","last_clean":"2023-10-09T04:06:01.390689+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:06:01.390689+0000","last_undegraded":"2023-10-09T04:06:01.390689+0000","last_fullsized":"2023-10-09T04:06:01.390689+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":81,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.400892+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:06:01.400892+0000","last_peered":"2023-10-09T04:06:01.400892+0000","last_clean":"2023-10-09T04:06:01.400892+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:06:01.400892+0000","last_undegraded":"2023-10-09T04:06:01.400892+0000","last_fullsized":"2023-10-09T04:06:01.400892+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"119'5610","reported_seq":5634,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.223792+0000","last_change":"2023-10-09T04:04:51.281106+0000","last_active":"2023-10-09T04:06:05.223792+0000","last_peered":"2023-10-09T04:06:05.223792+0000","last_clean":"2023-10-09T04:06:05.223792+0000","last_became_active":"2023-10-09T04:04:51.280862+0000","last_became_peered":"2023-10-09T04:04:51.280862+0000","last_unstale":"2023-10-09T04:06:05.223792+0000","last_undegraded":"2023-10-09T04:06:05.223792+0000","last_fullsized":"2023-10-09T04:06:05.223792+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:47.717190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367656960,"num_objects":5610,"num_object_clones":0,"num_object_copies":16830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5610,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":79,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395691+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:06:01.395691+0000","last_peered":"2023-10-09T04:06:01.395691+0000","last_clean":"2023-10-09T04:06:01.395691+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:06:01.395691+0000","last_undegraded":"2023-10-09T04:06:01.395691+0000","last_fullsized":"2023-10-09T04:06:01.395691+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397224+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:06:01.397224+0000","last_peered":"2023-10-09T04:06:01.397224+0000","last_clean":"2023-10-09T04:06:01.397224+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:06:01.397224+0000","last_undegraded":"2023-10-09T04:06:01.397224+0000","last_fullsized":"2023-10-09T04:06:01.397224+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"119'5567","reported_seq":5593,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.262703+0000","last_change":"2023-10-09T04:04:51.277948+0000","last_active":"2023-10-09T04:06:05.262703+0000","last_peered":"2023-10-09T04:06:05.262703+0000","last_clean":"2023-10-09T04:06:05.262703+0000","last_became_active":"2023-10-09T04:04:51.277674+0000","last_became_peered":"2023-10-09T04:04:51.277674+0000","last_unstale":"2023-10-09T04:06:05.262703+0000","last_undegraded":"2023-10-09T04:06:05.262703+0000","last_fullsized":"2023-10-09T04:06:05.262703+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:12.182469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364838912,"num_objects":5567,"num_object_clones":0,"num_object_copies":16701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5567,"num_write_kb":356288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":80,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395506+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:06:01.395506+0000","last_peered":"2023-10-09T04:06:01.395506+0000","last_clean":"2023-10-09T04:06:01.395506+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:06:01.395506+0000","last_undegraded":"2023-10-09T04:06:01.395506+0000","last_fullsized":"2023-10-09T04:06:01.395506+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":94,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.404561+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:06:01.404561+0000","last_peered":"2023-10-09T04:06:01.404561+0000","last_clean":"2023-10-09T04:06:01.404561+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:06:01.404561+0000","last_undegraded":"2023-10-09T04:06:01.404561+0000","last_fullsized":"2023-10-09T04:06:01.404561+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"119'5487","reported_seq":5512,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.218601+0000","last_change":"2023-10-09T04:04:51.279912+0000","last_active":"2023-10-09T04:06:05.218601+0000","last_peered":"2023-10-09T04:06:05.218601+0000","last_clean":"2023-10-09T04:06:05.218601+0000","last_became_active":"2023-10-09T04:04:51.279633+0000","last_became_peered":"2023-10-09T04:04:51.279633+0000","last_unstale":"2023-10-09T04:06:05.218601+0000","last_undegraded":"2023-10-09T04:06:05.218601+0000","last_fullsized":"2023-10-09T04:06:05.218601+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:48.533141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359596032,"num_objects":5487,"num_object_clones":0,"num_object_copies":16461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":351168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":62,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395555+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:06:01.395555+0000","last_peered":"2023-10-09T04:06:01.395555+0000","last_clean":"2023-10-09T04:06:01.395555+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:06:01.395555+0000","last_undegraded":"2023-10-09T04:06:01.395555+0000","last_fullsized":"2023-10-09T04:06:01.395555+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":77,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.391003+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:06:01.391003+0000","last_peered":"2023-10-09T04:06:01.391003+0000","last_clean":"2023-10-09T04:06:01.391003+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:06:01.391003+0000","last_undegraded":"2023-10-09T04:06:01.391003+0000","last_fullsized":"2023-10-09T04:06:01.391003+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"119'5729","reported_seq":5755,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:06.628256+0000","last_change":"2023-10-09T04:04:51.276847+0000","last_active":"2023-10-09T04:06:06.628256+0000","last_peered":"2023-10-09T04:06:06.628256+0000","last_clean":"2023-10-09T04:06:06.628256+0000","last_became_active":"2023-10-09T04:04:51.276578+0000","last_became_peered":"2023-10-09T04:04:51.276578+0000","last_unstale":"2023-10-09T04:06:06.628256+0000","last_undegraded":"2023-10-09T04:06:06.628256+0000","last_fullsized":"2023-10-09T04:06:06.628256+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:27.626784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375455744,"num_objects":5729,"num_object_clones":0,"num_object_copies":17187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5729,"num_write_kb":366656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397157+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:06:01.397157+0000","last_peered":"2023-10-09T04:06:01.397157+0000","last_clean":"2023-10-09T04:06:01.397157+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:06:01.397157+0000","last_undegraded":"2023-10-09T04:06:01.397157+0000","last_fullsized":"2023-10-09T04:06:01.397157+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":71,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390413+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:06:01.390413+0000","last_peered":"2023-10-09T04:06:01.390413+0000","last_clean":"2023-10-09T04:06:01.390413+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:06:01.390413+0000","last_undegraded":"2023-10-09T04:06:01.390413+0000","last_fullsized":"2023-10-09T04:06:01.390413+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"119'5513","reported_seq":5538,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.147192+0000","last_change":"2023-10-09T04:04:51.279219+0000","last_active":"2023-10-09T04:06:05.147192+0000","last_peered":"2023-10-09T04:06:05.147192+0000","last_clean":"2023-10-09T04:06:05.147192+0000","last_became_active":"2023-10-09T04:04:51.278902+0000","last_became_peered":"2023-10-09T04:04:51.278902+0000","last_unstale":"2023-10-09T04:06:05.147192+0000","last_undegraded":"2023-10-09T04:06:05.147192+0000","last_fullsized":"2023-10-09T04:06:05.147192+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:37.687116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":79,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397746+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:06:01.397746+0000","last_peered":"2023-10-09T04:06:01.397746+0000","last_clean":"2023-10-09T04:06:01.397746+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:06:01.397746+0000","last_undegraded":"2023-10-09T04:06:01.397746+0000","last_fullsized":"2023-10-09T04:06:01.397746+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":77,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.405230+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:06:01.405230+0000","last_peered":"2023-10-09T04:06:01.405230+0000","last_clean":"2023-10-09T04:06:01.405230+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:06:01.405230+0000","last_undegraded":"2023-10-09T04:06:01.405230+0000","last_fullsized":"2023-10-09T04:06:01.405230+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"119'5433","reported_seq":5459,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:04.348431+0000","last_change":"2023-10-09T04:04:51.275697+0000","last_active":"2023-10-09T04:06:04.348431+0000","last_peered":"2023-10-09T04:06:04.348431+0000","last_clean":"2023-10-09T04:06:04.348431+0000","last_became_active":"2023-10-09T04:04:51.275402+0000","last_became_peered":"2023-10-09T04:04:51.275402+0000","last_unstale":"2023-10-09T04:06:04.348431+0000","last_undegraded":"2023-10-09T04:06:04.348431+0000","last_fullsized":"2023-10-09T04:06:04.348431+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5433,"log_dups_size":0,"ondisk_log_size":5433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:26.490300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356057088,"num_objects":5433,"num_object_clones":0,"num_object_copies":16299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5433,"num_write_kb":347712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.391112+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:06:01.391112+0000","last_peered":"2023-10-09T04:06:01.391112+0000","last_clean":"2023-10-09T04:06:01.391112+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:06:01.391112+0000","last_undegraded":"2023-10-09T04:06:01.391112+0000","last_fullsized":"2023-10-09T04:06:01.391112+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":94,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.405354+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:06:01.405354+0000","last_peered":"2023-10-09T04:06:01.405354+0000","last_clean":"2023-10-09T04:06:01.405354+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:06:01.405354+0000","last_undegraded":"2023-10-09T04:06:01.405354+0000","last_fullsized":"2023-10-09T04:06:01.405354+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"119'5596","reported_seq":5622,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.250694+0000","last_change":"2023-10-09T04:04:51.279552+0000","last_active":"2023-10-09T04:06:05.250694+0000","last_peered":"2023-10-09T04:06:05.250694+0000","last_clean":"2023-10-09T04:06:05.250694+0000","last_became_active":"2023-10-09T04:04:51.279242+0000","last_became_peered":"2023-10-09T04:04:51.279242+0000","last_unstale":"2023-10-09T04:06:05.250694+0000","last_undegraded":"2023-10-09T04:06:05.250694+0000","last_fullsized":"2023-10-09T04:06:05.250694+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5596,"log_dups_size":0,"ondisk_log_size":5596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:44.784925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366739456,"num_objects":5596,"num_object_clones":0,"num_object_copies":16788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5596,"num_write_kb":358144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.404364+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:06:01.404364+0000","last_peered":"2023-10-09T04:06:01.404364+0000","last_clean":"2023-10-09T04:06:01.404364+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:06:01.404364+0000","last_undegraded":"2023-10-09T04:06:01.404364+0000","last_fullsized":"2023-10-09T04:06:01.404364+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":70,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.408458+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:06:01.408458+0000","last_peered":"2023-10-09T04:06:01.408458+0000","last_clean":"2023-10-09T04:06:01.408458+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:06:01.408458+0000","last_undegraded":"2023-10-09T04:06:01.408458+0000","last_fullsized":"2023-10-09T04:06:01.408458+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"119'5563","reported_seq":5587,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.211078+0000","last_change":"2023-10-09T04:04:51.279867+0000","last_active":"2023-10-09T04:06:05.211078+0000","last_peered":"2023-10-09T04:06:05.211078+0000","last_clean":"2023-10-09T04:06:05.211078+0000","last_became_active":"2023-10-09T04:04:51.279588+0000","last_became_peered":"2023-10-09T04:04:51.279588+0000","last_unstale":"2023-10-09T04:06:05.211078+0000","last_undegraded":"2023-10-09T04:06:05.211078+0000","last_fullsized":"2023-10-09T04:06:05.211078+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.265205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364576768,"num_objects":5563,"num_object_clones":0,"num_object_copies":16689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":356032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":82,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397959+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:06:01.397959+0000","last_peered":"2023-10-09T04:06:01.397959+0000","last_clean":"2023-10-09T04:06:01.397959+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:06:01.397959+0000","last_undegraded":"2023-10-09T04:06:01.397959+0000","last_fullsized":"2023-10-09T04:06:01.397959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":68,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413596+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:06:01.413596+0000","last_peered":"2023-10-09T04:06:01.413596+0000","last_clean":"2023-10-09T04:06:01.413596+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:06:01.413596+0000","last_undegraded":"2023-10-09T04:06:01.413596+0000","last_fullsized":"2023-10-09T04:06:01.413596+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":91,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.414468+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:06:01.414468+0000","last_peered":"2023-10-09T04:06:01.414468+0000","last_clean":"2023-10-09T04:06:01.414468+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:06:01.414468+0000","last_undegraded":"2023-10-09T04:06:01.414468+0000","last_fullsized":"2023-10-09T04:06:01.414468+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":95,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397051+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:06:01.397051+0000","last_peered":"2023-10-09T04:06:01.397051+0000","last_clean":"2023-10-09T04:06:01.397051+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:01.397051+0000","last_undegraded":"2023-10-09T04:06:01.397051+0000","last_fullsized":"2023-10-09T04:06:01.397051+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413256+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:06:01.413256+0000","last_peered":"2023-10-09T04:06:01.413256+0000","last_clean":"2023-10-09T04:06:01.413256+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:06:01.413256+0000","last_undegraded":"2023-10-09T04:06:01.413256+0000","last_fullsized":"2023-10-09T04:06:01.413256+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"119'5559","reported_seq":5585,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:06.690371+0000","last_change":"2023-10-09T04:04:51.276404+0000","last_active":"2023-10-09T04:06:06.690371+0000","last_peered":"2023-10-09T04:06:06.690371+0000","last_clean":"2023-10-09T04:06:06.690371+0000","last_became_active":"2023-10-09T04:04:51.276109+0000","last_became_peered":"2023-10-09T04:04:51.276109+0000","last_unstale":"2023-10-09T04:06:06.690371+0000","last_undegraded":"2023-10-09T04:06:06.690371+0000","last_fullsized":"2023-10-09T04:06:06.690371+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:04.681582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364314624,"num_objects":5559,"num_object_clones":0,"num_object_copies":16677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5559,"num_write_kb":355776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"119'5667","reported_seq":5693,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.220453+0000","last_change":"2023-10-09T04:04:51.277034+0000","last_active":"2023-10-09T04:06:05.220453+0000","last_peered":"2023-10-09T04:06:05.220453+0000","last_clean":"2023-10-09T04:06:05.220453+0000","last_became_active":"2023-10-09T04:04:51.276751+0000","last_became_peered":"2023-10-09T04:04:51.276751+0000","last_unstale":"2023-10-09T04:06:05.220453+0000","last_undegraded":"2023-10-09T04:06:05.220453+0000","last_fullsized":"2023-10-09T04:06:05.220453+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5667,"log_dups_size":0,"ondisk_log_size":5667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:13.233763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371392512,"num_objects":5667,"num_object_clones":0,"num_object_copies":17001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5667,"num_write_kb":362688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":82,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397826+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:06:01.397826+0000","last_peered":"2023-10-09T04:06:01.397826+0000","last_clean":"2023-10-09T04:06:01.397826+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:06:01.397826+0000","last_undegraded":"2023-10-09T04:06:01.397826+0000","last_fullsized":"2023-10-09T04:06:01.397826+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.404945+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:06:01.404945+0000","last_peered":"2023-10-09T04:06:01.404945+0000","last_clean":"2023-10-09T04:06:01.404945+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:06:01.404945+0000","last_undegraded":"2023-10-09T04:06:01.404945+0000","last_fullsized":"2023-10-09T04:06:01.404945+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.414396+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:06:01.414396+0000","last_peered":"2023-10-09T04:06:01.414396+0000","last_clean":"2023-10-09T04:06:01.414396+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:06:01.414396+0000","last_undegraded":"2023-10-09T04:06:01.414396+0000","last_fullsized":"2023-10-09T04:06:01.414396+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"119'5523","reported_seq":5548,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.183833+0000","last_change":"2023-10-09T04:04:51.279627+0000","last_active":"2023-10-09T04:06:05.183833+0000","last_peered":"2023-10-09T04:06:05.183833+0000","last_clean":"2023-10-09T04:06:05.183833+0000","last_became_active":"2023-10-09T04:04:51.279326+0000","last_became_peered":"2023-10-09T04:04:51.279326+0000","last_unstale":"2023-10-09T04:06:05.183833+0000","last_undegraded":"2023-10-09T04:06:05.183833+0000","last_fullsized":"2023-10-09T04:06:05.183833+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:58.605251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361955328,"num_objects":5523,"num_object_clones":0,"num_object_copies":16569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5523,"num_write_kb":353472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":101,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.392233+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:06:01.392233+0000","last_peered":"2023-10-09T04:06:01.392233+0000","last_clean":"2023-10-09T04:06:01.392233+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:06:01.392233+0000","last_undegraded":"2023-10-09T04:06:01.392233+0000","last_fullsized":"2023-10-09T04:06:01.392233+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413228+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:06:01.413228+0000","last_peered":"2023-10-09T04:06:01.413228+0000","last_clean":"2023-10-09T04:06:01.413228+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:06:01.413228+0000","last_undegraded":"2023-10-09T04:06:01.413228+0000","last_fullsized":"2023-10-09T04:06:01.413228+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413665+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:06:01.413665+0000","last_peered":"2023-10-09T04:06:01.413665+0000","last_clean":"2023-10-09T04:06:01.413665+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:06:01.413665+0000","last_undegraded":"2023-10-09T04:06:01.413665+0000","last_fullsized":"2023-10-09T04:06:01.413665+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"119'5475","reported_seq":5500,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.128134+0000","last_change":"2023-10-09T04:04:51.280419+0000","last_active":"2023-10-09T04:06:05.128134+0000","last_peered":"2023-10-09T04:06:05.128134+0000","last_clean":"2023-10-09T04:06:05.128134+0000","last_became_active":"2023-10-09T04:04:51.280174+0000","last_became_peered":"2023-10-09T04:04:51.280174+0000","last_unstale":"2023-10-09T04:06:05.128134+0000","last_undegraded":"2023-10-09T04:06:05.128134+0000","last_fullsized":"2023-10-09T04:06:05.128134+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:03.946653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358809600,"num_objects":5475,"num_object_clones":0,"num_object_copies":16425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":350400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":82,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397301+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:06:01.397301+0000","last_peered":"2023-10-09T04:06:01.397301+0000","last_clean":"2023-10-09T04:06:01.397301+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:06:01.397301+0000","last_undegraded":"2023-10-09T04:06:01.397301+0000","last_fullsized":"2023-10-09T04:06:01.397301+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":459,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395368+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:06:01.395368+0000","last_peered":"2023-10-09T04:06:01.395368+0000","last_clean":"2023-10-09T04:06:01.395368+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:06:01.395368+0000","last_undegraded":"2023-10-09T04:06:01.395368+0000","last_fullsized":"2023-10-09T04:06:01.395368+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":42,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397209+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:06:01.397209+0000","last_peered":"2023-10-09T04:06:01.397209+0000","last_clean":"2023-10-09T04:06:01.397209+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:06:01.397209+0000","last_undegraded":"2023-10-09T04:06:01.397209+0000","last_fullsized":"2023-10-09T04:06:01.397209+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.391200+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:06:01.391200+0000","last_peered":"2023-10-09T04:06:01.391200+0000","last_clean":"2023-10-09T04:06:01.391200+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:06:01.391200+0000","last_undegraded":"2023-10-09T04:06:01.391200+0000","last_fullsized":"2023-10-09T04:06:01.391200+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":99,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395444+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:06:01.395444+0000","last_peered":"2023-10-09T04:06:01.395444+0000","last_clean":"2023-10-09T04:06:01.395444+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:06:01.395444+0000","last_undegraded":"2023-10-09T04:06:01.395444+0000","last_fullsized":"2023-10-09T04:06:01.395444+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400081+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:06:02.400081+0000","last_peered":"2023-10-09T04:06:02.400081+0000","last_clean":"2023-10-09T04:06:02.400081+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:06:02.400081+0000","last_undegraded":"2023-10-09T04:06:02.400081+0000","last_fullsized":"2023-10-09T04:06:02.400081+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":86,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390986+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:06:01.390986+0000","last_peered":"2023-10-09T04:06:01.390986+0000","last_clean":"2023-10-09T04:06:01.390986+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:06:01.390986+0000","last_undegraded":"2023-10-09T04:06:01.390986+0000","last_fullsized":"2023-10-09T04:06:01.390986+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"119'5769","reported_seq":5795,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:08.097520+0000","last_change":"2023-10-09T04:04:51.279899+0000","last_active":"2023-10-09T04:06:08.097520+0000","last_peered":"2023-10-09T04:06:08.097520+0000","last_clean":"2023-10-09T04:06:08.097520+0000","last_became_active":"2023-10-09T04:04:51.279611+0000","last_became_peered":"2023-10-09T04:04:51.279611+0000","last_unstale":"2023-10-09T04:06:08.097520+0000","last_undegraded":"2023-10-09T04:06:08.097520+0000","last_fullsized":"2023-10-09T04:06:08.097520+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5769,"log_dups_size":0,"ondisk_log_size":5769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:31.582709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378077184,"num_objects":5769,"num_object_clones":0,"num_object_copies":17307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":369216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":82,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.395518+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:06:01.395518+0000","last_peered":"2023-10-09T04:06:01.395518+0000","last_clean":"2023-10-09T04:06:01.395518+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:06:01.395518+0000","last_undegraded":"2023-10-09T04:06:01.395518+0000","last_fullsized":"2023-10-09T04:06:01.395518+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":67,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.414545+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:06:01.414545+0000","last_peered":"2023-10-09T04:06:01.414545+0000","last_clean":"2023-10-09T04:06:01.414545+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:06:01.414545+0000","last_undegraded":"2023-10-09T04:06:01.414545+0000","last_fullsized":"2023-10-09T04:06:01.414545+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390196+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:06:01.390196+0000","last_peered":"2023-10-09T04:06:01.390196+0000","last_clean":"2023-10-09T04:06:01.390196+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:06:01.390196+0000","last_undegraded":"2023-10-09T04:06:01.390196+0000","last_fullsized":"2023-10-09T04:06:01.390196+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"119'5687","reported_seq":5713,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:08.111958+0000","last_change":"2023-10-09T04:04:51.278104+0000","last_active":"2023-10-09T04:06:08.111958+0000","last_peered":"2023-10-09T04:06:08.111958+0000","last_clean":"2023-10-09T04:06:08.111958+0000","last_became_active":"2023-10-09T04:04:51.277849+0000","last_became_peered":"2023-10-09T04:04:51.277849+0000","last_unstale":"2023-10-09T04:06:08.111958+0000","last_undegraded":"2023-10-09T04:06:08.111958+0000","last_fullsized":"2023-10-09T04:06:08.111958+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.049817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372703232,"num_objects":5687,"num_object_clones":0,"num_object_copies":17061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5687,"num_write_kb":363968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":79,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.400983+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:06:01.400983+0000","last_peered":"2023-10-09T04:06:01.400983+0000","last_clean":"2023-10-09T04:06:01.400983+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:06:01.400983+0000","last_undegraded":"2023-10-09T04:06:01.400983+0000","last_fullsized":"2023-10-09T04:06:01.400983+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":52,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.413968+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:06:01.413968+0000","last_peered":"2023-10-09T04:06:01.413968+0000","last_clean":"2023-10-09T04:06:01.413968+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:06:01.413968+0000","last_undegraded":"2023-10-09T04:06:01.413968+0000","last_fullsized":"2023-10-09T04:06:01.413968+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":76,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397889+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:06:01.397889+0000","last_peered":"2023-10-09T04:06:01.397889+0000","last_clean":"2023-10-09T04:06:01.397889+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:06:01.397889+0000","last_undegraded":"2023-10-09T04:06:01.397889+0000","last_fullsized":"2023-10-09T04:06:01.397889+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"119'5578","reported_seq":5603,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.193570+0000","last_change":"2023-10-09T04:04:51.275204+0000","last_active":"2023-10-09T04:06:05.193570+0000","last_peered":"2023-10-09T04:06:05.193570+0000","last_clean":"2023-10-09T04:06:05.193570+0000","last_became_active":"2023-10-09T04:04:51.274910+0000","last_became_peered":"2023-10-09T04:04:51.274910+0000","last_unstale":"2023-10-09T04:06:05.193570+0000","last_undegraded":"2023-10-09T04:06:05.193570+0000","last_fullsized":"2023-10-09T04:06:05.193570+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:08.996248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365559808,"num_objects":5578,"num_object_clones":0,"num_object_copies":16734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5578,"num_write_kb":356992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":82,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.391269+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:06:01.391269+0000","last_peered":"2023-10-09T04:06:01.391269+0000","last_clean":"2023-10-09T04:06:01.391269+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:06:01.391269+0000","last_undegraded":"2023-10-09T04:06:01.391269+0000","last_fullsized":"2023-10-09T04:06:01.391269+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":75,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390903+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:06:01.390903+0000","last_peered":"2023-10-09T04:06:01.390903+0000","last_clean":"2023-10-09T04:06:01.390903+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:06:01.390903+0000","last_undegraded":"2023-10-09T04:06:01.390903+0000","last_fullsized":"2023-10-09T04:06:01.390903+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.404434+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:06:01.404434+0000","last_peered":"2023-10-09T04:06:01.404434+0000","last_clean":"2023-10-09T04:06:01.404434+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:06:01.404434+0000","last_undegraded":"2023-10-09T04:06:01.404434+0000","last_fullsized":"2023-10-09T04:06:01.404434+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"119'5591","reported_seq":5616,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.191770+0000","last_change":"2023-10-09T04:04:51.277953+0000","last_active":"2023-10-09T04:06:05.191770+0000","last_peered":"2023-10-09T04:06:05.191770+0000","last_clean":"2023-10-09T04:06:05.191770+0000","last_became_active":"2023-10-09T04:04:51.277659+0000","last_became_peered":"2023-10-09T04:04:51.277659+0000","last_unstale":"2023-10-09T04:06:05.191770+0000","last_undegraded":"2023-10-09T04:06:05.191770+0000","last_fullsized":"2023-10-09T04:06:05.191770+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:22.890014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366411776,"num_objects":5591,"num_object_clones":0,"num_object_copies":16773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5591,"num_write_kb":357824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":60,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.390438+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:06:01.390438+0000","last_peered":"2023-10-09T04:06:01.390438+0000","last_clean":"2023-10-09T04:06:01.390438+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:06:01.390438+0000","last_undegraded":"2023-10-09T04:06:01.390438+0000","last_fullsized":"2023-10-09T04:06:01.390438+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.391827+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:06:01.391827+0000","last_peered":"2023-10-09T04:06:01.391827+0000","last_clean":"2023-10-09T04:06:01.391827+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:06:01.391827+0000","last_undegraded":"2023-10-09T04:06:01.391827+0000","last_fullsized":"2023-10-09T04:06:01.391827+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":80,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.398507+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:06:01.398507+0000","last_peered":"2023-10-09T04:06:01.398507+0000","last_clean":"2023-10-09T04:06:01.398507+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:06:01.398507+0000","last_undegraded":"2023-10-09T04:06:01.398507+0000","last_fullsized":"2023-10-09T04:06:01.398507+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.405012+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:06:01.405012+0000","last_peered":"2023-10-09T04:06:01.405012+0000","last_clean":"2023-10-09T04:06:01.405012+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:06:01.405012+0000","last_undegraded":"2023-10-09T04:06:01.405012+0000","last_fullsized":"2023-10-09T04:06:01.405012+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":63,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397091+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:06:01.397091+0000","last_peered":"2023-10-09T04:06:01.397091+0000","last_clean":"2023-10-09T04:06:01.397091+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:06:01.397091+0000","last_undegraded":"2023-10-09T04:06:01.397091+0000","last_fullsized":"2023-10-09T04:06:01.397091+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":94,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.405821+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:06:01.405821+0000","last_peered":"2023-10-09T04:06:01.405821+0000","last_clean":"2023-10-09T04:06:01.405821+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:06:01.405821+0000","last_undegraded":"2023-10-09T04:06:01.405821+0000","last_fullsized":"2023-10-09T04:06:01.405821+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":87,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397406+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:06:01.397406+0000","last_peered":"2023-10-09T04:06:01.397406+0000","last_clean":"2023-10-09T04:06:01.397406+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:06:01.397406+0000","last_undegraded":"2023-10-09T04:06:01.397406+0000","last_fullsized":"2023-10-09T04:06:01.397406+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":102,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.397068+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:06:01.397068+0000","last_peered":"2023-10-09T04:06:01.397068+0000","last_clean":"2023-10-09T04:06:01.397068+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:06:01.397068+0000","last_undegraded":"2023-10-09T04:06:01.397068+0000","last_fullsized":"2023-10-09T04:06:01.397068+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":76,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400767+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:06:02.400767+0000","last_peered":"2023-10-09T04:06:02.400767+0000","last_clean":"2023-10-09T04:06:02.400767+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:06:02.400767+0000","last_undegraded":"2023-10-09T04:06:02.400767+0000","last_fullsized":"2023-10-09T04:06:02.400767+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-10-09T04:06:01.404300+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:06:01.404300+0000","last_peered":"2023-10-09T04:06:01.404300+0000","last_clean":"2023-10-09T04:06:01.404300+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:06:01.404300+0000","last_undegraded":"2023-10-09T04:06:01.404300+0000","last_fullsized":"2023-10-09T04:06:01.404300+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5855444992,"num_objects":89347,"num_object_clones":0,"num_object_copies":268041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89347,"num_write_kb":5718208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17543331840,"data_stored":17543331840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89347,"ondisk_log_size":89347,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201443,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2448208,"kb_used_data":2134536,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":91296944,"statfs":{"total":95995035648,"available":93488070656,"internally_reserved":0,"allocated":2185764864,"data_stored":2185276747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.50700000000000001,"15min":0.51300000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.63700000000000001,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":1.913}]},{"osd":1,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61599999999999999,"15min":0.624},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.72599999999999998,"15min":0.70999999999999996},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.964,"5min":3.363,"15min":3.363},"last":0.95499999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":2.7410000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60399999999999998,"15min":0.60499999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":2.7050000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":1.008,"15min":0.72199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":31.170999999999999,"5min":31.170999999999999,"15min":31.170999999999999},"last":2.6269999999999998},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.071,"15min":0.76700000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":31.184000000000001,"5min":31.184000000000001,"15min":31.184000000000001},"last":2.637}]},{"osd":4,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.498,"15min":0.51600000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28599999999999998},"max":{"1min":0.73299999999999998,"5min":1.0169999999999999,"15min":6.3129999999999997},"last":2.2650000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.53700000000000003,"15min":0.52900000000000003},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.27500000000000002},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":3.8769999999999998},"last":0.48599999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.38700000000000001,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":0.76400000000000001,"5min":1.4330000000000001,"15min":1.464},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":0.76900000000000002,"5min":0.95599999999999996,"15min":3.1840000000000002},"last":2.351}]},{"osd":6,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.54900000000000004,"15min":0.54000000000000004},"min":{"1min":0.42599999999999999,"5min":0.33200000000000002,"15min":0.32200000000000001},"max":{"1min":0.78100000000000003,"5min":0.92500000000000004,"15min":1.6890000000000001},"last":2.1030000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.39700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":1.9350000000000001},"last":3.8010000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463080,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2091200,"kb_used_data":1785720,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91653952,"statfs":{"total":95995035648,"available":93853646848,"internally_reserved":0,"allocated":1828577280,"data_stored":1828105078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:10 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52200000000000002,"15min":0.53000000000000003},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":1.909},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53700000000000003,"15min":0.53600000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":3.0950000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":0.627,"15min":0.55700000000000005},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.21,"5min":6.21,"15min":6.21},"last":3.282},{"interface":"front","average":{"1min":1.464,"5min":0.70099999999999996,"15min":0.57199999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":12.616,"5min":12.616,"15min":12.616},"last":2.2040000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:05:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":1.9670000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":1.9930000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9590000000000001,"5min":1.0840000000000001,"15min":0.77200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":38.606999999999999,"5min":38.606999999999999,"15min":38.606999999999999},"last":4.8129999999999997},{"interface":"front","average":{"1min":2.6680000000000001,"5min":1.0569999999999999,"15min":0.78900000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":37.914000000000001,"5min":37.914000000000001,"15min":37.914000000000001},"last":4.2759999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48799999999999999,"15min":0.46500000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.53400000000000003,"15min":0.501},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.33000000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.55400000000000005,"15min":0.51700000000000002},"min":{"1min":0.33200000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.61499999999999999,"15min":0.57099999999999995},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":2.1139999999999999,"5min":2.1139999999999999,"15min":2.1139999999999999},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59999999999999998,"15min":0.57599999999999996},"min":{"1min":0.34799999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":2.238,"5min":2.238,"15min":2.238},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.65700000000000003,"15min":0.61599999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.2480000000000002,"5min":2.2480000000000002,"15min":2.2480000000000002},"last":0.372}]}]},{"osd":1,"up_from":89,"seq":382252089389,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1865468,"kb_used_data":1811764,"kb_used_omap":9,"kb_used_meta":53686,"kb_avail":91879684,"statfs":{"total":95995035648,"available":94084796416,"internally_reserved":0,"allocated":1855246336,"data_stored":1854719684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":54975465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":0.60099999999999998,"15min":0.44800000000000001},"min":{"1min":0.28399999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.3600000000000003,"5min":6.3600000000000003,"15min":6.3600000000000003},"last":0.251},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.64100000000000001,"15min":0.47499999999999998},"min":{"1min":0.307,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.976,"5min":7.976,"15min":7.976},"last":4.569}]},{"osd":2,"last update":"Mon Oct 9 04:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.378,"15min":0.35499999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.4129999999999998,"5min":3.4129999999999998,"15min":3.4129999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.38300000000000001,"15min":0.372},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":0.52100000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.66900000000000004,"15min":0.47299999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.3380000000000001,"5min":6.3380000000000001,"15min":6.3380000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.5880000000000001,"5min":0.624,"15min":0.46300000000000002},"min":{"1min":0.38900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.8630000000000004,"5min":7.8630000000000004,"15min":7.8630000000000004},"last":4.6580000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":0.82599999999999996,"15min":0.73699999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.9089999999999998,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.3169999999999999,"5min":0.80400000000000005,"15min":0.73599999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":4.008,"5min":4.008,"15min":4.008},"last":0.92100000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":0.84999999999999998,"15min":0.76600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":7.9459999999999997},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.371,"5min":0.82799999999999996,"15min":0.76000000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":4.0759999999999996,"5min":4.0759999999999996,"15min":4.0759999999999996},"last":1.0069999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":0.83999999999999997,"15min":0.75600000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.5780000000000001,"5min":0.878,"15min":0.78600000000000003},"min":{"1min":0.441,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":6.3140000000000001,"5min":6.3140000000000001,"15min":6.3140000000000001},"last":0.53800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":0.86799999999999999,"15min":0.78600000000000003},"min":{"1min":0.435,"5min":0.38,"15min":0.38},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.611,"5min":0.90900000000000003,"15min":0.81599999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.274,"5min":6.274,"15min":6.274},"last":0.51300000000000001}]}]},{"osd":0,"up_from":84,"seq":360777252915,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493000,"kb_used_data":1439232,"kb_used_omap":28,"kb_used_meta":53731,"kb_avail":92252152,"statfs":{"total":95995035648,"available":94466203648,"internally_reserved":0,"allocated":1473773568,"data_stored":1473233724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":55021392},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.52400000000000002,"15min":0.42399999999999999},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":7.2999999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.53500000000000003,"15min":0.42099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.5140000000000002,"5min":7.5140000000000002,"15min":7.5140000000000002},"last":4.3330000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:05:08 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.40999999999999998,"15min":0.41499999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.61699999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":4.2629999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42399999999999999,"15min":0.41999999999999998},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.766,"5min":1.766,"15min":1.766},"last":2.1920000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3449999999999998,"5min":1.5820000000000001,"15min":0.78800000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":53.387,"5min":53.387,"15min":53.387},"last":7.2370000000000001},{"interface":"front","average":{"1min":6.931,"5min":1.7390000000000001,"15min":0.874},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":54.411000000000001,"5min":54.411000000000001,"15min":54.411000000000001},"last":4.4429999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.623,"15min":0.59899999999999998},"min":{"1min":0.41499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.6850000000000001,"5min":2.6850000000000001,"15min":2.6850000000000001},"last":4.2110000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.61199999999999999,"15min":0.60999999999999999},"min":{"1min":0.437,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79500000000000004,"5min":1.024,"15min":1.024},"last":2.226}]},{"osd":5,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61799999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.81299999999999994,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":2.2509999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46300000000000002,"5min":0.44,"15min":0.44},"max":{"1min":1.577,"5min":1.577,"15min":1.577},"last":7.1859999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.65600000000000003,"15min":0.65900000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":7.2690000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.66900000000000004,"15min":0.65400000000000003},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":2.9670000000000001,"5min":2.9670000000000001,"15min":2.9670000000000001},"last":4.4119999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.65400000000000003,"15min":0.65200000000000002},"min":{"1min":0.47299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.607,"5min":1.607,"15min":1.607},"last":7.3330000000000002},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.69599999999999995,"15min":0.67700000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":3.0009999999999999,"5min":3.0009999999999999,"15min":3.0009999999999999},"last":4.3019999999999996}]}]},{"osd":2,"up_from":94,"seq":403726925862,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2888176,"kb_used_data":2812200,"kb_used_omap":81,"kb_used_meta":75886,"kb_avail":90856976,"statfs":{"total":95995035648,"available":93037543424,"internally_reserved":0,"allocated":2879692800,"data_stored":2879175815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":77707482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"min":{"1min":0.22800000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":2.839},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.39400000000000002,"15min":0.39200000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":12.619}]},{"osd":1,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.39600000000000002,"15min":0.38200000000000001},"min":{"1min":0.26600000000000001,"5min":0.221,"15min":0.221},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":8.5980000000000008},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.39900000000000002,"15min":0.377},"min":{"1min":0.252,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.6909999999999998,"5min":3.6909999999999998,"15min":3.6909999999999998},"last":5.9340000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":0.86199999999999999,"15min":0.54200000000000004},"min":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":2.8599999999999999},{"interface":"front","average":{"1min":2.4390000000000001,"5min":0.79100000000000004,"15min":0.51700000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":27.710999999999999,"5min":27.710999999999999,"15min":27.710999999999999},"last":12.662000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":5.9550000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.60299999999999998,"15min":0.59799999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":2.7509999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.65600000000000003,"15min":0.65500000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":5.9029999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63400000000000001,"15min":0.64100000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82999999999999996,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":5.976}]},{"osd":6,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.64100000000000001,"15min":0.63300000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":2.8130000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":5.8780000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.67300000000000004,"15min":0.67100000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":2.7869999999999999},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.337,"5min":1.337,"15min":1.337},"last":5.8300000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762336,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3628780,"kb_used_data":3547172,"kb_used_omap":7,"kb_used_meta":81592,"kb_avail":90116372,"statfs":{"total":95995035648,"available":92279164928,"internally_reserved":0,"allocated":3632304128,"data_stored":3631789797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":83550824},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":10.885999999999999,"5min":2.9089999999999998,"15min":1.579},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":106.22,"5min":106.22,"15min":106.22},"last":3.637},{"interface":"front","average":{"1min":11.151,"5min":2.96,"15min":1.595},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":106.986,"5min":106.986,"15min":106.986},"last":2.8050000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.3440000000000003,"5min":1.732,"15min":0.79600000000000004},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":106.121,"5min":106.121,"15min":106.121},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.46100000000000002,"15min":0.38700000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":3.4590000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.468,"5min":1.788,"15min":0.84099999999999997},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":106.105,"5min":106.105,"15min":106.105},"last":3.4809999999999999},{"interface":"front","average":{"1min":7.4180000000000001,"5min":1.7729999999999999,"15min":0.83199999999999996},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":106.70399999999999},"last":15.760999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.4279999999999999,"5min":1.96,"15min":1.048},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":106.85899999999999,"5min":106.85899999999999,"15min":106.85899999999999},"last":2.722},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2130000000000001,"5min":4.2130000000000001,"15min":4.2130000000000001},"last":0.73699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.64000000000000001,"15min":0.60899999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.887,"5min":2.887,"15min":2.887},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.0429999999999999,"15min":1.101},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.1160000000000001,"5min":9.9369999999999994,"15min":9.9369999999999994},"last":3.3999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.3689999999999998,"5min":2.363,"15min":1.5289999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":106.036,"5min":106.036,"15min":106.036},"last":3.4350000000000001},{"interface":"front","average":{"1min":7.5259999999999998,"5min":1.9970000000000001,"15min":1.075},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":106.741,"5min":106.741,"15min":106.741},"last":2.7839999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.4710000000000001,"5min":1.9970000000000001,"15min":1.085},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":106.82599999999999,"5min":106.82599999999999,"15min":106.82599999999999},"last":2.7599999999999998},{"interface":"front","average":{"1min":7.3520000000000003,"5min":1.9510000000000001,"15min":1.0509999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":106.084,"5min":106.084,"15min":106.084},"last":0.76500000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986352,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2075972,"kb_used_data":1770492,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91669180,"statfs":{"total":95995035648,"available":93869240320,"internally_reserved":0,"allocated":1812983808,"data_stored":1812507100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.62,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55200000000000005,"15min":0.56599999999999995},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":1.427}]},{"osd":1,"last update":"Mon Oct 9 04:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.56699999999999995,"15min":0.55600000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":2.657},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.58099999999999996,"15min":0.55000000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.6339999999999999,"5min":2.6339999999999999,"15min":2.6339999999999999},"last":0.82599999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:05:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58199999999999996,"15min":0.58099999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56599999999999995,"15min":0.56299999999999994},"min":{"1min":0.42799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.34999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:05:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.64700000000000002,"15min":0.57599999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.2869999999999999,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":2.633},{"interface":"front","average":{"1min":1.196,"5min":0.68500000000000005,"15min":0.59999999999999998},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8470000000000004,"5min":4.8470000000000004,"15min":4.8470000000000004},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.45700000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70699999999999996,"5min":0.753,"15min":1.038},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52300000000000002,"15min":0.48899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.85499999999999998},"last":3.7879999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":0.70099999999999996,"5min":0.82099999999999995,"15min":0.92700000000000005},"last":2.6000000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.54700000000000004,"15min":0.52900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66800000000000004,"5min":0.78200000000000003,"15min":0.93000000000000005},"last":0.502}]},{"osd":7,"last update":"Mon Oct 9 04:05:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57099999999999995,"15min":0.54300000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32600000000000001},"max":{"1min":0.69899999999999995,"5min":0.83599999999999997,"15min":1.0660000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":0.80100000000000005,"5min":0.80800000000000005,"15min":1.079},"last":5.5819999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724716,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2092276,"kb_used_data":1782700,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91652876,"statfs":{"total":95995035648,"available":93852545024,"internally_reserved":0,"allocated":1825484800,"data_stored":1825020162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:08 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56399999999999995,"15min":0.58599999999999997},"min":{"1min":0.32300000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72199999999999998,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.54900000000000004,"15min":0.56200000000000006},"min":{"1min":0.373,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66800000000000004,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.28000000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:05:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.59299999999999997,"15min":0.55900000000000005},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.9470000000000001,"5min":2.9470000000000001,"15min":2.9470000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.1739999999999999,"5min":2.1739999999999999,"15min":2.1739999999999999},"last":0.31}]},{"osd":2,"last update":"Mon Oct 9 04:05:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.56899999999999995,"15min":0.57499999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:05:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3920000000000003,"5min":1.347,"15min":0.83899999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":60.396999999999998,"5min":60.396999999999998,"15min":60.396999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":4.399,"5min":1.365,"15min":0.85899999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":60.383000000000003,"5min":60.383000000000003,"15min":60.383000000000003},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:05:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.496,"15min":0.48299999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51600000000000001,"15min":0.50900000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:05:43 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.55200000000000005,"15min":0.52600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":1.756,"5min":1.756,"15min":1.756},"last":0.316},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.59899999999999998,"15min":0.55800000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":0.38500000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:05:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.64600000000000002,"15min":0.622},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":1.6439999999999999,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":0.443}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1482293248,"data_stored":1482293248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1869217792,"data_stored":1869217792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2886074368,"data_stored":2886074368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3639345152,"data_stored":3639345152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1816461312,"data_stored":1816461312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1826881536,"data_stored":1826881536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1827274752,"data_stored":1827274752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2195783680,"data_stored":2195783680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:06:12.519 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:06:12.572 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:12.631 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:12 smithi163 ceph-mon[213841]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:12.631 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:12 smithi163 ceph-mon[213841]: pgmap v293: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:06:12.631 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/885622142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:06:12.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:12 smithi163 ceph-mon[221067]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:12.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:12 smithi163 ceph-mon[221067]: pgmap v293: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:06:12.631 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/885622142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:06:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:12 smithi181 ceph-mon[196979]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:12 smithi181 ceph-mon[196979]: pgmap v293: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:06:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/885622142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:06:12.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:12.918 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:06:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:06:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:06:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:06:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49817 6.c 6:3ffecb2a:::benchmark_data_smithi163_287034_object49816:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.224765943999998, 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21672543999999999, 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49817 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687659+0000", 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687682+0000", 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3472000000000002e-05 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687694+0000", 2023-10-09T04:06:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1080999999999999e-05 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687701+0000", 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9000000000000006e-06 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687750+0000", 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8877e-05 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687874+0000", 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123812 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.688008+0000", 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134256 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.690036+0000", 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020273660000000001 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692667+0000", 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026314289999999998 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904254+0000", 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21158665099999999 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904341+0000", 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7026000000000003e-05 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904386+0000", 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4781999999999999e-05 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49825 6.c 6:38f6223e:::benchmark_data_smithi163_287034_object49824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.219853577999999, 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.214667942, 2023-10-09T04:06:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49825 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:06:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692571+0000", 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692600+0000", 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9017999999999999e-05 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692612+0000", 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.171e-05 2023-10-09T04:06:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692620+0000", 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1240000000000005e-06 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692694+0000", 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4127000000000004e-05 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692812+0000", 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011761599999999999 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692939+0000", 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012688700000000001 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904425+0000", 2023-10-09T04:06:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.211486216 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904454+0000", 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9196999999999998e-05 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907121+0000", 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002666982 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907194+0000", 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3281999999999994e-05 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907241+0000", 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6106999999999997e-05 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:49821 6.7 e111/102 6:e6e39e35:::benchmark_data_smithi163_287034_object49820:head v 111'3139, mlcod=111'3137)", 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.693011+0000", 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.219415116, 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21149889599999999, 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693011+0000", 2023-10-09T04:06:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693011+0000", 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693010+0000", 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693046+0000", 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5963000000000001e-05 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902261+0000", 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20921489300000001 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902282+0000", 2023-10-09T04:06:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0928999999999999e-05 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902343+0000", 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0392e-05 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902372+0000", 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9485999999999999e-05 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904484+0000", 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021115830000000002 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904510+0000", 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6239999999999999e-05 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:49817 6.c e111/102 ondisk, result = 0)", 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.694745+0000", 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.217681089999999, 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20965001799999999, 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694745+0000", 2023-10-09T04:06:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694745+0000", 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694744+0000", 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694751+0000", 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.657e-06 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902268+0000", 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20751671599999999 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902279+0000", 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1395e-05 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904236+0000", 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019574560000000002 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904245+0000", 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:06:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904395+0000", 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015013 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49829 6.6 6:64b4de1c:::benchmark_data_smithi163_287034_object49828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.695061+0000", 2023-10-09T04:06:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.217364920999998, 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21301373700000001, 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49829 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695061+0000", 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695061+0000", 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695059+0000", 2023-10-09T04:06:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695085+0000", 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5497e-05 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902270+0000", 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20718461099999999 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902285+0000", 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5475e-05 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902344+0000", 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9011000000000001e-05 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902545+0000", 2023-10-09T04:06:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020052399999999999 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902710+0000", 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165833 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.906460+0000", 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037491299999999998 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.906756+0000", 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029698300000000001 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907983+0000", 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012264559999999999 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.908037+0000", 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3680000000000001e-05 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.908075+0000", 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8368999999999998e-05 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:72550 6.b 6:d0472b57:::benchmark_data_smithi163_287034_object72549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.323803+0000", 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.588623633000001, 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19669533, 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 72550 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323803+0000", 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323803+0000", 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323802+0000", 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323823+0000", 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0995000000000001e-05 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323832+0000", 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2e-06 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323839+0000", 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5989999999999998e-06 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323927+0000", 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8055999999999999e-05 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.324061+0000", 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013399899999999999 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.324213+0000", 2023-10-09T04:06:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000152514 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.331884+0000", 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076709819999999998 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.331895+0000", 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1195e-05 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520404+0000", 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18850883199999999 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520458+0000", 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3542999999999999e-05 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520498+0000", 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0225999999999997e-05 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:72552 6.b 6:d712c061:::benchmark_data_smithi163_287034_object72551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.328495+0000", 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.583931613000001, 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.192125461, 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 72552 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328495+0000", 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328495+0000", 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328493+0000", 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328517+0000", 2023-10-09T04:06:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3468e-05 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328530+0000", 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3723000000000001e-05 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328538+0000", 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0180000000000001e-06 2023-10-09T04:06:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.328986+0000", 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044755199999999999 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.329120+0000", 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013450100000000001 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.329250+0000", 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012955099999999999 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.332106+0000", 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028562079999999998 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335356+0000", 2023-10-09T04:06:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032495179999999999 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520543+0000", 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18518739400000001 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520582+0000", 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8729999999999997e-05 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520620+0000", 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8278000000000002e-05 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:72555 6.c 6:347acb2a:::benchmark_data_smithi163_287034_object72554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.334893+0000", 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.577532729000001, 2023-10-09T04:06:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19192100200000001, 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 72555 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.334893+0000", 2023-10-09T04:06:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.334893+0000", 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.334892+0000", 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.334916+0000", 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4278e-05 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.334929+0000", 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2754e-05 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.334937+0000", 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7959999999999996e-06 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336435+0000", 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001498779 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.519746+0000", 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18331107399999999 2023-10-09T04:06:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.519918+0000", 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017123 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.525090+0000", 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051727730000000003 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526546+0000", 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001455736 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526732+0000", 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018541900000000001 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526778+0000", 2023-10-09T04:06:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6266000000000001e-05 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526814+0000", 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6529999999999998e-05 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:72557 6.7 e115/102 6:edc7b404:::benchmark_data_smithi163_287034_object72556:head v 115'4466, mlcod=115'4462)", 2023-10-09T04:06:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.335466+0000", 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.576960052, 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.191593179, 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335466+0000", 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335466+0000", 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335466+0000", 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335498+0000", 2023-10-09T04:06:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2551000000000001e-05 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335505+0000", 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5640000000000002e-06 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.335510+0000", 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8130000000000003e-06 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.519803+0000", 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.184293344 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.519827+0000", 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4477e-05 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.527022+0000", 2023-10-09T04:06:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071947729999999998 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.527059+0000", 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7052999999999999e-05 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:72559 6.c 6:3ca0f443:::benchmark_data_smithi163_287034_object72558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.336375+0000", 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.576051162999999, 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.190529595, 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 72559 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336375+0000", 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336375+0000", 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336374+0000", 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336406+0000", 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2183e-05 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336419+0000", 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3359e-05 2023-10-09T04:06:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.336427+0000", 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1559999999999995e-06 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.522364+0000", 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18593690299999999 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.522509+0000", 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014499200000000001 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.522652+0000", 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014285900000000001 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526386+0000", 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003733569 2023-10-09T04:06:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526698+0000", 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000312844 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526846+0000", 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014737800000000001 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526877+0000", 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0945000000000003e-05 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.526905+0000", 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7815000000000001e-05 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74814 6.8 e115/102 6:16d1b9ba:::benchmark_data_smithi163_287034_object74813:head v 115'4588, mlcod=115'4586)", 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.629639+0000", 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.282787540000001, 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20429694800000001, 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629639+0000", 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629639+0000", 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629638+0000", 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629678+0000", 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0129999999999997e-05 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629686+0000", 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3080000000000002e-06 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629691+0000", 2023-10-09T04:06:12.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.254e-06 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629806+0000", 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115343 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629827+0000", 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1121999999999999e-05 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637214+0000", 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0073863389999999996 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833936+0000", 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19672177099999999 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74819 6.8 e115/102 6:15765e6f:::benchmark_data_smithi163_287034_object74818:head v 115'4589, mlcod=115'4586)", 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.632540+0000", 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.279886297000001, 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20142286600000001, 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632540+0000", 2023-10-09T04:06:12.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632540+0000", 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632539+0000", 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632581+0000", 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1063999999999999e-05 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632588+0000", 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8990000000000001e-06 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632594+0000", 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1619999999999999e-06 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632633+0000", 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9753999999999999e-05 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632652+0000", 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9153000000000001e-05 2023-10-09T04:06:12.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833941+0000", 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.201288997 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833963+0000", 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1254e-05 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:12.994 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74820 6.8 e115/102 6:1dfd3969:::benchmark_data_smithi163_287034_object74819:head v 115'4590, mlcod=115'4586)", 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.279818066000001, 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.201374003, 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:12.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:12.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632638+0000", 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9832000000000002e-05 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632643+0000", 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3720000000000004e-06 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632648+0000", 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6940000000000003e-06 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635127+0000", 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024792460000000001 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635147+0000", 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9981999999999999e-05 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833965+0000", 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19881852699999999 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:12.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833982+0000", 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6657999999999999e-05 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:74826 6.c 6:3c5736f5:::benchmark_data_smithi163_287034_object74825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.635420+0000", 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.277006214, 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20139859900000001, 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.000 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 74826 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635420+0000", 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635420+0000", 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635419+0000", 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635443+0000", 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4479999999999999e-05 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635455+0000", 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2052000000000001e-05 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635462+0000", 2023-10-09T04:06:13.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3490000000000003e-06 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635506+0000", 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3711999999999998e-05 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635652+0000", 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014605400000000001 2023-10-09T04:06:13.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635788+0000", 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013596999999999999 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833768+0000", 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.197980351 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833867+0000", 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8288000000000005e-05 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836718+0000", 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028510089999999998 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836774+0000", 2023-10-09T04:06:13.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6316000000000001e-05 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836819+0000", 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4450999999999998e-05 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:74834 6.6 6:6df70a8f:::benchmark_data_smithi163_287034_object74833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:13.007 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.636840+0000", 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.275586398000002, 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20286137700000001, 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 74834 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636840+0000", 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636840+0000", 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636839+0000", 2023-10-09T04:06:13.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636867+0000", 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8098000000000001e-05 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636883+0000", 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5999999999999999e-05 2023-10-09T04:06:13.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636900+0000", 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6571000000000001e-05 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636945+0000", 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5806999999999997e-05 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637072+0000", 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127007 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637161+0000", 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8671000000000005e-05 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836353+0000", 2023-10-09T04:06:13.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.199191808 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836502+0000", 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000148782 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839613+0000", 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031116669999999998 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839658+0000", 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4487999999999998e-05 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839701+0000", 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3353999999999999e-05 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74824 6.f e115/102 6:fbaaa0a0:::benchmark_data_smithi163_287034_object74823:head v 115'4572, mlcod=115'4570)", 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.637749+0000", 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.274677064999999, 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.201700351, 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637749+0000", 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637749+0000", 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637748+0000", 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:06:13.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637771+0000", 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2456999999999999e-05 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833661+0000", 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19588987499999999 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833675+0000", 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.483e-05 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833727+0000", 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1326000000000003e-05 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833756+0000", 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9680999999999999e-05 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839417+0000", 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005661068 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839449+0000", 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2044e-05 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.019 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:74826 6.c e115/102 ondisk, result = 0)", 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.638598+0000", 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.273828071000001, 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19517346999999999, 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638598+0000", 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638598+0000", 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638597+0000", 2023-10-09T04:06:13.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638603+0000", 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2780000000000001e-06 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833666+0000", 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.195063559 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833679+0000", 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2798e-05 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833754+0000", 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4839999999999995e-05 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833762+0000", 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9500000000000001e-06 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833772+0000", 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6870000000000006e-06 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.024 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74831 6.5 e115/102 6:ae4cc717:::benchmark_data_smithi163_287034_object74830:head v 115'4683, mlcod=115'4681)", 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.639852+0000", 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.272574222999999, 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19929846000000001, 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639852+0000", 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639852+0000", 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639851+0000", 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639901+0000", 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9347e-05 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833669+0000", 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.193768836 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833681+0000", 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1488000000000001e-05 2023-10-09T04:06:13.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833804+0000", 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123533 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833826+0000", 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1231999999999999e-05 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839114+0000", 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052885750000000002 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839150+0000", 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6173000000000001e-05 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:76838 6.b 6:dfe81907:::benchmark_data_smithi163_287034_object76837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:51.775593+0000", 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.136833369000001, 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.190251841, 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 76838 2023-10-09T04:06:13.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775593+0000", 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775593+0000", 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775592+0000", 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775620+0000", 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8019999999999999e-05 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775629+0000", 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5140000000000001e-06 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775635+0000", 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9490000000000001e-06 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.778109+0000", 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024740460000000001 2023-10-09T04:06:13.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.778241+0000", 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013273899999999999 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.778379+0000", 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013787599999999999 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.783398+0000", 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050188409999999996 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.783495+0000", 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6557000000000004e-05 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965772+0000", 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18227702100000001 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965815+0000", 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2794999999999999e-05 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965845+0000", 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0073000000000002e-05 2023-10-09T04:06:13.036 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:76840 6.b 6:ddcc3ebe:::benchmark_data_smithi163_287034_object76839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:51.775653+0000", 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.136773066, 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19028984299999999, 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:06:13.037 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 76840 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775653+0000", 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775653+0000", 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775653+0000", 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775674+0000", 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1291999999999999e-05 2023-10-09T04:06:13.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775680+0000", 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0070000000000003e-06 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.775685+0000", 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3650000000000003e-06 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.781139+0000", 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054539799999999998 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.781268+0000", 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129283 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.781388+0000", 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011981600000000001 2023-10-09T04:06:13.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.783583+0000", 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021949830000000002 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965106+0000", 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.181522304 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965876+0000", 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077038400000000004 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965915+0000", 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8578999999999998e-05 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:51.965943+0000", 2023-10-09T04:06:13.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8391999999999999e-05 2023-10-09T04:06:13.044 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.044 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.044 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.044 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:06:13.044 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:06:13.044 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:06:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:13.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:13.045 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 95222 95206 81.4666 48.5 0.0353949 0.0122709 2023-10-09T04:06:13.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:13.200 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:13.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:13.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/885622142' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:06:13.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T04:06:13.523 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[213841]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/885622142' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T04:06:13.524 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:13 smithi163 ceph-mon[221067]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/885622142' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: failed to encode map e120 with expected crc 2023-10-09T04:06:13.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:13 smithi181 ceph-mon[196979]: failed to encode map e120 with expected crc 2023-10-09T04:06:14.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 96027 96011 81.0457 50.3125 0.0229264 0.0123342 2023-10-09T04:06:14.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:14.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:14.237 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:06:14.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:14.395 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:14.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:14 smithi163 ceph-mon[213841]: pgmap v295: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T04:06:14.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:14.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:14.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:14.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:14.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:14 smithi181 ceph-mon[196979]: pgmap v295: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T04:06:14.791 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:14.812 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:14.814 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:14 smithi163 ceph-mon[221067]: pgmap v295: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T04:06:15.020 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 96935 96919 80.7218 56.75 0.0158783 0.0123836 2023-10-09T04:06:15.420 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:15.434 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:15.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:15.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:15.643 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:15.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:15 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:15.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:15 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:15.746 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:15 smithi163 ceph-mon[213841]: pgmap v296: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-09T04:06:15.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:15 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:15.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:15 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:15.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:15 smithi163 ceph-mon[221067]: pgmap v296: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-09T04:06:15.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:15 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:15.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:15 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:15.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:15 smithi181 ceph-mon[196979]: pgmap v296: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-09T04:06:15.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:15.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:16.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:16.021 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 97801 97785 80.3715 54.125 0.02085 0.0124371 2023-10-09T04:06:16.084 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:16.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:06:16.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:16.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:16.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:16.536 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:06:16.601 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:06:16.632 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:06:16.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:16.679 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:06:16.711 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:06:16.714 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:16.757 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:06:16.786 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:06:16.822 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:06:16.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2925357568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:06:16.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2925357568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:17.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2925357568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:17.021 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 98577 98561 79.9577 48.5 0.0903063 0.0124954 2023-10-09T04:06:17.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:17.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:17.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:17.628 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:17.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:17.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:17.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:17.853 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:18.021 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 99374 99358 79.5714 49.8125 0.0196384 0.0125619 2023-10-09T04:06:18.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:18.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:18.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:18.160 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:18 smithi163 ceph-mon[213841]: pgmap v297: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T04:06:18.210 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:18.283 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:18.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:18.393 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:18.400 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:18.423 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:18 smithi181 ceph-mon[196979]: pgmap v297: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T04:06:18.425 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:18.436 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:18 smithi163 ceph-mon[221067]: pgmap v297: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T04:06:18.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:18.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:18.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:18.641 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:18.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:18.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:18.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:18.844 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:18.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:18.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:19.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:19.023 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 100204 100188 79.2185 51.875 0.0159607 0.0126181 2023-10-09T04:06:19.064 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:19.131 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:19.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:19.139 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:06:19.145 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:06:19.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:19.318 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:20.009 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:20.009 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:20.024 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:06:20.024301+0000 min lat: 0.00118809 max lat: 0.296599 avg lat: 0.0126849 2023-10-09T04:06:20.024 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:06:20.024 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 100941 100925 78.8042 46.0625 0.0270491 0.0126849 2023-10-09T04:06:20.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:06:20.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:20.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:20.441 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":297,"stamp":"2023-10-09T04:06:17.304445+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6357915552,"num_objects":97029,"num_object_clones":0,"num_object_copies":291087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97029,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":97305,"num_write_kb":6210765,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97221,"ondisk_log_size":97221,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20008568,"kb_used_data":18537976,"kb_used_omap":126,"kb_used_meta":1470401,"kb_avail":729952648,"statfs":{"total":767960285184,"available":747471511552,"internally_reserved":0,"allocated":18982887424,"data_stored":18978872135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1505691259},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,11,10,7,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":533135360,"num_objects":8135,"num_object_clones":0,"num_object_copies":24405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8135,"num_write_kb":520640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004759"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":97,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387394+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:06:02.387394+0000","last_peered":"2023-10-09T04:06:02.387394+0000","last_clean":"2023-10-09T04:06:02.387394+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:06:02.387394+0000","last_undegraded":"2023-10-09T04:06:02.387394+0000","last_fullsized":"2023-10-09T04:06:02.387394+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":82,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:05.283716+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:06:05.283716+0000","last_peered":"2023-10-09T04:06:05.283716+0000","last_clean":"2023-10-09T04:06:05.283716+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:06:05.283716+0000","last_undegraded":"2023-10-09T04:06:05.283716+0000","last_fullsized":"2023-10-09T04:06:05.283716+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398713+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:06:02.398713+0000","last_peered":"2023-10-09T04:06:02.398713+0000","last_clean":"2023-10-09T04:06:02.398713+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:02.398713+0000","last_undegraded":"2023-10-09T04:06:02.398713+0000","last_fullsized":"2023-10-09T04:06:02.398713+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400599+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:06:02.400599+0000","last_peered":"2023-10-09T04:06:02.400599+0000","last_clean":"2023-10-09T04:06:02.400599+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:06:02.400599+0000","last_undegraded":"2023-10-09T04:06:02.400599+0000","last_fullsized":"2023-10-09T04:06:02.400599+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":100,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378857+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:06:02.378857+0000","last_peered":"2023-10-09T04:06:02.378857+0000","last_clean":"2023-10-09T04:06:02.378857+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:06:02.378857+0000","last_undegraded":"2023-10-09T04:06:02.378857+0000","last_fullsized":"2023-10-09T04:06:02.378857+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385981+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:06:02.385981+0000","last_peered":"2023-10-09T04:06:02.385981+0000","last_clean":"2023-10-09T04:06:02.385981+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:06:02.385981+0000","last_undegraded":"2023-10-09T04:06:02.385981+0000","last_fullsized":"2023-10-09T04:06:02.385981+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400842+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:06:02.400842+0000","last_peered":"2023-10-09T04:06:02.400842+0000","last_clean":"2023-10-09T04:06:02.400842+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:06:02.400842+0000","last_undegraded":"2023-10-09T04:06:02.400842+0000","last_fullsized":"2023-10-09T04:06:02.400842+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":96,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378837+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:06:02.378837+0000","last_peered":"2023-10-09T04:06:02.378837+0000","last_clean":"2023-10-09T04:06:02.378837+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:06:02.378837+0000","last_undegraded":"2023-10-09T04:06:02.378837+0000","last_fullsized":"2023-10-09T04:06:02.378837+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384317+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:06:02.384317+0000","last_peered":"2023-10-09T04:06:02.384317+0000","last_clean":"2023-10-09T04:06:02.384317+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:06:02.384317+0000","last_undegraded":"2023-10-09T04:06:02.384317+0000","last_fullsized":"2023-10-09T04:06:02.384317+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":43,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378911+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:06:02.378911+0000","last_peered":"2023-10-09T04:06:02.378911+0000","last_clean":"2023-10-09T04:06:02.378911+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:06:02.378911+0000","last_undegraded":"2023-10-09T04:06:02.378911+0000","last_fullsized":"2023-10-09T04:06:02.378911+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":82,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381602+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:06:02.381602+0000","last_peered":"2023-10-09T04:06:02.381602+0000","last_clean":"2023-10-09T04:06:02.381602+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:06:02.381602+0000","last_undegraded":"2023-10-09T04:06:02.381602+0000","last_fullsized":"2023-10-09T04:06:02.381602+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":69,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.397038+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:06:02.397038+0000","last_peered":"2023-10-09T04:06:02.397038+0000","last_clean":"2023-10-09T04:06:02.397038+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:06:02.397038+0000","last_undegraded":"2023-10-09T04:06:02.397038+0000","last_fullsized":"2023-10-09T04:06:02.397038+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387562+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:06:02.387562+0000","last_peered":"2023-10-09T04:06:02.387562+0000","last_clean":"2023-10-09T04:06:02.387562+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:06:02.387562+0000","last_undegraded":"2023-10-09T04:06:02.387562+0000","last_fullsized":"2023-10-09T04:06:02.387562+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387416+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:06:02.387416+0000","last_peered":"2023-10-09T04:06:02.387416+0000","last_clean":"2023-10-09T04:06:02.387416+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:06:02.387416+0000","last_undegraded":"2023-10-09T04:06:02.387416+0000","last_fullsized":"2023-10-09T04:06:02.387416+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381672+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:06:02.381672+0000","last_peered":"2023-10-09T04:06:02.381672+0000","last_clean":"2023-10-09T04:06:02.381672+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:06:02.381672+0000","last_undegraded":"2023-10-09T04:06:02.381672+0000","last_fullsized":"2023-10-09T04:06:02.381672+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":82,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.379079+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:06:02.379079+0000","last_peered":"2023-10-09T04:06:02.379079+0000","last_clean":"2023-10-09T04:06:02.379079+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:06:02.379079+0000","last_undegraded":"2023-10-09T04:06:02.379079+0000","last_fullsized":"2023-10-09T04:06:02.379079+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400430+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:06:02.400430+0000","last_peered":"2023-10-09T04:06:02.400430+0000","last_clean":"2023-10-09T04:06:02.400430+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:06:02.400430+0000","last_undegraded":"2023-10-09T04:06:02.400430+0000","last_fullsized":"2023-10-09T04:06:02.400430+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384333+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:06:02.384333+0000","last_peered":"2023-10-09T04:06:02.384333+0000","last_clean":"2023-10-09T04:06:02.384333+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:06:02.384333+0000","last_undegraded":"2023-10-09T04:06:02.384333+0000","last_fullsized":"2023-10-09T04:06:02.384333+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381788+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:06:02.381788+0000","last_peered":"2023-10-09T04:06:02.381788+0000","last_clean":"2023-10-09T04:06:02.381788+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:06:02.381788+0000","last_undegraded":"2023-10-09T04:06:02.381788+0000","last_fullsized":"2023-10-09T04:06:02.381788+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400545+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:06:02.400545+0000","last_peered":"2023-10-09T04:06:02.400545+0000","last_clean":"2023-10-09T04:06:02.400545+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:06:02.400545+0000","last_undegraded":"2023-10-09T04:06:02.400545+0000","last_fullsized":"2023-10-09T04:06:02.400545+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385519+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:06:02.385519+0000","last_peered":"2023-10-09T04:06:02.385519+0000","last_clean":"2023-10-09T04:06:02.385519+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:06:02.385519+0000","last_undegraded":"2023-10-09T04:06:02.385519+0000","last_fullsized":"2023-10-09T04:06:02.385519+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":82,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.379250+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:06:02.379250+0000","last_peered":"2023-10-09T04:06:02.379250+0000","last_clean":"2023-10-09T04:06:02.379250+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:06:02.379250+0000","last_undegraded":"2023-10-09T04:06:02.379250+0000","last_fullsized":"2023-10-09T04:06:02.379250+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"120'6159","reported_seq":6184,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.226065+0000","last_change":"2023-10-09T04:04:51.281106+0000","last_active":"2023-10-09T04:06:15.226065+0000","last_peered":"2023-10-09T04:06:15.226065+0000","last_clean":"2023-10-09T04:06:15.226065+0000","last_became_active":"2023-10-09T04:04:51.280862+0000","last_became_peered":"2023-10-09T04:04:51.280862+0000","last_unstale":"2023-10-09T04:06:15.226065+0000","last_undegraded":"2023-10-09T04:06:15.226065+0000","last_fullsized":"2023-10-09T04:06:15.226065+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:47.717190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403636224,"num_objects":6159,"num_object_clones":0,"num_object_copies":18477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":80,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384040+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:06:02.384040+0000","last_peered":"2023-10-09T04:06:02.384040+0000","last_clean":"2023-10-09T04:06:02.384040+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:06:02.384040+0000","last_undegraded":"2023-10-09T04:06:02.384040+0000","last_fullsized":"2023-10-09T04:06:02.384040+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384111+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:06:02.384111+0000","last_peered":"2023-10-09T04:06:02.384111+0000","last_clean":"2023-10-09T04:06:02.384111+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:06:02.384111+0000","last_undegraded":"2023-10-09T04:06:02.384111+0000","last_fullsized":"2023-10-09T04:06:02.384111+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"120'6074","reported_seq":6101,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.278627+0000","last_change":"2023-10-09T04:04:51.277948+0000","last_active":"2023-10-09T04:06:15.278627+0000","last_peered":"2023-10-09T04:06:15.278627+0000","last_clean":"2023-10-09T04:06:15.278627+0000","last_became_active":"2023-10-09T04:04:51.277674+0000","last_became_peered":"2023-10-09T04:04:51.277674+0000","last_unstale":"2023-10-09T04:06:15.278627+0000","last_undegraded":"2023-10-09T04:06:15.278627+0000","last_fullsized":"2023-10-09T04:06:15.278627+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6074,"log_dups_size":0,"ondisk_log_size":6074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:12.182469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398065664,"num_objects":6074,"num_object_clones":0,"num_object_copies":18222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6074,"num_write_kb":388736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.383910+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:06:02.383910+0000","last_peered":"2023-10-09T04:06:02.383910+0000","last_clean":"2023-10-09T04:06:02.383910+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:06:02.383910+0000","last_undegraded":"2023-10-09T04:06:02.383910+0000","last_fullsized":"2023-10-09T04:06:02.383910+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387304+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:06:02.387304+0000","last_peered":"2023-10-09T04:06:02.387304+0000","last_clean":"2023-10-09T04:06:02.387304+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:06:02.387304+0000","last_undegraded":"2023-10-09T04:06:02.387304+0000","last_fullsized":"2023-10-09T04:06:02.387304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"120'6009","reported_seq":6035,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.187390+0000","last_change":"2023-10-09T04:04:51.279912+0000","last_active":"2023-10-09T04:06:15.187390+0000","last_peered":"2023-10-09T04:06:15.187390+0000","last_clean":"2023-10-09T04:06:15.187390+0000","last_became_active":"2023-10-09T04:04:51.279633+0000","last_became_peered":"2023-10-09T04:04:51.279633+0000","last_unstale":"2023-10-09T04:06:15.187390+0000","last_undegraded":"2023-10-09T04:06:15.187390+0000","last_fullsized":"2023-10-09T04:06:15.187390+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:48.533141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393805824,"num_objects":6009,"num_object_clones":0,"num_object_copies":18027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6009,"num_write_kb":384576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":63,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381747+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:06:02.381747+0000","last_peered":"2023-10-09T04:06:02.381747+0000","last_clean":"2023-10-09T04:06:02.381747+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:06:02.381747+0000","last_undegraded":"2023-10-09T04:06:02.381747+0000","last_fullsized":"2023-10-09T04:06:02.381747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385713+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:06:02.385713+0000","last_peered":"2023-10-09T04:06:02.385713+0000","last_clean":"2023-10-09T04:06:02.385713+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:06:02.385713+0000","last_undegraded":"2023-10-09T04:06:02.385713+0000","last_fullsized":"2023-10-09T04:06:02.385713+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"120'6236","reported_seq":6263,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:16.687597+0000","last_change":"2023-10-09T04:04:51.276847+0000","last_active":"2023-10-09T04:06:16.687597+0000","last_peered":"2023-10-09T04:06:16.687597+0000","last_clean":"2023-10-09T04:06:16.687597+0000","last_became_active":"2023-10-09T04:04:51.276578+0000","last_became_peered":"2023-10-09T04:04:51.276578+0000","last_unstale":"2023-10-09T04:06:16.687597+0000","last_undegraded":"2023-10-09T04:06:16.687597+0000","last_fullsized":"2023-10-09T04:06:16.687597+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6236,"log_dups_size":0,"ondisk_log_size":6236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:27.626784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408682496,"num_objects":6236,"num_object_clones":0,"num_object_copies":18708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6236,"num_write_kb":399104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384479+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:06:02.384479+0000","last_peered":"2023-10-09T04:06:02.384479+0000","last_clean":"2023-10-09T04:06:02.384479+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:06:02.384479+0000","last_undegraded":"2023-10-09T04:06:02.384479+0000","last_fullsized":"2023-10-09T04:06:02.384479+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400720+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:06:02.400720+0000","last_peered":"2023-10-09T04:06:02.400720+0000","last_clean":"2023-10-09T04:06:02.400720+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:06:02.400720+0000","last_undegraded":"2023-10-09T04:06:02.400720+0000","last_fullsized":"2023-10-09T04:06:02.400720+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"120'6027","reported_seq":6053,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.214337+0000","last_change":"2023-10-09T04:04:51.279219+0000","last_active":"2023-10-09T04:06:15.214337+0000","last_peered":"2023-10-09T04:06:15.214337+0000","last_clean":"2023-10-09T04:06:15.214337+0000","last_became_active":"2023-10-09T04:04:51.278902+0000","last_became_peered":"2023-10-09T04:04:51.278902+0000","last_unstale":"2023-10-09T04:06:15.214337+0000","last_undegraded":"2023-10-09T04:06:15.214337+0000","last_fullsized":"2023-10-09T04:06:15.214337+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:37.687116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394985472,"num_objects":6027,"num_object_clones":0,"num_object_copies":18081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6027,"num_write_kb":385728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":80,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.379206+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:06:02.379206+0000","last_peered":"2023-10-09T04:06:02.379206+0000","last_clean":"2023-10-09T04:06:02.379206+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:06:02.379206+0000","last_undegraded":"2023-10-09T04:06:02.379206+0000","last_fullsized":"2023-10-09T04:06:02.379206+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387462+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:06:02.387462+0000","last_peered":"2023-10-09T04:06:02.387462+0000","last_clean":"2023-10-09T04:06:02.387462+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:06:02.387462+0000","last_undegraded":"2023-10-09T04:06:02.387462+0000","last_fullsized":"2023-10-09T04:06:02.387462+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"120'5944","reported_seq":5971,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:14.329952+0000","last_change":"2023-10-09T04:04:51.275697+0000","last_active":"2023-10-09T04:06:14.329952+0000","last_peered":"2023-10-09T04:06:14.329952+0000","last_clean":"2023-10-09T04:06:14.329952+0000","last_became_active":"2023-10-09T04:04:51.275402+0000","last_became_peered":"2023-10-09T04:04:51.275402+0000","last_unstale":"2023-10-09T04:06:14.329952+0000","last_undegraded":"2023-10-09T04:06:14.329952+0000","last_fullsized":"2023-10-09T04:06:14.329952+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:26.490300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389545984,"num_objects":5944,"num_object_clones":0,"num_object_copies":17832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5944,"num_write_kb":380416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385792+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:06:02.385792+0000","last_peered":"2023-10-09T04:06:02.385792+0000","last_clean":"2023-10-09T04:06:02.385792+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:06:02.385792+0000","last_undegraded":"2023-10-09T04:06:02.385792+0000","last_fullsized":"2023-10-09T04:06:02.385792+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387514+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:06:02.387514+0000","last_peered":"2023-10-09T04:06:02.387514+0000","last_clean":"2023-10-09T04:06:02.387514+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:06:02.387514+0000","last_undegraded":"2023-10-09T04:06:02.387514+0000","last_fullsized":"2023-10-09T04:06:02.387514+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"120'6072","reported_seq":6099,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.277940+0000","last_change":"2023-10-09T04:04:51.279552+0000","last_active":"2023-10-09T04:06:15.277940+0000","last_peered":"2023-10-09T04:06:15.277940+0000","last_clean":"2023-10-09T04:06:15.277940+0000","last_became_active":"2023-10-09T04:04:51.279242+0000","last_became_peered":"2023-10-09T04:04:51.279242+0000","last_unstale":"2023-10-09T04:06:15.277940+0000","last_undegraded":"2023-10-09T04:06:15.277940+0000","last_fullsized":"2023-10-09T04:06:15.277940+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6072,"log_dups_size":0,"ondisk_log_size":6072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:44.784925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397934592,"num_objects":6072,"num_object_clones":0,"num_object_copies":18216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6072,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387184+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:06:02.387184+0000","last_peered":"2023-10-09T04:06:02.387184+0000","last_clean":"2023-10-09T04:06:02.387184+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:06:02.387184+0000","last_undegraded":"2023-10-09T04:06:02.387184+0000","last_fullsized":"2023-10-09T04:06:02.387184+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387977+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:06:02.387977+0000","last_peered":"2023-10-09T04:06:02.387977+0000","last_clean":"2023-10-09T04:06:02.387977+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:06:02.387977+0000","last_undegraded":"2023-10-09T04:06:02.387977+0000","last_fullsized":"2023-10-09T04:06:02.387977+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"120'6107","reported_seq":6133,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.183543+0000","last_change":"2023-10-09T04:04:51.279867+0000","last_active":"2023-10-09T04:06:15.183543+0000","last_peered":"2023-10-09T04:06:15.183543+0000","last_clean":"2023-10-09T04:06:15.183543+0000","last_became_active":"2023-10-09T04:04:51.279588+0000","last_became_peered":"2023-10-09T04:04:51.279588+0000","last_unstale":"2023-10-09T04:06:15.183543+0000","last_undegraded":"2023-10-09T04:06:15.183543+0000","last_fullsized":"2023-10-09T04:06:15.183543+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6107,"log_dups_size":0,"ondisk_log_size":6107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.265205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400228352,"num_objects":6107,"num_object_clones":0,"num_object_copies":18321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6107,"num_write_kb":390848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378948+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:06:02.378948+0000","last_peered":"2023-10-09T04:06:02.378948+0000","last_clean":"2023-10-09T04:06:02.378948+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:06:02.378948+0000","last_undegraded":"2023-10-09T04:06:02.378948+0000","last_fullsized":"2023-10-09T04:06:02.378948+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":69,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.397635+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:06:02.397635+0000","last_peered":"2023-10-09T04:06:02.397635+0000","last_clean":"2023-10-09T04:06:02.397635+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:06:02.397635+0000","last_undegraded":"2023-10-09T04:06:02.397635+0000","last_fullsized":"2023-10-09T04:06:02.397635+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":92,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398389+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:06:02.398389+0000","last_peered":"2023-10-09T04:06:02.398389+0000","last_clean":"2023-10-09T04:06:02.398389+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:06:02.398389+0000","last_undegraded":"2023-10-09T04:06:02.398389+0000","last_fullsized":"2023-10-09T04:06:02.398389+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":96,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384430+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:06:02.384430+0000","last_peered":"2023-10-09T04:06:02.384430+0000","last_clean":"2023-10-09T04:06:02.384430+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:02.384430+0000","last_undegraded":"2023-10-09T04:06:02.384430+0000","last_fullsized":"2023-10-09T04:06:02.384430+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.397167+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:06:02.397167+0000","last_peered":"2023-10-09T04:06:02.397167+0000","last_clean":"2023-10-09T04:06:02.397167+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:06:02.397167+0000","last_undegraded":"2023-10-09T04:06:02.397167+0000","last_fullsized":"2023-10-09T04:06:02.397167+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"120'6087","reported_seq":6114,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:16.687760+0000","last_change":"2023-10-09T04:04:51.276404+0000","last_active":"2023-10-09T04:06:16.687760+0000","last_peered":"2023-10-09T04:06:16.687760+0000","last_clean":"2023-10-09T04:06:16.687760+0000","last_became_active":"2023-10-09T04:04:51.276109+0000","last_became_peered":"2023-10-09T04:04:51.276109+0000","last_unstale":"2023-10-09T04:06:16.687760+0000","last_undegraded":"2023-10-09T04:06:16.687760+0000","last_fullsized":"2023-10-09T04:06:16.687760+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6087,"log_dups_size":0,"ondisk_log_size":6087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:04.681582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398917632,"num_objects":6087,"num_object_clones":0,"num_object_copies":18261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6087,"num_write_kb":389568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"120'6171","reported_seq":6198,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.267516+0000","last_change":"2023-10-09T04:04:51.277034+0000","last_active":"2023-10-09T04:06:15.267516+0000","last_peered":"2023-10-09T04:06:15.267516+0000","last_clean":"2023-10-09T04:06:15.267516+0000","last_became_active":"2023-10-09T04:04:51.276751+0000","last_became_peered":"2023-10-09T04:04:51.276751+0000","last_unstale":"2023-10-09T04:06:15.267516+0000","last_undegraded":"2023-10-09T04:06:15.267516+0000","last_fullsized":"2023-10-09T04:06:15.267516+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6171,"log_dups_size":0,"ondisk_log_size":6171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:13.233763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404422656,"num_objects":6171,"num_object_clones":0,"num_object_copies":18513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6171,"num_write_kb":394944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384411+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:06:02.384411+0000","last_peered":"2023-10-09T04:06:02.384411+0000","last_clean":"2023-10-09T04:06:02.384411+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:06:02.384411+0000","last_undegraded":"2023-10-09T04:06:02.384411+0000","last_fullsized":"2023-10-09T04:06:02.384411+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387513+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:06:02.387513+0000","last_peered":"2023-10-09T04:06:02.387513+0000","last_clean":"2023-10-09T04:06:02.387513+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:06:02.387513+0000","last_undegraded":"2023-10-09T04:06:02.387513+0000","last_fullsized":"2023-10-09T04:06:02.387513+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398321+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:06:02.398321+0000","last_peered":"2023-10-09T04:06:02.398321+0000","last_clean":"2023-10-09T04:06:02.398321+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:06:02.398321+0000","last_undegraded":"2023-10-09T04:06:02.398321+0000","last_fullsized":"2023-10-09T04:06:02.398321+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"120'6032","reported_seq":6058,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.136161+0000","last_change":"2023-10-09T04:04:51.279627+0000","last_active":"2023-10-09T04:06:15.136161+0000","last_peered":"2023-10-09T04:06:15.136161+0000","last_clean":"2023-10-09T04:06:15.136161+0000","last_became_active":"2023-10-09T04:04:51.279326+0000","last_became_peered":"2023-10-09T04:04:51.279326+0000","last_unstale":"2023-10-09T04:06:15.136161+0000","last_undegraded":"2023-10-09T04:06:15.136161+0000","last_fullsized":"2023-10-09T04:06:15.136161+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6032,"log_dups_size":0,"ondisk_log_size":6032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:58.605251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395313152,"num_objects":6032,"num_object_clones":0,"num_object_copies":18096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6032,"num_write_kb":386048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":102,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398311+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:06:02.398311+0000","last_peered":"2023-10-09T04:06:02.398311+0000","last_clean":"2023-10-09T04:06:02.398311+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:06:02.398311+0000","last_undegraded":"2023-10-09T04:06:02.398311+0000","last_fullsized":"2023-10-09T04:06:02.398311+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.397044+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:06:02.397044+0000","last_peered":"2023-10-09T04:06:02.397044+0000","last_clean":"2023-10-09T04:06:02.397044+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:06:02.397044+0000","last_undegraded":"2023-10-09T04:06:02.397044+0000","last_fullsized":"2023-10-09T04:06:02.397044+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.397703+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:06:02.397703+0000","last_peered":"2023-10-09T04:06:02.397703+0000","last_clean":"2023-10-09T04:06:02.397703+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:06:02.397703+0000","last_undegraded":"2023-10-09T04:06:02.397703+0000","last_fullsized":"2023-10-09T04:06:02.397703+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"120'5978","reported_seq":6004,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.163809+0000","last_change":"2023-10-09T04:04:51.280419+0000","last_active":"2023-10-09T04:06:15.163809+0000","last_peered":"2023-10-09T04:06:15.163809+0000","last_clean":"2023-10-09T04:06:15.163809+0000","last_became_active":"2023-10-09T04:04:51.280174+0000","last_became_peered":"2023-10-09T04:04:51.280174+0000","last_unstale":"2023-10-09T04:06:15.163809+0000","last_undegraded":"2023-10-09T04:06:15.163809+0000","last_fullsized":"2023-10-09T04:06:15.163809+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5978,"log_dups_size":0,"ondisk_log_size":5978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:03.946653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391774208,"num_objects":5978,"num_object_clones":0,"num_object_copies":17934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5978,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384251+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:06:02.384251+0000","last_peered":"2023-10-09T04:06:02.384251+0000","last_clean":"2023-10-09T04:06:02.384251+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:06:02.384251+0000","last_undegraded":"2023-10-09T04:06:02.384251+0000","last_fullsized":"2023-10-09T04:06:02.384251+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":460,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381147+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:06:02.381147+0000","last_peered":"2023-10-09T04:06:02.381147+0000","last_clean":"2023-10-09T04:06:02.381147+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:06:02.381147+0000","last_undegraded":"2023-10-09T04:06:02.381147+0000","last_fullsized":"2023-10-09T04:06:02.381147+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":43,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378979+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:06:02.378979+0000","last_peered":"2023-10-09T04:06:02.378979+0000","last_clean":"2023-10-09T04:06:02.378979+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:06:02.378979+0000","last_undegraded":"2023-10-09T04:06:02.378979+0000","last_fullsized":"2023-10-09T04:06:02.378979+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":61,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385920+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:06:02.385920+0000","last_peered":"2023-10-09T04:06:02.385920+0000","last_clean":"2023-10-09T04:06:02.385920+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:06:02.385920+0000","last_undegraded":"2023-10-09T04:06:02.385920+0000","last_fullsized":"2023-10-09T04:06:02.385920+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":100,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381160+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:06:02.381160+0000","last_peered":"2023-10-09T04:06:02.381160+0000","last_clean":"2023-10-09T04:06:02.381160+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:06:02.381160+0000","last_undegraded":"2023-10-09T04:06:02.381160+0000","last_fullsized":"2023-10-09T04:06:02.381160+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:06.698041+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:06:06.698041+0000","last_peered":"2023-10-09T04:06:06.698041+0000","last_clean":"2023-10-09T04:06:06.698041+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:06:06.698041+0000","last_undegraded":"2023-10-09T04:06:06.698041+0000","last_fullsized":"2023-10-09T04:06:06.698041+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400705+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:06:02.400705+0000","last_peered":"2023-10-09T04:06:02.400705+0000","last_clean":"2023-10-09T04:06:02.400705+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:06:02.400705+0000","last_undegraded":"2023-10-09T04:06:02.400705+0000","last_fullsized":"2023-10-09T04:06:02.400705+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"120'6008","reported_seq":6035,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:13.111158+0000","last_change":"2023-10-09T04:04:51.279899+0000","last_active":"2023-10-09T04:06:13.111158+0000","last_peered":"2023-10-09T04:06:13.111158+0000","last_clean":"2023-10-09T04:06:13.111158+0000","last_became_active":"2023-10-09T04:04:51.279611+0000","last_became_peered":"2023-10-09T04:04:51.279611+0000","last_unstale":"2023-10-09T04:06:13.111158+0000","last_undegraded":"2023-10-09T04:06:13.111158+0000","last_fullsized":"2023-10-09T04:06:13.111158+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6008,"log_dups_size":0,"ondisk_log_size":6008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:31.582709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393740288,"num_objects":6008,"num_object_clones":0,"num_object_copies":18024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6008,"num_write_kb":384512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381671+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:06:02.381671+0000","last_peered":"2023-10-09T04:06:02.381671+0000","last_clean":"2023-10-09T04:06:02.381671+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:06:02.381671+0000","last_undegraded":"2023-10-09T04:06:02.381671+0000","last_fullsized":"2023-10-09T04:06:02.381671+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398115+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:06:02.398115+0000","last_peered":"2023-10-09T04:06:02.398115+0000","last_clean":"2023-10-09T04:06:02.398115+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:06:02.398115+0000","last_undegraded":"2023-10-09T04:06:02.398115+0000","last_fullsized":"2023-10-09T04:06:02.398115+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:11.707629+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:06:11.707629+0000","last_peered":"2023-10-09T04:06:11.707629+0000","last_clean":"2023-10-09T04:06:11.707629+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:06:11.707629+0000","last_undegraded":"2023-10-09T04:06:11.707629+0000","last_fullsized":"2023-10-09T04:06:11.707629+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"120'5936","reported_seq":5961,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:13.137388+0000","last_change":"2023-10-09T04:04:51.278104+0000","last_active":"2023-10-09T04:06:13.137388+0000","last_peered":"2023-10-09T04:06:13.137388+0000","last_clean":"2023-10-09T04:06:13.137388+0000","last_became_active":"2023-10-09T04:04:51.277849+0000","last_became_peered":"2023-10-09T04:04:51.277849+0000","last_unstale":"2023-10-09T04:06:13.137388+0000","last_undegraded":"2023-10-09T04:06:13.137388+0000","last_fullsized":"2023-10-09T04:06:13.137388+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.049817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389021696,"num_objects":5936,"num_object_clones":0,"num_object_copies":17808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5936,"num_write_kb":379904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":80,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.379318+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:06:02.379318+0000","last_peered":"2023-10-09T04:06:02.379318+0000","last_clean":"2023-10-09T04:06:02.379318+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:06:02.379318+0000","last_undegraded":"2023-10-09T04:06:02.379318+0000","last_fullsized":"2023-10-09T04:06:02.379318+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":53,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398320+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:06:02.398320+0000","last_peered":"2023-10-09T04:06:02.398320+0000","last_clean":"2023-10-09T04:06:02.398320+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:06:02.398320+0000","last_undegraded":"2023-10-09T04:06:02.398320+0000","last_fullsized":"2023-10-09T04:06:02.398320+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378927+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:06:02.378927+0000","last_peered":"2023-10-09T04:06:02.378927+0000","last_clean":"2023-10-09T04:06:02.378927+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:06:02.378927+0000","last_undegraded":"2023-10-09T04:06:02.378927+0000","last_fullsized":"2023-10-09T04:06:02.378927+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"120'6063","reported_seq":6089,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.170131+0000","last_change":"2023-10-09T04:04:51.275204+0000","last_active":"2023-10-09T04:06:15.170131+0000","last_peered":"2023-10-09T04:06:15.170131+0000","last_clean":"2023-10-09T04:06:15.170131+0000","last_became_active":"2023-10-09T04:04:51.274910+0000","last_became_peered":"2023-10-09T04:04:51.274910+0000","last_unstale":"2023-10-09T04:06:15.170131+0000","last_undegraded":"2023-10-09T04:06:15.170131+0000","last_fullsized":"2023-10-09T04:06:15.170131+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6063,"log_dups_size":0,"ondisk_log_size":6063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:08.996248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397344768,"num_objects":6063,"num_object_clones":0,"num_object_copies":18189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6063,"num_write_kb":388032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385971+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:06:02.385971+0000","last_peered":"2023-10-09T04:06:02.385971+0000","last_clean":"2023-10-09T04:06:02.385971+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:06:02.385971+0000","last_undegraded":"2023-10-09T04:06:02.385971+0000","last_fullsized":"2023-10-09T04:06:02.385971+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":76,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400647+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:06:02.400647+0000","last_peered":"2023-10-09T04:06:02.400647+0000","last_clean":"2023-10-09T04:06:02.400647+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:06:02.400647+0000","last_undegraded":"2023-10-09T04:06:02.400647+0000","last_fullsized":"2023-10-09T04:06:02.400647+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387198+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:06:02.387198+0000","last_peered":"2023-10-09T04:06:02.387198+0000","last_clean":"2023-10-09T04:06:02.387198+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:06:02.387198+0000","last_undegraded":"2023-10-09T04:06:02.387198+0000","last_fullsized":"2023-10-09T04:06:02.387198+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"120'6102","reported_seq":6128,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:15.197842+0000","last_change":"2023-10-09T04:04:51.277953+0000","last_active":"2023-10-09T04:06:15.197842+0000","last_peered":"2023-10-09T04:06:15.197842+0000","last_clean":"2023-10-09T04:06:15.197842+0000","last_became_active":"2023-10-09T04:04:51.277659+0000","last_became_peered":"2023-10-09T04:04:51.277659+0000","last_unstale":"2023-10-09T04:06:15.197842+0000","last_undegraded":"2023-10-09T04:06:15.197842+0000","last_fullsized":"2023-10-09T04:06:15.197842+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6102,"log_dups_size":0,"ondisk_log_size":6102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:22.890014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399900672,"num_objects":6102,"num_object_clones":0,"num_object_copies":18306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6102,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":61,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400803+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:06:02.400803+0000","last_peered":"2023-10-09T04:06:02.400803+0000","last_clean":"2023-10-09T04:06:02.400803+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:06:02.400803+0000","last_undegraded":"2023-10-09T04:06:02.400803+0000","last_fullsized":"2023-10-09T04:06:02.400803+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.398636+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:06:02.398636+0000","last_peered":"2023-10-09T04:06:02.398636+0000","last_clean":"2023-10-09T04:06:02.398636+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:06:02.398636+0000","last_undegraded":"2023-10-09T04:06:02.398636+0000","last_fullsized":"2023-10-09T04:06:02.398636+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.385273+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:06:02.385273+0000","last_peered":"2023-10-09T04:06:02.385273+0000","last_clean":"2023-10-09T04:06:02.385273+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:06:02.385273+0000","last_undegraded":"2023-10-09T04:06:02.385273+0000","last_fullsized":"2023-10-09T04:06:02.385273+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387597+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:06:02.387597+0000","last_peered":"2023-10-09T04:06:02.387597+0000","last_clean":"2023-10-09T04:06:02.387597+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:06:02.387597+0000","last_undegraded":"2023-10-09T04:06:02.387597+0000","last_fullsized":"2023-10-09T04:06:02.387597+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":64,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384522+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:06:02.384522+0000","last_peered":"2023-10-09T04:06:02.384522+0000","last_clean":"2023-10-09T04:06:02.384522+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:06:02.384522+0000","last_undegraded":"2023-10-09T04:06:02.384522+0000","last_fullsized":"2023-10-09T04:06:02.384522+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387550+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:06:02.387550+0000","last_peered":"2023-10-09T04:06:02.387550+0000","last_clean":"2023-10-09T04:06:02.387550+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:06:02.387550+0000","last_undegraded":"2023-10-09T04:06:02.387550+0000","last_fullsized":"2023-10-09T04:06:02.387550+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.379065+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:06:02.379065+0000","last_peered":"2023-10-09T04:06:02.379065+0000","last_clean":"2023-10-09T04:06:02.379065+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:06:02.379065+0000","last_undegraded":"2023-10-09T04:06:02.379065+0000","last_fullsized":"2023-10-09T04:06:02.379065+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":103,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.378911+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:06:02.378911+0000","last_peered":"2023-10-09T04:06:02.378911+0000","last_clean":"2023-10-09T04:06:02.378911+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:06:02.378911+0000","last_undegraded":"2023-10-09T04:06:02.378911+0000","last_fullsized":"2023-10-09T04:06:02.378911+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":77,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:06.698149+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:06:06.698149+0000","last_peered":"2023-10-09T04:06:06.698149+0000","last_clean":"2023-10-09T04:06:06.698149+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:06:06.698149+0000","last_undegraded":"2023-10-09T04:06:06.698149+0000","last_fullsized":"2023-10-09T04:06:06.698149+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.387174+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:06:02.387174+0000","last_peered":"2023-10-09T04:06:02.387174+0000","last_clean":"2023-10-09T04:06:02.387174+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:06:02.387174+0000","last_undegraded":"2023-10-09T04:06:02.387174+0000","last_fullsized":"2023-10-09T04:06:02.387174+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6357319680,"num_objects":97005,"num_object_clones":0,"num_object_copies":291015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97005,"num_write_kb":6208320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19051642880,"data_stored":19051642880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97005,"ondisk_log_size":97005,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201445,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2632088,"kb_used_data":2326608,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91113064,"statfs":{"total":95995035648,"available":93299777536,"internally_reserved":0,"allocated":2382446592,"data_stored":2381958381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,5,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":0.69499999999999995,"15min":0.57499999999999996},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"last":2.444},{"interface":"front","average":{"1min":1.389,"5min":0.69199999999999995,"15min":0.57599999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.1479999999999997,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":1.6100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61599999999999999,"15min":0.624},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.72599999999999998,"15min":0.70999999999999996},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.964,"5min":3.363,"15min":3.363},"last":1.8100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":4.7030000000000003,"5min":1.3779999999999999,"15min":0.82399999999999995},"min":{"1min":0.32500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":58.615000000000002,"5min":58.615000000000002,"15min":58.615000000000002},"last":1.6799999999999999},{"interface":"front","average":{"1min":5.1870000000000003,"5min":1.52,"15min":0.91000000000000003},"min":{"1min":0.38200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":58.798000000000002,"5min":58.798000000000002,"15min":58.798000000000002},"last":1.6990000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":1.008,"15min":0.72199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":31.170999999999999,"5min":31.170999999999999,"15min":31.170999999999999},"last":1.9379999999999999},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.071,"15min":0.76700000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":31.184000000000001,"5min":31.184000000000001,"15min":31.184000000000001},"last":1.9470000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.2650000000000001,"5min":2.2650000000000001,"15min":6.3129999999999997},"last":1.4219999999999999},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.61499999999999999,"15min":0.55600000000000005},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":2.3180000000000001,"5min":2.3180000000000001,"15min":3.8769999999999998},"last":1.5700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53400000000000003,"15min":0.52400000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.4059999999999999,"5min":1.4330000000000001,"15min":1.464},"last":1.4059999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.63800000000000001,"15min":0.57999999999999996},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":5.1399999999999997,"5min":5.1399999999999997,"15min":5.1399999999999997},"last":1.456}]},{"osd":6,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.59399999999999997,"15min":0.55500000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.1030000000000002,"5min":2.1030000000000002,"15min":2.1030000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.63500000000000001,"15min":0.59799999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":3.8010000000000002},"last":1.484}]}]},{"osd":6,"up_from":47,"seq":201863463082,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226824,"kb_used_data":1929536,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91518328,"statfs":{"total":95995035648,"available":93714767872,"internally_reserved":0,"allocated":1975844864,"data_stored":1975372568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.70499999999999996,"15min":0.59099999999999997},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":14.557,"5min":14.557,"15min":14.557},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":0.84899999999999998,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":14.632999999999999},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":0.627,"15min":0.55700000000000005},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.21,"5min":6.21,"15min":6.21},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.464,"5min":0.70099999999999996,"15min":0.57199999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":12.616,"5min":12.616,"15min":12.616},"last":2.8660000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":1.125,"15min":0.748},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":42.195999999999998,"5min":42.195999999999998,"15min":42.195999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.4180000000000001,"5min":1.177,"15min":0.80100000000000005},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":42.165999999999997,"5min":42.165999999999997,"15min":42.165999999999997},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9590000000000001,"5min":1.0840000000000001,"15min":0.77200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":38.606999999999999,"5min":38.606999999999999,"15min":38.606999999999999},"last":1.603},{"interface":"front","average":{"1min":2.6680000000000001,"5min":1.0569999999999999,"15min":0.78900000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":37.914000000000001,"5min":37.914000000000001,"15min":37.914000000000001},"last":0.97199999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48799999999999999,"15min":0.46500000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.31},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.53400000000000003,"15min":0.501},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.497}]},{"osd":5,"last update":"Mon Oct 9 04:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.55400000000000005,"15min":0.51700000000000002},"min":{"1min":0.33200000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":1.5669999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.61499999999999999,"15min":0.57099999999999995},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":2.1139999999999999,"5min":2.1139999999999999,"15min":2.1139999999999999},"last":3.391}]},{"osd":7,"last update":"Mon Oct 9 04:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59999999999999998,"15min":0.57599999999999996},"min":{"1min":0.34799999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":2.238,"5min":2.238,"15min":2.238},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.65700000000000003,"15min":0.61599999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.2480000000000002,"5min":2.2480000000000002,"15min":2.2480000000000002},"last":0.40999999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089390,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1964808,"kb_used_data":1907008,"kb_used_omap":9,"kb_used_meta":57782,"kb_avail":91780344,"statfs":{"total":95995035648,"available":93983072256,"internally_reserved":0,"allocated":1952776192,"data_stored":1952245681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":59169769},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":0.60099999999999998,"15min":0.44800000000000001},"min":{"1min":0.28399999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.3600000000000003,"5min":6.3600000000000003,"15min":6.3600000000000003},"last":2.5600000000000001},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.64100000000000001,"15min":0.47499999999999998},"min":{"1min":0.307,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.976,"5min":7.976,"15min":7.976},"last":2.9870000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":0.621,"15min":0.436},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.383,"5min":6.383,"15min":6.383},"last":0.31},{"interface":"front","average":{"1min":3.194,"5min":0.94899999999999995,"15min":0.56100000000000005},"min":{"1min":0.33700000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.833,"5min":10.833,"15min":10.833},"last":2.956}]},{"osd":3,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.66900000000000004,"15min":0.47299999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.3380000000000001,"5min":6.3380000000000001,"15min":6.3380000000000001},"last":2.9399999999999999},{"interface":"front","average":{"1min":1.5880000000000001,"5min":0.624,"15min":0.46300000000000002},"min":{"1min":0.38900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.8630000000000004,"5min":7.8630000000000004,"15min":7.8630000000000004},"last":3.02}]},{"osd":4,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":0.82599999999999996,"15min":0.73699999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.9089999999999998,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.433},{"interface":"front","average":{"1min":1.3169999999999999,"5min":0.80400000000000005,"15min":0.73599999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":4.008,"5min":4.008,"15min":4.008},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":0.84999999999999998,"15min":0.76600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":7.9459999999999997},"last":2.903},{"interface":"front","average":{"1min":1.371,"5min":0.82799999999999996,"15min":0.76000000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":4.0759999999999996,"5min":4.0759999999999996,"15min":4.0759999999999996},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":0.83999999999999997,"15min":0.75600000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.5780000000000001,"5min":0.878,"15min":0.78600000000000003},"min":{"1min":0.441,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":6.3140000000000001,"5min":6.3140000000000001,"15min":6.3140000000000001},"last":0.48399999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":0.86799999999999999,"15min":0.78600000000000003},"min":{"1min":0.435,"5min":0.38,"15min":0.38},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":0.498},{"interface":"front","average":{"1min":1.611,"5min":0.90900000000000003,"15min":0.81599999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.274,"5min":6.274,"15min":6.274},"last":1.8069999999999999}]}]},{"osd":0,"up_from":84,"seq":360777252917,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622932,"kb_used_data":1577356,"kb_used_omap":28,"kb_used_meta":45539,"kb_avail":92122220,"statfs":{"total":95995035648,"available":94333153280,"internally_reserved":0,"allocated":1615212544,"data_stored":1614668841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":46632784},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.52400000000000002,"15min":0.42399999999999999},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":118.48},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.53500000000000003,"15min":0.42099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.5140000000000002,"5min":7.5140000000000002,"15min":7.5140000000000002},"last":118.384}]},{"osd":2,"last update":"Mon Oct 9 04:06:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":0.81000000000000005,"15min":0.54800000000000004},"min":{"1min":0.255,"5min":0.224,"15min":0.224},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":118.54000000000001},{"interface":"front","average":{"1min":2.1480000000000001,"5min":0.77000000000000002,"15min":0.53600000000000003},"min":{"1min":0.23499999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":118.727}]},{"osd":3,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3449999999999998,"5min":1.5820000000000001,"15min":0.78800000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":53.387,"5min":53.387,"15min":53.387},"last":118.73999999999999},{"interface":"front","average":{"1min":6.931,"5min":1.7390000000000001,"15min":0.874},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":54.411000000000001,"5min":54.411000000000001,"15min":54.411000000000001},"last":118.584}]},{"osd":4,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.623,"15min":0.59899999999999998},"min":{"1min":0.41499999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.6850000000000001,"5min":2.6850000000000001,"15min":2.6850000000000001},"last":118.633},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.61199999999999999,"15min":0.60999999999999999},"min":{"1min":0.437,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79500000000000004,"5min":1.024,"15min":1.024},"last":118.913}]},{"osd":5,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.61799999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.81299999999999994,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":118.943},{"interface":"front","average":{"1min":0.627,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46300000000000002,"5min":0.44,"15min":0.44},"max":{"1min":1.577,"5min":1.577,"15min":1.577},"last":118.898}]},{"osd":6,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.65600000000000003,"15min":0.65900000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":118.864},{"interface":"front","average":{"1min":0.749,"5min":0.66900000000000004,"15min":0.65400000000000003},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":2.9670000000000001,"5min":2.9670000000000001,"15min":2.9670000000000001},"last":118.70399999999999}]},{"osd":7,"last update":"Mon Oct 9 04:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.65400000000000003,"15min":0.65200000000000002},"min":{"1min":0.47299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.607,"5min":1.607,"15min":1.607},"last":118.834},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.69599999999999995,"15min":0.67700000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":3.0009999999999999,"5min":3.0009999999999999,"15min":3.0009999999999999},"last":118.66200000000001}]}]},{"osd":2,"up_from":94,"seq":403726925864,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3136508,"kb_used_data":3064628,"kb_used_omap":81,"kb_used_meta":71790,"kb_avail":90608644,"statfs":{"total":95995035648,"available":92783251456,"internally_reserved":0,"allocated":3138179072,"data_stored":3137658228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":73513178},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":0.63400000000000001,"15min":0.436},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.5700000000000003,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.9199999999999999,"5min":0.89900000000000002,"15min":0.56100000000000005},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.619,"5min":12.619,"15min":12.619},"last":9.0749999999999993}]},{"osd":1,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":5.492,"5min":1.419,"15min":0.72299999999999998},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":68.009,"5min":68.009,"15min":68.009},"last":4.3490000000000002},{"interface":"front","average":{"1min":5.9630000000000001,"5min":1.518,"15min":0.75},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":66.975999999999999,"5min":66.975999999999999,"15min":66.975999999999999},"last":3.9580000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":0.86199999999999999,"15min":0.54200000000000004},"min":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":5.8879999999999999},{"interface":"front","average":{"1min":2.4390000000000001,"5min":0.79100000000000004,"15min":0.51700000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":27.710999999999999,"5min":27.710999999999999,"15min":27.710999999999999},"last":4.101}]},{"osd":4,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":0.97199999999999998,"15min":0.71999999999999997},"min":{"1min":0.41799999999999998,"5min":0.375,"15min":0.375},"max":{"1min":10.553000000000001,"5min":10.553000000000001,"15min":10.553000000000001},"last":1.613},{"interface":"front","average":{"1min":1.377,"5min":0.75900000000000001,"15min":0.65000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.7930000000000001,"5min":3.7930000000000001,"15min":3.7930000000000001},"last":1.512}]},{"osd":5,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":0.86299999999999999,"15min":0.72399999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.9029999999999996,"5min":5.9029999999999996,"15min":5.9029999999999996},"last":4.0030000000000001},{"interface":"front","average":{"1min":2.4409999999999998,"5min":0.99299999999999999,"15min":0.76100000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":10.507999999999999,"5min":10.507999999999999,"15min":10.507999999999999},"last":1.5920000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":0.79700000000000004,"15min":0.68500000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":3.746,"5min":3.746,"15min":3.746},"last":1.5609999999999999},{"interface":"front","average":{"1min":1.8129999999999999,"5min":0.90700000000000003,"15min":0.755},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.8780000000000001,"5min":5.8780000000000001,"15min":5.8780000000000001},"last":4.0739999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":0.82799999999999996,"15min":0.72199999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.714,"5min":3.714,"15min":3.714},"last":1.54},{"interface":"front","average":{"1min":1.873,"5min":0.94299999999999995,"15min":0.78800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.8300000000000001,"5min":5.8300000000000001,"15min":5.8300000000000001},"last":4.0469999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762338,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3965048,"kb_used_data":3871152,"kb_used_omap":7,"kb_used_meta":93880,"kb_avail":89780104,"statfs":{"total":95995035648,"available":91934826496,"internally_reserved":0,"allocated":3964059648,"data_stored":3963541458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":96133736},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":10.885999999999999,"5min":2.9089999999999998,"15min":1.579},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":106.22,"5min":106.22,"15min":106.22},"last":8.2270000000000003},{"interface":"front","average":{"1min":11.151,"5min":2.96,"15min":1.595},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":106.986,"5min":106.986,"15min":106.986},"last":0.63300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.3440000000000003,"5min":1.732,"15min":0.79600000000000004},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":106.121,"5min":106.121,"15min":106.121},"last":8.1519999999999992},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.46100000000000002,"15min":0.38700000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":8.1760000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.468,"5min":1.788,"15min":0.84099999999999997},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":106.105,"5min":106.105,"15min":106.105},"last":10.645},{"interface":"front","average":{"1min":7.4180000000000001,"5min":1.7729999999999999,"15min":0.83199999999999996},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.68600000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.4279999999999999,"5min":1.96,"15min":1.048},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":106.85899999999999,"5min":106.85899999999999,"15min":106.85899999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2130000000000001,"5min":4.2130000000000001,"15min":4.2130000000000001},"last":3.7989999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.64000000000000001,"15min":0.60899999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.887,"5min":2.887,"15min":2.887},"last":3.8759999999999999},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.0429999999999999,"15min":1.101},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.1160000000000001,"5min":9.9369999999999994,"15min":9.9369999999999994},"last":6.4050000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.3689999999999998,"5min":2.363,"15min":1.5289999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":106.036,"5min":106.036,"15min":106.036},"last":6.3620000000000001},{"interface":"front","average":{"1min":7.5259999999999998,"5min":1.9970000000000001,"15min":1.075},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":106.741,"5min":106.741,"15min":106.741},"last":0.65700000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.4710000000000001,"5min":1.9970000000000001,"15min":1.085},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":106.82599999999999,"5min":106.82599999999999,"15min":106.82599999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":7.3520000000000003,"5min":1.9510000000000001,"15min":1.0509999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":106.084,"5min":106.084,"15min":106.084},"last":3.8450000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986354,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2227212,"kb_used_data":1929924,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91517940,"statfs":{"total":95995035648,"available":93714370560,"internally_reserved":0,"allocated":1976242176,"data_stored":1975765374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":0.69299999999999995,"15min":0.58199999999999996},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.9900000000000002,"5min":8.9900000000000002,"15min":8.9900000000000002},"last":4},{"interface":"front","average":{"1min":2.1200000000000001,"5min":0.86099999999999999,"15min":0.66900000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.1370000000000005,"5min":8.1370000000000005,"15min":8.1370000000000005},"last":2.8650000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.56699999999999995,"15min":0.55600000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.58099999999999996,"15min":0.55000000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.6339999999999999,"5min":2.6339999999999999,"15min":2.6339999999999999},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":0.85199999999999998,"15min":0.67100000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":8.8680000000000003,"5min":8.8680000000000003,"15min":8.8680000000000003},"last":8.8680000000000003},{"interface":"front","average":{"1min":1.331,"5min":0.71999999999999997,"15min":0.61399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.855,"5min":3.855,"15min":3.855},"last":0.38}]},{"osd":3,"last update":"Mon Oct 9 04:05:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.64700000000000002,"15min":0.57599999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.2869999999999999,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.196,"5min":0.68500000000000005,"15min":0.59999999999999998},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8470000000000004,"5min":4.8470000000000004,"15min":4.8470000000000004},"last":9.0329999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.45700000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70699999999999996,"5min":0.753,"15min":1.038},"last":1.274},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52300000000000002,"15min":0.48899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.85499999999999998},"last":4.3520000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":0.70099999999999996,"5min":0.82099999999999995,"15min":0.92700000000000005},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.54700000000000004,"15min":0.52900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66800000000000004,"5min":0.78200000000000003,"15min":0.93000000000000005},"last":0.42099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.58899999999999997,"15min":0.55300000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":9.0050000000000008},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.69499999999999995,"15min":0.629},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":5.5819999999999999,"5min":5.5819999999999999,"15min":5.5819999999999999},"last":1.2110000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724718,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2233148,"kb_used_data":1931764,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91512004,"statfs":{"total":95995035648,"available":93708292096,"internally_reserved":0,"allocated":1978126336,"data_stored":1977661604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.64700000000000002,"15min":0.61399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5339999999999998,"5min":4.5339999999999998,"15min":4.5339999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":1.575,"5min":0.75,"15min":0.629},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":4.1429999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:05:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.59299999999999997,"15min":0.55900000000000005},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.9470000000000001,"5min":2.9470000000000001,"15min":2.9470000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.1739999999999999,"5min":2.1739999999999999,"15min":2.1739999999999999},"last":3.6709999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":0.90500000000000003,"15min":0.68700000000000006},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.6880000000000006,"5min":9.6880000000000006,"15min":9.6880000000000006},"last":4.4269999999999996},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.81100000000000005,"15min":0.629},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.484,"5min":9.484,"15min":9.484},"last":1.4890000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:05:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3920000000000003,"5min":1.347,"15min":0.83899999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":60.396999999999998,"5min":60.396999999999998,"15min":60.396999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":4.399,"5min":1.365,"15min":0.85899999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":60.383000000000003,"5min":60.383000000000003,"15min":60.383000000000003},"last":0.52500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:05:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.496,"15min":0.48299999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51600000000000001,"15min":0.50900000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.31900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:05:43 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.55200000000000005,"15min":0.52600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":1.756,"5min":1.756,"15min":1.756},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.59899999999999998,"15min":0.55800000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":0.379}]},{"osd":7,"last update":"Mon Oct 9 04:05:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.59599999999999997,"15min":0.58499999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.64600000000000002,"15min":0.622},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.34599999999999997},"max":{"1min":1.6439999999999999,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":0.41799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1621884928,"data_stored":1621884928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1952579584,"data_stored":1952579584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3146973184,"data_stored":3146973184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3968204800,"data_stored":3968204800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1981022208,"data_stored":1981022208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1994784768,"data_stored":1994784768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1990918144,"data_stored":1990918144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2395275264,"data_stored":2395275264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:06:20.448 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:06:20.448 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:06:20.448 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-10-09T04:06:20.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:06:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:20 smithi181 ceph-mon[196979]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:20.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:20 smithi181 ceph-mon[196979]: pgmap v298: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T04:06:20.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[213841]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:20.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:20.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:20.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[213841]: pgmap v298: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T04:06:20.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[221067]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:20.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:20.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:20.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:20 smithi163 ceph-mon[221067]: pgmap v298: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-09T04:06:20.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:20.681 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:20.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:06:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:06:20] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-10-09T04:06:20.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:06:21.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 101678 101662 78.4001 46.0625 0.0122255 0.01275 2023-10-09T04:06:21.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:21.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:21.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:21.262 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:21.286 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:21.380 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:21.400 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:21.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:21.457 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:21.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:21.560 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:21.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:21.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:21.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:21.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:21.830 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:22.024 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 102437 102421 78.0226 47.4375 0.0273913 0.0128113 2023-10-09T04:06:22.403 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:06:22.646 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:22 smithi163 ceph-mon[213841]: pgmap v299: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T04:06:22.646 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/308556428' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:22.647 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:22 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:22.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:22 smithi163 ceph-mon[221067]: pgmap v299: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T04:06:22.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/308556428' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:22.647 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:22 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:22.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:22 smithi181 ceph-mon[196979]: pgmap v299: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T04:06:22.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/308556428' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:22.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:22 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:23.024 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 103040 103024 77.5369 37.6875 0.0190751 0.012892 2023-10-09T04:06:23.106 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:23.107 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:23.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:23.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:23.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:23.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:23.313 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:23.405 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:23 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:23.405 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:23.405 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:23 smithi181 ceph-mon[196979]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T04:06:23.405 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:23 smithi181 ceph-mon[196979]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[213841]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T04:06:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[213841]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[213841]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.407 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[213841]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:23.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:23.603 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:23.636 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:23 smithi181 ceph-mon[196979]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:23 smithi181 ceph-mon[196979]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:23.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:23.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[221067]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T04:06:23.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[221067]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[221067]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:23 smithi163 ceph-mon[221067]: failed to encode map e121 with expected crc 2023-10-09T04:06:23.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:06:23.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:23.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:23.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:24.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 103803 103787 77.1808 47.6875 0.014274 0.012952 2023-10-09T04:06:24.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:24.114 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:24.171 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:24.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:24.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:24.346 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:24.479 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:24.486 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:24 smithi181 ceph-mon[196979]: pgmap v301: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T04:06:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:24.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:24.525 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:24.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:24.667 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:24.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:24 smithi163 ceph-mon[213841]: pgmap v301: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T04:06:24.676 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:24 smithi163 ceph-mon[221067]: pgmap v301: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-10-09T04:06:24.809 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:24.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:24.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:24.994 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:25.025 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 104619 104603 76.8729 51 0.022891 0.0130031 2023-10-09T04:06:25.212 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:06:25.254 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:06:25.284 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:06:25.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:25.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:25.306 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:06:25.390 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:25.424 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:25.481 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:06:25.519 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:06:25.561 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:25.566 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:06:25.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3945095119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:25.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:25.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3945095119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:25.601 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:06:25.628 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.628 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.629 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.629 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.629 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.629 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.629 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.629 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:06:25.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:25.757 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:25.759 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3945095119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:25.768 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:06:25.784 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:06:26.025 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 105253 105237 76.4399 39.625 0.0108147 0.0130774 2023-10-09T04:06:26.321 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:06:26.321 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:11424 6.2 6:46bfba83:::benchmark_data_smithi163_287034_object11423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:03.627400+0000", 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 82.691584921, 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.130163053, 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 11424 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627400+0000", 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627400+0000", 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627398+0000", 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627440+0000", 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1733999999999997e-05 2023-10-09T04:06:26.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627452+0000", 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.226e-05 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627459+0000", 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1779999999999997e-06 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627496+0000", 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6961000000000001e-05 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627620+0000", 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012327900000000001 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.627759+0000", 2023-10-09T04:06:26.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013894999999999999 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.628532+0000", 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00077343599999999996 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.629256+0000", 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00072403300000000001 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.757459+0000", 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12820324499999999 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.757528+0000", 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8353000000000001e-05 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.757563+0000", 2023-10-09T04:06:26.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4949e-05 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:11430 6.e 6:70ec3524:::benchmark_data_smithi163_287034_object11429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:03.629900+0000", 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 82.689083991000004, 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.127687089, 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.329 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 11430 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.629900+0000", 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.629900+0000", 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.629900+0000", 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.630043+0000", 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000143697 2023-10-09T04:06:26.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.630057+0000", 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3219e-05 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.630063+0000", 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1530000000000002e-06 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.630114+0000", 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1138999999999999e-05 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.630248+0000", 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013386 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.630342+0000", 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4573000000000004e-05 2023-10-09T04:06:26.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.631085+0000", 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074247099999999997 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.742984+0000", 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.111899002 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.757522+0000", 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014537767 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.757564+0000", 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2211e-05 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:03.757588+0000", 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3733000000000001e-05 2023-10-09T04:06:26.335 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:20547 6.e 6:7bc01f43:::benchmark_data_smithi163_287034_object20546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:07.563361+0000", 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 78.755623704000001, 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14434345300000001, 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.336 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 20547 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.563361+0000", 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.563361+0000", 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.563360+0000", 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.563391+0000", 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0892999999999997e-05 2023-10-09T04:06:26.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.563399+0000", 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.622e-06 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.563404+0000", 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2299999999999999e-06 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.564363+0000", 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00095890999999999999 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.564495+0000", 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013219499999999999 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.564631+0000", 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013620799999999999 2023-10-09T04:06:26.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566253+0000", 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0016216819999999999 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.568701+0000", 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002447814 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.707639+0000", 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13893799400000001 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.707675+0000", 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6655000000000001e-05 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.707704+0000", 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8989999999999999e-05 2023-10-09T04:06:26.342 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:20551 6.e 6:767e41f5:::benchmark_data_smithi163_287034_object20550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:07.566343+0000", 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 78.752641367999999, 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.143248984, 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 20551 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566343+0000", 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566343+0000", 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566342+0000", 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566443+0000", 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100778 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566454+0000", 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1168999999999999e-05 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566460+0000", 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7830000000000004e-06 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566509+0000", 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9159000000000001e-05 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566614+0000", 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105064 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566728+0000", 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113689 2023-10-09T04:06:26.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.568634+0000", 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001906617 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.569010+0000", 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037608599999999998 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.709503+0000", 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14049292099999999 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.709552+0000", 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8708000000000002e-05 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.709592+0000", 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0077999999999998e-05 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.349 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:20552 6.e 6:7941e2b0:::benchmark_data_smithi163_287034_object20551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:07.566471+0000", 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 78.752513223999998, 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.143223618, 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.350 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 20552 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566471+0000", 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566471+0000", 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566471+0000", 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566584+0000", 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113566 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566593+0000", 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4360000000000002e-06 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.566598+0000", 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4009999999999996e-06 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.567459+0000", 2023-10-09T04:06:26.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086051300000000003 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.567581+0000", 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122026 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.567699+0000", 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011838800000000001 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.568914+0000", 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001215011 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.569559+0000", 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000644934 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.709623+0000", 2023-10-09T04:06:26.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14006433099999999 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.709663+0000", 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9957999999999999e-05 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:07.709695+0000", 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1619000000000003e-05 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.356 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:49827 6.2 6:49f3043b:::benchmark_data_smithi163_287034_object49826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:26.692786+0000", 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 59.626198465999998, 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21363768299999999, 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.357 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 49827 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692786+0000", 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692786+0000", 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692785+0000", 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692818+0000", 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2650000000000001e-05 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692826+0000", 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7659999999999998e-06 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692831+0000", 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7319999999999998e-06 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.694701+0000", 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018707089999999999 2023-10-09T04:06:26.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.694816+0000", 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114417 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.694947+0000", 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013104300000000001 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696455+0000", 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015081649999999999 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.697142+0000", 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00068721600000000004 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906190+0000", 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20904785400000001 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906245+0000", 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4871999999999999e-05 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906424+0000", 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178889 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.363 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:49833 6.e 6:7268cd15:::benchmark_data_smithi163_287034_object49832:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:26.696825+0000", 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 59.622159150999998, 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.209572658, 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.364 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 49833 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696825+0000", 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696825+0000", 2023-10-09T04:06:26.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696824+0000", 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696860+0000", 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5899999999999998e-05 2023-10-09T04:06:26.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696870+0000", 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6730000000000004e-06 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696875+0000", 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5350000000000004e-06 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696915+0000", 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9725000000000003e-05 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.697042+0000", 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000126513 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.697156+0000", 2023-10-09T04:06:26.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114887 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.699954+0000", 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027972190000000001 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.701695+0000", 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017411029999999999 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906317+0000", 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20462179699999999 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906364+0000", 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7437e-05 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906398+0000", 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4020000000000003e-05 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:49835 6.e 6:7d233238:::benchmark_data_smithi163_287034_object49834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:26.696886+0000", 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 59.622098575000003, 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21014442899999999, 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.371 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 49835 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696886+0000", 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696886+0000", 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696885+0000", 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696927+0000", 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1510000000000001e-05 2023-10-09T04:06:26.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696938+0000", 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1018e-05 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696943+0000", 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0690000000000001e-06 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.698768+0000", 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001825328 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.698874+0000", 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010574200000000001 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.698992+0000", 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118168 2023-10-09T04:06:26.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700016+0000", 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010237969999999999 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.701882+0000", 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001866424 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906967+0000", 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20508501800000001 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.906998+0000", 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1071000000000001e-05 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.907030+0000", 2023-10-09T04:06:26.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2011000000000002e-05 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:49838 6.2 6:4c737c00:::benchmark_data_smithi163_287034_object49837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:26.700063+0000", 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 59.618921600999997, 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20796438, 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.378 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 49838 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700063+0000", 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700063+0000", 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700062+0000", 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700121+0000", 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8569000000000001e-05 2023-10-09T04:06:26.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700135+0000", 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4001e-05 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700139+0000", 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6589999999999998e-06 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700177+0000", 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8127999999999998e-05 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700267+0000", 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9497e-05 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.700360+0000", 2023-10-09T04:06:26.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2651000000000005e-05 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.701693+0000", 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013331230000000001 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.702123+0000", 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00043067700000000002 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.907937+0000", 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20581380199999999 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.907991+0000", 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4348000000000002e-05 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.908027+0000", 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5751999999999999e-05 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:51587 6.2 6:4f72754b:::benchmark_data_smithi163_287034_object51586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:28.213620+0000", 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 58.105364363, 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.145629861, 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.385 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 51587 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213620+0000", 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213620+0000", 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213619+0000", 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213675+0000", 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5581999999999997e-05 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213687+0000", 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1895999999999999e-05 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213692+0000", 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9450000000000001e-06 2023-10-09T04:06:26.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213731+0000", 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9221000000000003e-05 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213828+0000", 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6941000000000006e-05 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.213930+0000", 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102278 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.217204+0000", 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032737360000000002 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.219414+0000", 2023-10-09T04:06:26.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022097950000000001 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.359156+0000", 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13974228599999999 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.359214+0000", 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8437999999999999e-05 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.359250+0000", 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5704000000000002e-05 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:51588 6.2 6:45b1a934:::benchmark_data_smithi163_287034_object51587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:28.214046+0000", 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 58.104938056999998, 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14531429400000001, 2023-10-09T04:06:26.392 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 51588 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.214046+0000", 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.214046+0000", 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.214046+0000", 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.214080+0000", 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4584e-05 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.214097+0000", 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6915999999999999e-05 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.214108+0000", 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0468999999999999e-05 2023-10-09T04:06:26.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.215566+0000", 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014585990000000001 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.215704+0000", 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137589 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.215840+0000", 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013656900000000001 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.217330+0000", 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001489525 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.219541+0000", 2023-10-09T04:06:26.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022108259999999999 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.359282+0000", 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13974093500000001 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.359326+0000", 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4601999999999997e-05 2023-10-09T04:06:26.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:28.359361+0000", 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.447e-05 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:72548 6.2 6:46c52c9d:::benchmark_data_smithi163_287034_object72547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:47.323810+0000", 2023-10-09T04:06:26.399 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 38.995174896999998, 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.19648188599999999, 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72548 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323810+0000", 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323810+0000", 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323809+0000", 2023-10-09T04:06:26.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323869+0000", 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9994000000000002e-05 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323880+0000", 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1333e-05 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323887+0000", 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3380000000000001e-06 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323932+0000", 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5408999999999998e-05 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.324047+0000", 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011438999999999999 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.324163+0000", 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116001 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.328074+0000", 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039111570000000002 2023-10-09T04:06:26.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.337441+0000", 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093677050000000005 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520193+0000", 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18275150400000001 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520248+0000", 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4836999999999997e-05 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520291+0000", 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3723000000000001e-05 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:72549 6.2 6:41d0a08c:::benchmark_data_smithi163_287034_object72548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:47.323896+0000", 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 38.995088852000002, 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.19651426299999999, 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72549 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.407 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323896+0000", 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323896+0000", 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323895+0000", 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323919+0000", 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3813000000000001e-05 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323923+0000", 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2259999999999999e-06 2023-10-09T04:06:26.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323927+0000", 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2390000000000004e-06 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.326012+0000", 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020843860000000001 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.326146+0000", 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013386999999999999 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.326302+0000", 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015593599999999999 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.328207+0000", 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001905124 2023-10-09T04:06:26.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.337674+0000", 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0094675530000000001 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520330+0000", 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18265537300000001 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520376+0000", 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5980999999999999e-05 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520410+0000", 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4146999999999997e-05 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:74843 6.2 6:493f9c78:::benchmark_data_smithi163_287034_object74842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:49.651863+0000", 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 36.667121000000002, 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18964536400000001, 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 74843 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.414 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651863+0000", 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651863+0000", 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651862+0000", 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651908+0000", 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5918999999999999e-05 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651924+0000", 2023-10-09T04:06:26.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.539e-05 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651932+0000", 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9459999999999998e-06 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.651980+0000", 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7970000000000003e-05 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.652087+0000", 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010767799999999999 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.652195+0000", 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108213 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.654786+0000", 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025902479999999999 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.795811+0000", 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.141024819 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.841419+0000", 2023-10-09T04:06:26.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.04560898 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.841464+0000", 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4679999999999999e-05 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.841509+0000", 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4650000000000001e-05 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.420 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:76847 6.2 6:495bac06:::benchmark_data_smithi163_287034_object76846:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:51.783638+0000", 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.535346232999999, 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18217398100000001, 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.421 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 76847 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783638+0000", 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783638+0000", 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783637+0000", 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783735+0000", 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8292e-05 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783750+0000", 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4149e-05 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783756+0000", 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.613e-06 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783819+0000", 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2699000000000004e-05 2023-10-09T04:06:26.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.783922+0000", 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010331399999999999 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.784057+0000", 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013483899999999999 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.786393+0000", 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023362259999999998 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.791897+0000", 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055036709999999999 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.965732+0000", 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17383549700000001 2023-10-09T04:06:26.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.965768+0000", 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5136999999999999e-05 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.965812+0000", 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4663999999999998e-05 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:76862 6.2 6:4d290d90:::benchmark_data_smithi163_287034_object76861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:51.795278+0000", 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.523706904000001, 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.176895738, 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 76862 2023-10-09T04:06:26.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795278+0000", 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795278+0000", 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795276+0000", 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795316+0000", 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9569e-05 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795330+0000", 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4645e-05 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795336+0000", 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9020000000000004e-06 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795386+0000", 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9870000000000002e-05 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795542+0000", 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015644399999999999 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.795704+0000", 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016176799999999999 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.797767+0000", 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020631209999999998 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.797858+0000", 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0402999999999995e-05 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.972063+0000", 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17420564299999999 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.972129+0000", 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6092000000000004e-05 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:51.972173+0000", 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3847000000000002e-05 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:90356 6.2 6:4e29003c:::benchmark_data_smithi163_287034_object90355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:06.993679+0000", 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.325304977999998, 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.17778324100000001, 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 90356 2023-10-09T04:06:26.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993679+0000", 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993679+0000", 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993678+0000", 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993735+0000", 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6459999999999998e-05 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993749+0000", 2023-10-09T04:06:26.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.454e-05 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993755+0000", 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1009999999999997e-06 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993798+0000", 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2469000000000003e-05 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.993920+0000", 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121842 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.994045+0000", 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125258 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.996983+0000", 2023-10-09T04:06:26.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029375360000000001 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:06.997388+0000", 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000405731 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:07.171360+0000", 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.173971552 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:07.171421+0000", 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1639999999999999e-05 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:07.171463+0000", 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1288000000000002e-05 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:91744 6.2 6:4fec3bae:::benchmark_data_smithi163_287034_object91743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:08.748277+0000", 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.570707357, 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12785660700000001, 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 91744 2023-10-09T04:06:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748277+0000", 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748277+0000", 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748276+0000", 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748306+0000", 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9961999999999999e-05 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748318+0000", 2023-10-09T04:06:26.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1426999999999999e-05 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748322+0000", 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5329999999999996e-06 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748380+0000", 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7918999999999999e-05 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748490+0000", 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000110389 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.748588+0000", 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7950999999999998e-05 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.753582+0000", 2023-10-09T04:06:26.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0049937649999999998 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.869096+0000", 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115513506 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.876071+0000", 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069758069999999997 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.876107+0000", 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5422000000000001e-05 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.876134+0000", 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6908999999999999e-05 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:91746 6.2 6:46da87b2:::benchmark_data_smithi163_287034_object91745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:08.749562+0000", 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.569422716999998, 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12671427800000001, 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 91746 2023-10-09T04:06:26.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.749562+0000", 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.749562+0000", 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.749561+0000", 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.749638+0000", 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6942999999999998e-05 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.749653+0000", 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4843e-05 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.749657+0000", 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5329999999999996e-06 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.750887+0000", 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001229791 2023-10-09T04:06:26.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.751015+0000", 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012748300000000001 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.751172+0000", 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015752 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.753698+0000", 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025263859999999998 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.869159+0000", 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115460226 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.876182+0000", 2023-10-09T04:06:26.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0070231160000000003 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.876236+0000", 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4595999999999997e-05 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:08.876276+0000", 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9651999999999999e-05 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.455 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:94429 6.2 6:496efee2:::benchmark_data_smithi163_287034_object94428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:11.953788+0000", 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 14.365196673, 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15854573299999999, 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:26.456 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 94429 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953788+0000", 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953788+0000", 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:26.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953786+0000", 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953836+0000", 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9784999999999999e-05 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953848+0000", 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1651000000000001e-05 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953854+0000", 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5989999999999998e-06 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.953984+0000", 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013020999999999999 2023-10-09T04:06:26.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.954125+0000", 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000140715 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.954269+0000", 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000143545 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.958079+0000", 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003810665 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.964914+0000", 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068342710000000003 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:12.112232+0000", 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14731793800000001 2023-10-09T04:06:26.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:12.112291+0000", 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9506000000000003e-05 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:12.112334+0000", 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2425000000000001e-05 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.462 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.463 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:26.463 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:26.463 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:06:26.698 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:06:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:26 smithi181 ceph-mon[196979]: pgmap v302: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-10-09T04:06:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:26 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:26 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:26.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:26 smithi163 ceph-mon[213841]: pgmap v302: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-10-09T04:06:26.775 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:26 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:26.775 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:26 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:26.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:26 smithi163 ceph-mon[221067]: pgmap v302: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-10-09T04:06:26.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:26 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:26.775 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:26 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:27.025 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 106032 106016 76.1209 48.6875 0.0276732 0.0131324 2023-10-09T04:06:27.055 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:27.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:27.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:06:27.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:27.267 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:27.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:27.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:27.432 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:27 smithi181 ceph-mon[196979]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:27.459 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":302,"stamp":"2023-10-09T04:06:25.313390+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6781736864,"num_objects":103496,"num_object_clones":0,"num_object_copies":310488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103496,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":103772,"num_write_kb":6624653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103688,"ondisk_log_size":103688,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21451432,"kb_used_data":19902120,"kb_used_omap":126,"kb_used_meta":1549121,"kb_avail":728509784,"statfs":{"total":767960285184,"available":745994018816,"internally_reserved":0,"allocated":20379770880,"data_stored":20375755027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1586300539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":457244672,"num_objects":6977,"num_object_clones":0,"num_object_copies":20931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6977,"num_write_kb":446528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009590"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":98,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424155+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:06:12.424155+0000","last_peered":"2023-10-09T04:06:12.424155+0000","last_clean":"2023-10-09T04:06:12.424155+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:06:12.424155+0000","last_undegraded":"2023-10-09T04:06:12.424155+0000","last_fullsized":"2023-10-09T04:06:12.424155+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423922+0000","last_change":"2023-10-09T04:03:37.220097+0000","last_active":"2023-10-09T04:06:12.423922+0000","last_peered":"2023-10-09T04:06:12.423922+0000","last_clean":"2023-10-09T04:06:12.423922+0000","last_became_active":"2023-10-09T04:03:37.219727+0000","last_became_peered":"2023-10-09T04:03:37.219727+0000","last_unstale":"2023-10-09T04:06:12.423922+0000","last_undegraded":"2023-10-09T04:06:12.423922+0000","last_fullsized":"2023-10-09T04:06:12.423922+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:08.596320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.426716+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:06:12.426716+0000","last_peered":"2023-10-09T04:06:12.426716+0000","last_clean":"2023-10-09T04:06:12.426716+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:12.426716+0000","last_undegraded":"2023-10-09T04:06:12.426716+0000","last_fullsized":"2023-10-09T04:06:12.426716+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400599+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:06:02.400599+0000","last_peered":"2023-10-09T04:06:02.400599+0000","last_clean":"2023-10-09T04:06:02.400599+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:06:02.400599+0000","last_undegraded":"2023-10-09T04:06:02.400599+0000","last_fullsized":"2023-10-09T04:06:02.400599+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423227+0000","last_change":"2023-10-09T04:03:37.216503+0000","last_active":"2023-10-09T04:06:12.423227+0000","last_peered":"2023-10-09T04:06:12.423227+0000","last_clean":"2023-10-09T04:06:12.423227+0000","last_became_active":"2023-10-09T04:03:37.215427+0000","last_became_peered":"2023-10-09T04:03:37.215427+0000","last_unstale":"2023-10-09T04:06:12.423227+0000","last_undegraded":"2023-10-09T04:06:12.423227+0000","last_fullsized":"2023-10-09T04:06:12.423227+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:28.919418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.427317+0000","last_change":"2023-10-09T04:03:06.343224+0000","last_active":"2023-10-09T04:06:12.427317+0000","last_peered":"2023-10-09T04:06:12.427317+0000","last_clean":"2023-10-09T04:06:12.427317+0000","last_became_active":"2023-10-09T04:03:06.342869+0000","last_became_peered":"2023-10-09T04:03:06.342869+0000","last_unstale":"2023-10-09T04:06:12.427317+0000","last_undegraded":"2023-10-09T04:06:12.427317+0000","last_fullsized":"2023-10-09T04:06:12.427317+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:50.374931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400842+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:06:02.400842+0000","last_peered":"2023-10-09T04:06:02.400842+0000","last_clean":"2023-10-09T04:06:02.400842+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:06:02.400842+0000","last_undegraded":"2023-10-09T04:06:02.400842+0000","last_fullsized":"2023-10-09T04:06:02.400842+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":97,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423180+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:06:12.423180+0000","last_peered":"2023-10-09T04:06:12.423180+0000","last_clean":"2023-10-09T04:06:12.423180+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:06:12.423180+0000","last_undegraded":"2023-10-09T04:06:12.423180+0000","last_fullsized":"2023-10-09T04:06:12.423180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.432101+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:06:12.432101+0000","last_peered":"2023-10-09T04:06:12.432101+0000","last_clean":"2023-10-09T04:06:12.432101+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:06:12.432101+0000","last_undegraded":"2023-10-09T04:06:12.432101+0000","last_fullsized":"2023-10-09T04:06:12.432101+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423572+0000","last_change":"2023-10-09T04:03:37.221468+0000","last_active":"2023-10-09T04:06:12.423572+0000","last_peered":"2023-10-09T04:06:12.423572+0000","last_clean":"2023-10-09T04:06:12.423572+0000","last_became_active":"2023-10-09T04:03:37.221147+0000","last_became_peered":"2023-10-09T04:03:37.221147+0000","last_unstale":"2023-10-09T04:06:12.423572+0000","last_undegraded":"2023-10-09T04:06:12.423572+0000","last_fullsized":"2023-10-09T04:06:12.423572+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:49.678007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":82,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381602+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:06:02.381602+0000","last_peered":"2023-10-09T04:06:02.381602+0000","last_clean":"2023-10-09T04:06:02.381602+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:06:02.381602+0000","last_undegraded":"2023-10-09T04:06:02.381602+0000","last_fullsized":"2023-10-09T04:06:02.381602+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.426595+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:06:12.426595+0000","last_peered":"2023-10-09T04:06:12.426595+0000","last_clean":"2023-10-09T04:06:12.426595+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:06:12.426595+0000","last_undegraded":"2023-10-09T04:06:12.426595+0000","last_fullsized":"2023-10-09T04:06:12.426595+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424660+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:06:12.424660+0000","last_peered":"2023-10-09T04:06:12.424660+0000","last_clean":"2023-10-09T04:06:12.424660+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:06:12.424660+0000","last_undegraded":"2023-10-09T04:06:12.424660+0000","last_fullsized":"2023-10-09T04:06:12.424660+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":79,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424187+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:06:12.424187+0000","last_peered":"2023-10-09T04:06:12.424187+0000","last_clean":"2023-10-09T04:06:12.424187+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:06:12.424187+0000","last_undegraded":"2023-10-09T04:06:12.424187+0000","last_fullsized":"2023-10-09T04:06:12.424187+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381672+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:06:02.381672+0000","last_peered":"2023-10-09T04:06:02.381672+0000","last_clean":"2023-10-09T04:06:02.381672+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:06:02.381672+0000","last_undegraded":"2023-10-09T04:06:02.381672+0000","last_fullsized":"2023-10-09T04:06:02.381672+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423691+0000","last_change":"2023-10-09T04:03:37.222047+0000","last_active":"2023-10-09T04:06:12.423691+0000","last_peered":"2023-10-09T04:06:12.423691+0000","last_clean":"2023-10-09T04:06:12.423691+0000","last_became_active":"2023-10-09T04:03:37.221813+0000","last_became_peered":"2023-10-09T04:03:37.221813+0000","last_unstale":"2023-10-09T04:06:12.423691+0000","last_undegraded":"2023-10-09T04:06:12.423691+0000","last_fullsized":"2023-10-09T04:06:12.423691+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:38.790540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400430+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:06:02.400430+0000","last_peered":"2023-10-09T04:06:02.400430+0000","last_clean":"2023-10-09T04:06:02.400430+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:06:02.400430+0000","last_undegraded":"2023-10-09T04:06:02.400430+0000","last_fullsized":"2023-10-09T04:06:02.400430+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.432312+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:06:12.432312+0000","last_peered":"2023-10-09T04:06:12.432312+0000","last_clean":"2023-10-09T04:06:12.432312+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:06:12.432312+0000","last_undegraded":"2023-10-09T04:06:12.432312+0000","last_fullsized":"2023-10-09T04:06:12.432312+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381788+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:06:02.381788+0000","last_peered":"2023-10-09T04:06:02.381788+0000","last_clean":"2023-10-09T04:06:02.381788+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:06:02.381788+0000","last_undegraded":"2023-10-09T04:06:02.381788+0000","last_fullsized":"2023-10-09T04:06:02.381788+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400545+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:06:02.400545+0000","last_peered":"2023-10-09T04:06:02.400545+0000","last_clean":"2023-10-09T04:06:02.400545+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:06:02.400545+0000","last_undegraded":"2023-10-09T04:06:02.400545+0000","last_fullsized":"2023-10-09T04:06:02.400545+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.425861+0000","last_change":"2023-10-09T04:02:03.587579+0000","last_active":"2023-10-09T04:06:12.425861+0000","last_peered":"2023-10-09T04:06:12.425861+0000","last_clean":"2023-10-09T04:06:12.425861+0000","last_became_active":"2023-10-09T04:02:03.587014+0000","last_became_peered":"2023-10-09T04:02:03.587014+0000","last_unstale":"2023-10-09T04:06:12.425861+0000","last_undegraded":"2023-10-09T04:06:12.425861+0000","last_fullsized":"2023-10-09T04:06:12.425861+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:59.585822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.425057+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:06:12.425057+0000","last_peered":"2023-10-09T04:06:12.425057+0000","last_clean":"2023-10-09T04:06:12.425057+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:06:12.425057+0000","last_undegraded":"2023-10-09T04:06:12.425057+0000","last_fullsized":"2023-10-09T04:06:12.425057+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"121'6638","reported_seq":6665,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.204844+0000","last_change":"2023-10-09T04:04:51.281106+0000","last_active":"2023-10-09T04:06:25.204844+0000","last_peered":"2023-10-09T04:06:25.204844+0000","last_clean":"2023-10-09T04:06:25.204844+0000","last_became_active":"2023-10-09T04:04:51.280862+0000","last_became_peered":"2023-10-09T04:04:51.280862+0000","last_unstale":"2023-10-09T04:06:25.204844+0000","last_undegraded":"2023-10-09T04:06:25.204844+0000","last_fullsized":"2023-10-09T04:06:25.204844+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6638,"log_dups_size":0,"ondisk_log_size":6638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:47.717190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435027968,"num_objects":6638,"num_object_clones":0,"num_object_copies":19914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6638,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":80,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.384040+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:06:02.384040+0000","last_peered":"2023-10-09T04:06:02.384040+0000","last_clean":"2023-10-09T04:06:02.384040+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:06:02.384040+0000","last_undegraded":"2023-10-09T04:06:02.384040+0000","last_fullsized":"2023-10-09T04:06:02.384040+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":62,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.431683+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:06:12.431683+0000","last_peered":"2023-10-09T04:06:12.431683+0000","last_clean":"2023-10-09T04:06:12.431683+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:06:12.431683+0000","last_undegraded":"2023-10-09T04:06:12.431683+0000","last_fullsized":"2023-10-09T04:06:12.431683+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"121'6525","reported_seq":6552,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.257858+0000","last_change":"2023-10-09T04:04:51.277948+0000","last_active":"2023-10-09T04:06:25.257858+0000","last_peered":"2023-10-09T04:06:25.257858+0000","last_clean":"2023-10-09T04:06:25.257858+0000","last_became_active":"2023-10-09T04:04:51.277674+0000","last_became_peered":"2023-10-09T04:04:51.277674+0000","last_unstale":"2023-10-09T04:06:25.257858+0000","last_undegraded":"2023-10-09T04:06:25.257858+0000","last_fullsized":"2023-10-09T04:06:25.257858+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:12.182469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427622400,"num_objects":6525,"num_object_clones":0,"num_object_copies":19575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6525,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":81,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.383910+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:06:02.383910+0000","last_peered":"2023-10-09T04:06:02.383910+0000","last_clean":"2023-10-09T04:06:02.383910+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:06:02.383910+0000","last_undegraded":"2023-10-09T04:06:02.383910+0000","last_fullsized":"2023-10-09T04:06:02.383910+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423808+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:06:12.423808+0000","last_peered":"2023-10-09T04:06:12.423808+0000","last_clean":"2023-10-09T04:06:12.423808+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:06:12.423808+0000","last_undegraded":"2023-10-09T04:06:12.423808+0000","last_fullsized":"2023-10-09T04:06:12.423808+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"120'6303","reported_seq":6329,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:20.216863+0000","last_change":"2023-10-09T04:04:51.279912+0000","last_active":"2023-10-09T04:06:20.216863+0000","last_peered":"2023-10-09T04:06:20.216863+0000","last_clean":"2023-10-09T04:06:20.216863+0000","last_became_active":"2023-10-09T04:04:51.279633+0000","last_became_peered":"2023-10-09T04:04:51.279633+0000","last_unstale":"2023-10-09T04:06:20.216863+0000","last_undegraded":"2023-10-09T04:06:20.216863+0000","last_fullsized":"2023-10-09T04:06:20.216863+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:48.533141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413073408,"num_objects":6303,"num_object_clones":0,"num_object_copies":18909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6303,"num_write_kb":403392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":63,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381747+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:06:02.381747+0000","last_peered":"2023-10-09T04:06:02.381747+0000","last_clean":"2023-10-09T04:06:02.381747+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:06:02.381747+0000","last_undegraded":"2023-10-09T04:06:02.381747+0000","last_fullsized":"2023-10-09T04:06:02.381747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.426712+0000","last_change":"2023-10-09T04:02:35.094215+0000","last_active":"2023-10-09T04:06:12.426712+0000","last_peered":"2023-10-09T04:06:12.426712+0000","last_clean":"2023-10-09T04:06:12.426712+0000","last_became_active":"2023-10-09T04:02:35.093854+0000","last_became_peered":"2023-10-09T04:02:35.093854+0000","last_unstale":"2023-10-09T04:06:12.426712+0000","last_undegraded":"2023-10-09T04:06:12.426712+0000","last_fullsized":"2023-10-09T04:06:12.426712+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:24.574037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"120'6461","reported_seq":6486,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:21.689028+0000","last_change":"2023-10-09T04:04:51.276847+0000","last_active":"2023-10-09T04:06:21.689028+0000","last_peered":"2023-10-09T04:06:21.689028+0000","last_clean":"2023-10-09T04:06:21.689028+0000","last_became_active":"2023-10-09T04:04:51.276578+0000","last_became_peered":"2023-10-09T04:04:51.276578+0000","last_unstale":"2023-10-09T04:06:21.689028+0000","last_undegraded":"2023-10-09T04:06:21.689028+0000","last_fullsized":"2023-10-09T04:06:21.689028+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:27.626784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423428096,"num_objects":6461,"num_object_clones":0,"num_object_copies":19383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6461,"num_write_kb":413504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.433583+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:06:12.433583+0000","last_peered":"2023-10-09T04:06:12.433583+0000","last_clean":"2023-10-09T04:06:12.433583+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:06:12.433583+0000","last_undegraded":"2023-10-09T04:06:12.433583+0000","last_fullsized":"2023-10-09T04:06:12.433583+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400720+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:06:02.400720+0000","last_peered":"2023-10-09T04:06:02.400720+0000","last_clean":"2023-10-09T04:06:02.400720+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:06:02.400720+0000","last_undegraded":"2023-10-09T04:06:02.400720+0000","last_fullsized":"2023-10-09T04:06:02.400720+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"121'6456","reported_seq":6483,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.223933+0000","last_change":"2023-10-09T04:04:51.279219+0000","last_active":"2023-10-09T04:06:25.223933+0000","last_peered":"2023-10-09T04:06:25.223933+0000","last_clean":"2023-10-09T04:06:25.223933+0000","last_became_active":"2023-10-09T04:04:51.278902+0000","last_became_peered":"2023-10-09T04:04:51.278902+0000","last_unstale":"2023-10-09T04:06:25.223933+0000","last_undegraded":"2023-10-09T04:06:25.223933+0000","last_fullsized":"2023-10-09T04:06:25.223933+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6456,"log_dups_size":0,"ondisk_log_size":6456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:37.687116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423100416,"num_objects":6456,"num_object_clones":0,"num_object_copies":19368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6456,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":81,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423813+0000","last_change":"2023-10-09T04:03:37.220457+0000","last_active":"2023-10-09T04:06:12.423813+0000","last_peered":"2023-10-09T04:06:12.423813+0000","last_clean":"2023-10-09T04:06:12.423813+0000","last_became_active":"2023-10-09T04:03:37.220143+0000","last_became_peered":"2023-10-09T04:03:37.220143+0000","last_unstale":"2023-10-09T04:06:12.423813+0000","last_undegraded":"2023-10-09T04:06:12.423813+0000","last_fullsized":"2023-10-09T04:06:12.423813+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.640160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":79,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424583+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:06:12.424583+0000","last_peered":"2023-10-09T04:06:12.424583+0000","last_clean":"2023-10-09T04:06:12.424583+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:06:12.424583+0000","last_undegraded":"2023-10-09T04:06:12.424583+0000","last_fullsized":"2023-10-09T04:06:12.424583+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"121'6450","reported_seq":6478,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:24.328878+0000","last_change":"2023-10-09T04:04:51.275697+0000","last_active":"2023-10-09T04:06:24.328878+0000","last_peered":"2023-10-09T04:06:24.328878+0000","last_clean":"2023-10-09T04:06:24.328878+0000","last_became_active":"2023-10-09T04:04:51.275402+0000","last_became_peered":"2023-10-09T04:04:51.275402+0000","last_unstale":"2023-10-09T04:06:24.328878+0000","last_undegraded":"2023-10-09T04:06:24.328878+0000","last_fullsized":"2023-10-09T04:06:24.328878+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:26.490300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422707200,"num_objects":6450,"num_object_clones":0,"num_object_copies":19350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6450,"num_write_kb":412800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.426855+0000","last_change":"2023-10-09T04:02:35.094164+0000","last_active":"2023-10-09T04:06:12.426855+0000","last_peered":"2023-10-09T04:06:12.426855+0000","last_clean":"2023-10-09T04:06:12.426855+0000","last_became_active":"2023-10-09T04:02:35.093406+0000","last_became_peered":"2023-10-09T04:02:35.093406+0000","last_unstale":"2023-10-09T04:06:12.426855+0000","last_undegraded":"2023-10-09T04:06:12.426855+0000","last_fullsized":"2023-10-09T04:06:12.426855+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:22.566252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424810+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:06:12.424810+0000","last_peered":"2023-10-09T04:06:12.424810+0000","last_clean":"2023-10-09T04:06:12.424810+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:06:12.424810+0000","last_undegraded":"2023-10-09T04:06:12.424810+0000","last_fullsized":"2023-10-09T04:06:12.424810+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"121'6565","reported_seq":6593,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.262829+0000","last_change":"2023-10-09T04:04:51.279552+0000","last_active":"2023-10-09T04:06:25.262829+0000","last_peered":"2023-10-09T04:06:25.262829+0000","last_clean":"2023-10-09T04:06:25.262829+0000","last_became_active":"2023-10-09T04:04:51.279242+0000","last_became_peered":"2023-10-09T04:04:51.279242+0000","last_unstale":"2023-10-09T04:06:25.262829+0000","last_undegraded":"2023-10-09T04:06:25.262829+0000","last_fullsized":"2023-10-09T04:06:25.262829+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:44.784925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430243840,"num_objects":6565,"num_object_clones":0,"num_object_copies":19695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6565,"num_write_kb":420160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423671+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:06:12.423671+0000","last_peered":"2023-10-09T04:06:12.423671+0000","last_clean":"2023-10-09T04:06:12.423671+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:06:12.423671+0000","last_undegraded":"2023-10-09T04:06:12.423671+0000","last_fullsized":"2023-10-09T04:06:12.423671+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424542+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:06:12.424542+0000","last_peered":"2023-10-09T04:06:12.424542+0000","last_clean":"2023-10-09T04:06:12.424542+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:06:12.424542+0000","last_undegraded":"2023-10-09T04:06:12.424542+0000","last_fullsized":"2023-10-09T04:06:12.424542+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"121'6610","reported_seq":6637,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.221200+0000","last_change":"2023-10-09T04:04:51.279867+0000","last_active":"2023-10-09T04:06:25.221200+0000","last_peered":"2023-10-09T04:06:25.221200+0000","last_clean":"2023-10-09T04:06:25.221200+0000","last_became_active":"2023-10-09T04:04:51.279588+0000","last_became_peered":"2023-10-09T04:04:51.279588+0000","last_unstale":"2023-10-09T04:06:25.221200+0000","last_undegraded":"2023-10-09T04:06:25.221200+0000","last_fullsized":"2023-10-09T04:06:25.221200+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.265205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433192960,"num_objects":6610,"num_object_clones":0,"num_object_copies":19830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6610,"num_write_kb":423040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423307+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:06:12.423307+0000","last_peered":"2023-10-09T04:06:12.423307+0000","last_clean":"2023-10-09T04:06:12.423307+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:06:12.423307+0000","last_undegraded":"2023-10-09T04:06:12.423307+0000","last_fullsized":"2023-10-09T04:06:12.423307+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":70,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.430948+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:06:12.430948+0000","last_peered":"2023-10-09T04:06:12.430948+0000","last_clean":"2023-10-09T04:06:12.430948+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:06:12.430948+0000","last_undegraded":"2023-10-09T04:06:12.430948+0000","last_fullsized":"2023-10-09T04:06:12.430948+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.427284+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:06:12.427284+0000","last_peered":"2023-10-09T04:06:12.427284+0000","last_clean":"2023-10-09T04:06:12.427284+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:06:12.427284+0000","last_undegraded":"2023-10-09T04:06:12.427284+0000","last_fullsized":"2023-10-09T04:06:12.427284+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.432381+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:06:12.432381+0000","last_peered":"2023-10-09T04:06:12.432381+0000","last_clean":"2023-10-09T04:06:12.432381+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:12.432381+0000","last_undegraded":"2023-10-09T04:06:12.432381+0000","last_fullsized":"2023-10-09T04:06:12.432381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.426964+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:06:12.426964+0000","last_peered":"2023-10-09T04:06:12.426964+0000","last_clean":"2023-10-09T04:06:12.426964+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:06:12.426964+0000","last_undegraded":"2023-10-09T04:06:12.426964+0000","last_fullsized":"2023-10-09T04:06:12.426964+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"120'6318","reported_seq":6345,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:21.639069+0000","last_change":"2023-10-09T04:04:51.276404+0000","last_active":"2023-10-09T04:06:21.639069+0000","last_peered":"2023-10-09T04:06:21.639069+0000","last_clean":"2023-10-09T04:06:21.639069+0000","last_became_active":"2023-10-09T04:04:51.276109+0000","last_became_peered":"2023-10-09T04:04:51.276109+0000","last_unstale":"2023-10-09T04:06:21.639069+0000","last_undegraded":"2023-10-09T04:06:21.639069+0000","last_fullsized":"2023-10-09T04:06:21.639069+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6318,"log_dups_size":0,"ondisk_log_size":6318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:04.681582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414056448,"num_objects":6318,"num_object_clones":0,"num_object_copies":18954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6318,"num_write_kb":404352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"121'6635","reported_seq":6663,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.227691+0000","last_change":"2023-10-09T04:04:51.277034+0000","last_active":"2023-10-09T04:06:25.227691+0000","last_peered":"2023-10-09T04:06:25.227691+0000","last_clean":"2023-10-09T04:06:25.227691+0000","last_became_active":"2023-10-09T04:04:51.276751+0000","last_became_peered":"2023-10-09T04:04:51.276751+0000","last_unstale":"2023-10-09T04:06:25.227691+0000","last_undegraded":"2023-10-09T04:06:25.227691+0000","last_fullsized":"2023-10-09T04:06:25.227691+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6635,"log_dups_size":0,"ondisk_log_size":6635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:13.233763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434831360,"num_objects":6635,"num_object_clones":0,"num_object_copies":19905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6635,"num_write_kb":424640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":84,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.431844+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:06:12.431844+0000","last_peered":"2023-10-09T04:06:12.431844+0000","last_clean":"2023-10-09T04:06:12.431844+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:06:12.431844+0000","last_undegraded":"2023-10-09T04:06:12.431844+0000","last_fullsized":"2023-10-09T04:06:12.431844+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424264+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:06:12.424264+0000","last_peered":"2023-10-09T04:06:12.424264+0000","last_clean":"2023-10-09T04:06:12.424264+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:06:12.424264+0000","last_undegraded":"2023-10-09T04:06:12.424264+0000","last_fullsized":"2023-10-09T04:06:12.424264+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.427232+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:06:12.427232+0000","last_peered":"2023-10-09T04:06:12.427232+0000","last_clean":"2023-10-09T04:06:12.427232+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:06:12.427232+0000","last_undegraded":"2023-10-09T04:06:12.427232+0000","last_fullsized":"2023-10-09T04:06:12.427232+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"120'6267","reported_seq":6293,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:20.157101+0000","last_change":"2023-10-09T04:04:51.279627+0000","last_active":"2023-10-09T04:06:20.157101+0000","last_peered":"2023-10-09T04:06:20.157101+0000","last_clean":"2023-10-09T04:06:20.157101+0000","last_became_active":"2023-10-09T04:04:51.279326+0000","last_became_peered":"2023-10-09T04:04:51.279326+0000","last_unstale":"2023-10-09T04:06:20.157101+0000","last_undegraded":"2023-10-09T04:06:20.157101+0000","last_fullsized":"2023-10-09T04:06:20.157101+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6267,"log_dups_size":0,"ondisk_log_size":6267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:32:58.605251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410714112,"num_objects":6267,"num_object_clones":0,"num_object_copies":18801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6267,"num_write_kb":401088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":103,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.415611+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:06:12.415611+0000","last_peered":"2023-10-09T04:06:12.415611+0000","last_clean":"2023-10-09T04:06:12.415611+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:06:12.415611+0000","last_undegraded":"2023-10-09T04:06:12.415611+0000","last_fullsized":"2023-10-09T04:06:12.415611+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.426798+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:06:12.426798+0000","last_peered":"2023-10-09T04:06:12.426798+0000","last_clean":"2023-10-09T04:06:12.426798+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:06:12.426798+0000","last_undegraded":"2023-10-09T04:06:12.426798+0000","last_fullsized":"2023-10-09T04:06:12.426798+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.431028+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:06:12.431028+0000","last_peered":"2023-10-09T04:06:12.431028+0000","last_clean":"2023-10-09T04:06:12.431028+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:06:12.431028+0000","last_undegraded":"2023-10-09T04:06:12.431028+0000","last_fullsized":"2023-10-09T04:06:12.431028+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"121'6437","reported_seq":6464,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.141947+0000","last_change":"2023-10-09T04:04:51.280419+0000","last_active":"2023-10-09T04:06:25.141947+0000","last_peered":"2023-10-09T04:06:25.141947+0000","last_clean":"2023-10-09T04:06:25.141947+0000","last_became_active":"2023-10-09T04:04:51.280174+0000","last_became_peered":"2023-10-09T04:04:51.280174+0000","last_unstale":"2023-10-09T04:06:25.141947+0000","last_undegraded":"2023-10-09T04:06:25.141947+0000","last_fullsized":"2023-10-09T04:06:25.141947+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:03.946653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421855232,"num_objects":6437,"num_object_clones":0,"num_object_copies":19311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6437,"num_write_kb":411968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":84,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.431923+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:06:12.431923+0000","last_peered":"2023-10-09T04:06:12.431923+0000","last_clean":"2023-10-09T04:06:12.431923+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:06:12.431923+0000","last_undegraded":"2023-10-09T04:06:12.431923+0000","last_fullsized":"2023-10-09T04:06:12.431923+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":460,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381147+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:06:02.381147+0000","last_peered":"2023-10-09T04:06:02.381147+0000","last_clean":"2023-10-09T04:06:02.381147+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:06:02.381147+0000","last_undegraded":"2023-10-09T04:06:02.381147+0000","last_fullsized":"2023-10-09T04:06:02.381147+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423429+0000","last_change":"2023-10-09T04:03:37.221180+0000","last_active":"2023-10-09T04:06:12.423429+0000","last_peered":"2023-10-09T04:06:12.423429+0000","last_clean":"2023-10-09T04:06:12.423429+0000","last_became_active":"2023-10-09T04:03:37.220818+0000","last_became_peered":"2023-10-09T04:03:37.220818+0000","last_unstale":"2023-10-09T04:06:12.423429+0000","last_undegraded":"2023-10-09T04:06:12.423429+0000","last_fullsized":"2023-10-09T04:06:12.423429+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:52.056668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":62,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.430514+0000","last_change":"2023-10-09T03:54:13.388453+0000","last_active":"2023-10-09T04:06:12.430514+0000","last_peered":"2023-10-09T04:06:12.430514+0000","last_clean":"2023-10-09T04:06:12.430514+0000","last_became_active":"2023-10-09T03:54:13.388209+0000","last_became_peered":"2023-10-09T03:54:13.388209+0000","last_unstale":"2023-10-09T04:06:12.430514+0000","last_undegraded":"2023-10-09T04:06:12.430514+0000","last_fullsized":"2023-10-09T04:06:12.430514+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:29.906563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":100,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381160+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:06:02.381160+0000","last_peered":"2023-10-09T04:06:02.381160+0000","last_clean":"2023-10-09T04:06:02.381160+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:06:02.381160+0000","last_undegraded":"2023-10-09T04:06:02.381160+0000","last_fullsized":"2023-10-09T04:06:02.381160+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:06.698041+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:06:06.698041+0000","last_peered":"2023-10-09T04:06:06.698041+0000","last_clean":"2023-10-09T04:06:06.698041+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:06:06.698041+0000","last_undegraded":"2023-10-09T04:06:06.698041+0000","last_fullsized":"2023-10-09T04:06:06.698041+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.417174+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:06:12.417174+0000","last_peered":"2023-10-09T04:06:12.417174+0000","last_clean":"2023-10-09T04:06:12.417174+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:06:12.417174+0000","last_undegraded":"2023-10-09T04:06:12.417174+0000","last_fullsized":"2023-10-09T04:06:12.417174+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"121'6502","reported_seq":6530,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:23.122540+0000","last_change":"2023-10-09T04:04:51.279899+0000","last_active":"2023-10-09T04:06:23.122540+0000","last_peered":"2023-10-09T04:06:23.122540+0000","last_clean":"2023-10-09T04:06:23.122540+0000","last_became_active":"2023-10-09T04:04:51.279611+0000","last_became_peered":"2023-10-09T04:04:51.279611+0000","last_unstale":"2023-10-09T04:06:23.122540+0000","last_undegraded":"2023-10-09T04:06:23.122540+0000","last_fullsized":"2023-10-09T04:06:23.122540+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6502,"log_dups_size":0,"ondisk_log_size":6502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:31.582709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426115072,"num_objects":6502,"num_object_clones":0,"num_object_copies":19506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":416128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.381671+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:06:02.381671+0000","last_peered":"2023-10-09T04:06:02.381671+0000","last_clean":"2023-10-09T04:06:02.381671+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:06:02.381671+0000","last_undegraded":"2023-10-09T04:06:02.381671+0000","last_fullsized":"2023-10-09T04:06:02.381671+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.427373+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:06:12.427373+0000","last_peered":"2023-10-09T04:06:12.427373+0000","last_clean":"2023-10-09T04:06:12.427373+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:06:12.427373+0000","last_undegraded":"2023-10-09T04:06:12.427373+0000","last_fullsized":"2023-10-09T04:06:12.427373+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:11.707629+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:06:11.707629+0000","last_peered":"2023-10-09T04:06:11.707629+0000","last_clean":"2023-10-09T04:06:11.707629+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:06:11.707629+0000","last_undegraded":"2023-10-09T04:06:11.707629+0000","last_fullsized":"2023-10-09T04:06:11.707629+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"121'6393","reported_seq":6421,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:23.108350+0000","last_change":"2023-10-09T04:04:51.278104+0000","last_active":"2023-10-09T04:06:23.108350+0000","last_peered":"2023-10-09T04:06:23.108350+0000","last_clean":"2023-10-09T04:06:23.108350+0000","last_became_active":"2023-10-09T04:04:51.277849+0000","last_became_peered":"2023-10-09T04:04:51.277849+0000","last_unstale":"2023-10-09T04:06:23.108350+0000","last_undegraded":"2023-10-09T04:06:23.108350+0000","last_fullsized":"2023-10-09T04:06:23.108350+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.049817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418971648,"num_objects":6393,"num_object_clones":0,"num_object_copies":19179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6393,"num_write_kb":409152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":81,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.425134+0000","last_change":"2023-10-09T04:03:37.217253+0000","last_active":"2023-10-09T04:06:12.425134+0000","last_peered":"2023-10-09T04:06:12.425134+0000","last_clean":"2023-10-09T04:06:12.425134+0000","last_became_active":"2023-10-09T04:03:37.216951+0000","last_became_peered":"2023-10-09T04:03:37.216951+0000","last_unstale":"2023-10-09T04:06:12.425134+0000","last_undegraded":"2023-10-09T04:06:12.425134+0000","last_fullsized":"2023-10-09T04:06:12.425134+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:41.608600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":54,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.431435+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:06:12.431435+0000","last_peered":"2023-10-09T04:06:12.431435+0000","last_clean":"2023-10-09T04:06:12.431435+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:06:12.431435+0000","last_undegraded":"2023-10-09T04:06:12.431435+0000","last_fullsized":"2023-10-09T04:06:12.431435+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":78,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423263+0000","last_change":"2023-10-09T04:03:37.221060+0000","last_active":"2023-10-09T04:06:12.423263+0000","last_peered":"2023-10-09T04:06:12.423263+0000","last_clean":"2023-10-09T04:06:12.423263+0000","last_became_active":"2023-10-09T04:03:37.220009+0000","last_became_peered":"2023-10-09T04:03:37.220009+0000","last_unstale":"2023-10-09T04:06:12.423263+0000","last_undegraded":"2023-10-09T04:06:12.423263+0000","last_fullsized":"2023-10-09T04:06:12.423263+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:19.662106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.2","version":"121'6558","reported_seq":6585,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T04:06:25.203969+0000","last_change":"2023-10-09T04:04:51.275204+0000","last_active":"2023-10-09T04:06:25.203969+0000","last_peered":"2023-10-09T04:06:25.203969+0000","last_clean":"2023-10-09T04:06:25.203969+0000","last_became_active":"2023-10-09T04:04:51.274910+0000","last_became_peered":"2023-10-09T04:04:51.274910+0000","last_unstale":"2023-10-09T04:06:25.203969+0000","last_undegraded":"2023-10-09T04:06:25.203969+0000","last_fullsized":"2023-10-09T04:06:25.203969+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:08.996248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429785088,"num_objects":6558,"num_object_clones":0,"num_object_copies":19674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6558,"num_write_kb":419712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":84,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.427414+0000","last_change":"2023-10-09T04:03:06.342064+0000","last_active":"2023-10-09T04:06:12.427414+0000","last_peered":"2023-10-09T04:06:12.427414+0000","last_clean":"2023-10-09T04:06:12.427414+0000","last_became_active":"2023-10-09T04:03:06.341694+0000","last_became_peered":"2023-10-09T04:03:06.341694+0000","last_unstale":"2023-10-09T04:06:12.427414+0000","last_undegraded":"2023-10-09T04:06:12.427414+0000","last_fullsized":"2023-10-09T04:06:12.427414+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:51.920244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":77,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.417168+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:06:12.417168+0000","last_peered":"2023-10-09T04:06:12.417168+0000","last_clean":"2023-10-09T04:06:12.417168+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:06:12.417168+0000","last_undegraded":"2023-10-09T04:06:12.417168+0000","last_fullsized":"2023-10-09T04:06:12.417168+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":62,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423674+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:06:12.423674+0000","last_peered":"2023-10-09T04:06:12.423674+0000","last_clean":"2023-10-09T04:06:12.423674+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:06:12.423674+0000","last_undegraded":"2023-10-09T04:06:12.423674+0000","last_fullsized":"2023-10-09T04:06:12.423674+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"120'6354","reported_seq":6380,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:20.275560+0000","last_change":"2023-10-09T04:04:51.277953+0000","last_active":"2023-10-09T04:06:20.275560+0000","last_peered":"2023-10-09T04:06:20.275560+0000","last_clean":"2023-10-09T04:06:20.275560+0000","last_became_active":"2023-10-09T04:04:51.277659+0000","last_became_peered":"2023-10-09T04:04:51.277659+0000","last_unstale":"2023-10-09T04:06:20.275560+0000","last_undegraded":"2023-10-09T04:06:20.275560+0000","last_fullsized":"2023-10-09T04:06:20.275560+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":6354,"log_dups_size":0,"ondisk_log_size":6354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:22.890014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416415744,"num_objects":6354,"num_object_clones":0,"num_object_copies":19062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6354,"num_write_kb":406656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":61,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:02.400803+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:06:02.400803+0000","last_peered":"2023-10-09T04:06:02.400803+0000","last_clean":"2023-10-09T04:06:02.400803+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:06:02.400803+0000","last_undegraded":"2023-10-09T04:06:02.400803+0000","last_fullsized":"2023-10-09T04:06:02.400803+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.415504+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:06:12.415504+0000","last_peered":"2023-10-09T04:06:12.415504+0000","last_clean":"2023-10-09T04:06:12.415504+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:06:12.415504+0000","last_undegraded":"2023-10-09T04:06:12.415504+0000","last_fullsized":"2023-10-09T04:06:12.415504+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":82,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.433732+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:06:12.433732+0000","last_peered":"2023-10-09T04:06:12.433732+0000","last_clean":"2023-10-09T04:06:12.433732+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:06:12.433732+0000","last_undegraded":"2023-10-09T04:06:12.433732+0000","last_fullsized":"2023-10-09T04:06:12.433732+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424339+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:06:12.424339+0000","last_peered":"2023-10-09T04:06:12.424339+0000","last_clean":"2023-10-09T04:06:12.424339+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:06:12.424339+0000","last_undegraded":"2023-10-09T04:06:12.424339+0000","last_fullsized":"2023-10-09T04:06:12.424339+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":65,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.432475+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:06:12.432475+0000","last_peered":"2023-10-09T04:06:12.432475+0000","last_clean":"2023-10-09T04:06:12.432475+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:06:12.432475+0000","last_undegraded":"2023-10-09T04:06:12.432475+0000","last_fullsized":"2023-10-09T04:06:12.432475+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.424369+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:06:12.424369+0000","last_peered":"2023-10-09T04:06:12.424369+0000","last_clean":"2023-10-09T04:06:12.424369+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:06:12.424369+0000","last_undegraded":"2023-10-09T04:06:12.424369+0000","last_fullsized":"2023-10-09T04:06:12.424369+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":89,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423603+0000","last_change":"2023-10-09T04:03:37.221959+0000","last_active":"2023-10-09T04:06:12.423603+0000","last_peered":"2023-10-09T04:06:12.423603+0000","last_clean":"2023-10-09T04:06:12.423603+0000","last_became_active":"2023-10-09T04:03:37.221631+0000","last_became_peered":"2023-10-09T04:03:37.221631+0000","last_unstale":"2023-10-09T04:06:12.423603+0000","last_undegraded":"2023-10-09T04:06:12.423603+0000","last_fullsized":"2023-10-09T04:06:12.423603+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:43.026948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.423647+0000","last_change":"2023-10-09T04:03:37.215833+0000","last_active":"2023-10-09T04:06:12.423647+0000","last_peered":"2023-10-09T04:06:12.423647+0000","last_clean":"2023-10-09T04:06:12.423647+0000","last_became_active":"2023-10-09T04:03:37.215224+0000","last_became_peered":"2023-10-09T04:03:37.215224+0000","last_unstale":"2023-10-09T04:06:12.423647+0000","last_undegraded":"2023-10-09T04:06:12.423647+0000","last_fullsized":"2023-10-09T04:06:12.423647+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:45.054041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":77,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-09T04:06:06.698149+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:06:06.698149+0000","last_peered":"2023-10-09T04:06:06.698149+0000","last_clean":"2023-10-09T04:06:06.698149+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:06:06.698149+0000","last_undegraded":"2023-10-09T04:06:06.698149+0000","last_fullsized":"2023-10-09T04:06:06.698149+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T04:06:12.422563+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:06:12.422563+0000","last_peered":"2023-10-09T04:06:12.422563+0000","last_clean":"2023-10-09T04:06:12.422563+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:06:12.422563+0000","last_undegraded":"2023-10-09T04:06:12.422563+0000","last_fullsized":"2023-10-09T04:06:12.422563+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6781140992,"num_objects":103472,"num_object_clones":0,"num_object_copies":310416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":103472,"num_write_kb":6622208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20409417728,"data_stored":20409417728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103472,"ondisk_log_size":103472,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201446,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2741784,"kb_used_data":2428112,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":91003368,"statfs":{"total":95995035648,"available":93187448832,"internally_reserved":0,"allocated":2486386688,"data_stored":2485898477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":0.69499999999999995,"15min":0.57499999999999996},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"last":0.222},{"interface":"front","average":{"1min":1.389,"5min":0.69199999999999995,"15min":0.57599999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.1479999999999997,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":0.25600000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.61599999999999999,"15min":0.624},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.5950000000000002,"5min":2.5950000000000002,"15min":2.5950000000000002},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.72599999999999998,"15min":0.70999999999999996},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.964,"5min":3.363,"15min":3.363},"last":2.1059999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":4.7030000000000003,"5min":1.3779999999999999,"15min":0.82399999999999995},"min":{"1min":0.32500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":58.615000000000002,"5min":58.615000000000002,"15min":58.615000000000002},"last":1.6020000000000001},{"interface":"front","average":{"1min":5.1870000000000003,"5min":1.52,"15min":0.91000000000000003},"min":{"1min":0.38200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":58.798000000000002,"5min":58.798000000000002,"15min":58.798000000000002},"last":1.5640000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":1.008,"15min":0.72199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":31.170999999999999,"5min":31.170999999999999,"15min":31.170999999999999},"last":2.5750000000000002},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.071,"15min":0.76700000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":31.184000000000001,"5min":31.184000000000001,"15min":31.184000000000001},"last":2.5880000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.2650000000000001,"5min":2.2650000000000001,"15min":6.3129999999999997},"last":0.318},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.61499999999999999,"15min":0.55600000000000005},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":2.3180000000000001,"5min":2.3180000000000001,"15min":3.8769999999999998},"last":0.309}]},{"osd":5,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53400000000000003,"15min":0.52400000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.4059999999999999,"5min":1.4330000000000001,"15min":1.464},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.63800000000000001,"15min":0.57999999999999996},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":5.1399999999999997,"5min":5.1399999999999997,"15min":5.1399999999999997},"last":0.65100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.59399999999999997,"15min":0.55500000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.1030000000000002,"5min":2.1030000000000002,"15min":2.1030000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.63500000000000001,"15min":0.59799999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":3.8010000000000002},"last":4.1749999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463084,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2405968,"kb_used_data":2100488,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91339184,"statfs":{"total":95995035648,"available":93531324416,"internally_reserved":0,"allocated":2150899712,"data_stored":2150427322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.70499999999999996,"15min":0.59099999999999997},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":14.557,"5min":14.557,"15min":14.557},"last":0.313},{"interface":"front","average":{"1min":2.0960000000000001,"5min":0.84899999999999998,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":14.632999999999999},"last":0.29899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":0.627,"15min":0.55700000000000005},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.21,"5min":6.21,"15min":6.21},"last":0.251},{"interface":"front","average":{"1min":1.464,"5min":0.70099999999999996,"15min":0.57199999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":12.616,"5min":12.616,"15min":12.616},"last":0.46999999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":1.125,"15min":0.748},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":42.195999999999998,"5min":42.195999999999998,"15min":42.195999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":3.4180000000000001,"5min":1.177,"15min":0.80100000000000005},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":42.165999999999997,"5min":42.165999999999997,"15min":42.165999999999997},"last":0.437}]},{"osd":3,"last update":"Mon Oct 9 04:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9590000000000001,"5min":1.0840000000000001,"15min":0.77200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":38.606999999999999,"5min":38.606999999999999,"15min":38.606999999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":2.6680000000000001,"5min":1.0569999999999999,"15min":0.78900000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":37.914000000000001,"5min":37.914000000000001,"15min":37.914000000000001},"last":0.39500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48799999999999999,"15min":0.46500000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":1.77},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.53400000000000003,"15min":0.501},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.254},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":2.6589999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.55400000000000005,"15min":0.51700000000000002},"min":{"1min":0.33200000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":5.9710000000000001},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.61499999999999999,"15min":0.57099999999999995},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":2.1139999999999999,"5min":2.1139999999999999,"15min":2.1139999999999999},"last":5.3620000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59999999999999998,"15min":0.57599999999999996},"min":{"1min":0.34799999999999998,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":2.238,"5min":2.238,"15min":2.238},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.65700000000000003,"15min":0.61599999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.2480000000000002,"5min":2.2480000000000002,"15min":2.2480000000000002},"last":0.32800000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089392,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128336,"kb_used_data":2060680,"kb_used_omap":9,"kb_used_meta":67638,"kb_avail":91616816,"statfs":{"total":95995035648,"available":93815619584,"internally_reserved":0,"allocated":2110136320,"data_stored":2109605715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":69262313},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":0.60099999999999998,"15min":0.44800000000000001},"min":{"1min":0.28399999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.3600000000000003,"5min":6.3600000000000003,"15min":6.3600000000000003},"last":9.3550000000000004},{"interface":"front","average":{"1min":1.6679999999999999,"5min":0.64100000000000001,"15min":0.47499999999999998},"min":{"1min":0.307,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.976,"5min":7.976,"15min":7.976},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":0.621,"15min":0.436},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.383,"5min":6.383,"15min":6.383},"last":9.1590000000000007},{"interface":"front","average":{"1min":3.194,"5min":0.94899999999999995,"15min":0.56100000000000005},"min":{"1min":0.33700000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.833,"5min":10.833,"15min":10.833},"last":22.321000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":0.66900000000000004,"15min":0.47299999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.3380000000000001,"5min":6.3380000000000001,"15min":6.3380000000000001},"last":9.6859999999999999},{"interface":"front","average":{"1min":1.5880000000000001,"5min":0.624,"15min":0.46300000000000002},"min":{"1min":0.38900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.8630000000000004,"5min":7.8630000000000004,"15min":7.8630000000000004},"last":0.42099999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":0.82599999999999996,"15min":0.73699999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.9089999999999998,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.3169999999999999,"5min":0.80400000000000005,"15min":0.73599999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":4.008,"5min":4.008,"15min":4.008},"last":1.111}]},{"osd":5,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":0.84999999999999998,"15min":0.76600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":7.9459999999999997,"5min":7.9459999999999997,"15min":7.9459999999999997},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.371,"5min":0.82799999999999996,"15min":0.76000000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":4.0759999999999996,"5min":4.0759999999999996,"15min":4.0759999999999996},"last":1.075}]},{"osd":6,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":0.83999999999999997,"15min":0.75600000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.5780000000000001,"5min":0.878,"15min":0.78600000000000003},"min":{"1min":0.441,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":6.3140000000000001,"5min":6.3140000000000001,"15min":6.3140000000000001},"last":9.3160000000000007}]},{"osd":7,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":0.86799999999999999,"15min":0.78600000000000003},"min":{"1min":0.435,"5min":0.38,"15min":0.38},"max":{"1min":4.0800000000000001,"5min":4.0800000000000001,"15min":4.0800000000000001},"last":1.157},{"interface":"front","average":{"1min":1.611,"5min":0.90900000000000003,"15min":0.81599999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.274,"5min":6.274,"15min":6.274},"last":9.2759999999999998}]}]},{"osd":0,"up_from":84,"seq":360777252918,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1690324,"kb_used_data":1640652,"kb_used_omap":28,"kb_used_meta":49635,"kb_avail":92054828,"statfs":{"total":95995035648,"available":94264143872,"internally_reserved":0,"allocated":1680027648,"data_stored":1679483945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":50827088},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.52400000000000002,"15min":0.42399999999999999},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":7.5659999999999998,"5min":7.5659999999999998,"15min":7.5659999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.53500000000000003,"15min":0.42099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":7.5140000000000002,"5min":7.5140000000000002,"15min":7.5140000000000002},"last":1.3300000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":0.81000000000000005,"15min":0.54800000000000004},"min":{"1min":0.255,"5min":0.224,"15min":0.224},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":1.5449999999999999},{"interface":"front","average":{"1min":2.1480000000000001,"5min":0.77000000000000002,"15min":0.53600000000000003},"min":{"1min":0.23499999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":13.461}]},{"osd":3,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3449999999999998,"5min":1.5820000000000001,"15min":0.78800000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":53.387,"5min":53.387,"15min":53.387},"last":0.59699999999999998},{"interface":"front","average":{"1min":6.931,"5min":1.7390000000000001,"15min":0.874},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":54.411000000000001,"5min":54.411000000000001,"15min":54.411000000000001},"last":1.1180000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.149,"5min":1.9350000000000001,"15min":1.0369999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":118.633,"5min":118.633,"15min":118.633},"last":1.196},{"interface":"front","average":{"1min":8.0259999999999998,"5min":2.0950000000000002,"15min":1.1040000000000001},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":118.913,"5min":118.913,"15min":118.913},"last":4.7519999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.9290000000000003,"5min":2.0790000000000002,"15min":1.109},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":118.943,"5min":118.943,"15min":118.943},"last":4.8129999999999997},{"interface":"front","average":{"1min":7.4009999999999998,"5min":1.9910000000000001,"15min":1.089},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":118.898,"5min":118.898,"15min":118.898},"last":0.55000000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.3680000000000003,"5min":1.9970000000000001,"15min":1.1060000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":118.864,"5min":118.864,"15min":118.864},"last":0.52500000000000002},{"interface":"front","average":{"1min":7.1779999999999999,"5min":1.9750000000000001,"15min":1.089},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":118.70399999999999,"5min":118.70399999999999,"15min":118.70399999999999},"last":1.1559999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.9400000000000004,"5min":2.1120000000000001,"15min":1.1379999999999999},"min":{"1min":0.41799999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":118.834,"5min":118.834,"15min":118.834},"last":0.57299999999999995},{"interface":"front","average":{"1min":7.5739999999999998,"5min":2.077,"15min":1.137},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":118.66200000000001,"5min":118.66200000000001,"15min":118.66200000000001},"last":1.179}]}]},{"osd":2,"up_from":94,"seq":403726925866,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3402308,"kb_used_data":3316476,"kb_used_omap":81,"kb_used_meta":85742,"kb_avail":90342844,"statfs":{"total":95995035648,"available":92511072256,"internally_reserved":0,"allocated":3396071424,"data_stored":3395550486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":87800026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":0.63400000000000001,"15min":0.436},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.5700000000000003,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":0.67400000000000004},{"interface":"front","average":{"1min":2.9199999999999999,"5min":0.89900000000000002,"15min":0.56100000000000005},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.619,"5min":12.619,"15min":12.619},"last":0.374}]},{"osd":1,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":5.492,"5min":1.419,"15min":0.72299999999999998},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":68.009,"5min":68.009,"15min":68.009},"last":0.51300000000000001},{"interface":"front","average":{"1min":5.9630000000000001,"5min":1.518,"15min":0.75},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":66.975999999999999,"5min":66.975999999999999,"15min":66.975999999999999},"last":6.181}]},{"osd":3,"last update":"Mon Oct 9 04:05:38 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":0.86199999999999999,"15min":0.54200000000000004},"min":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":27.562000000000001,"5min":27.562000000000001,"15min":27.562000000000001},"last":2.8079999999999998},{"interface":"front","average":{"1min":2.4390000000000001,"5min":0.79100000000000004,"15min":0.51700000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":27.710999999999999,"5min":27.710999999999999,"15min":27.710999999999999},"last":1.7370000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":0.97199999999999998,"15min":0.71999999999999997},"min":{"1min":0.41799999999999998,"5min":0.375,"15min":0.375},"max":{"1min":10.553000000000001,"5min":10.553000000000001,"15min":10.553000000000001},"last":6.024},{"interface":"front","average":{"1min":1.377,"5min":0.75900000000000001,"15min":0.65000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.7930000000000001,"5min":3.7930000000000001,"15min":3.7930000000000001},"last":2.2570000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":0.86299999999999999,"15min":0.72399999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.9029999999999996,"5min":5.9029999999999996,"15min":5.9029999999999996},"last":0.439},{"interface":"front","average":{"1min":2.4409999999999998,"5min":0.99299999999999999,"15min":0.76100000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":10.507999999999999,"5min":10.507999999999999,"15min":10.507999999999999},"last":6.0810000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":0.79700000000000004,"15min":0.68500000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":3.746,"5min":3.746,"15min":3.746},"last":1.3160000000000001},{"interface":"front","average":{"1min":1.8129999999999999,"5min":0.90700000000000003,"15min":0.755},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.8780000000000001,"5min":5.8780000000000001,"15min":5.8780000000000001},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":0.82799999999999996,"15min":0.72199999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.714,"5min":3.714,"15min":3.714},"last":1.248},{"interface":"front","average":{"1min":1.873,"5min":0.94299999999999995,"15min":0.78800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.8300000000000001,"5min":5.8300000000000001,"15min":5.8300000000000001},"last":1.258}]}]},{"osd":3,"up_from":99,"seq":425201762340,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4291648,"kb_used_data":4179704,"kb_used_omap":7,"kb_used_meta":111928,"kb_avail":89453504,"statfs":{"total":95995035648,"available":91600388096,"internally_reserved":0,"allocated":4280016896,"data_stored":4279498612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":114614888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":10.885999999999999,"5min":2.9089999999999998,"15min":1.579},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":106.22,"5min":106.22,"15min":106.22},"last":1.369},{"interface":"front","average":{"1min":11.151,"5min":2.96,"15min":1.595},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":106.986,"5min":106.986,"15min":106.986},"last":0.70799999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.3440000000000003,"5min":1.732,"15min":0.79600000000000004},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":106.121,"5min":106.121,"15min":106.121},"last":6.2039999999999997},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.46100000000000002,"15min":0.38700000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":1.321}]},{"osd":2,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.468,"5min":1.788,"15min":0.84099999999999997},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":106.105,"5min":106.105,"15min":106.105},"last":1.3440000000000001},{"interface":"front","average":{"1min":7.4180000000000001,"5min":1.7729999999999999,"15min":0.83199999999999996},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":106.70399999999999,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.58899999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.4279999999999999,"5min":1.96,"15min":1.048},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":106.85899999999999,"5min":106.85899999999999,"15min":106.85899999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.66600000000000004,"15min":0.61899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2130000000000001,"5min":4.2130000000000001,"15min":4.2130000000000001},"last":6.1509999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.64000000000000001,"15min":0.60899999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.887,"5min":2.887,"15min":2.887},"last":6.1790000000000003},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.0429999999999999,"15min":1.101},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.1160000000000001,"5min":9.9369999999999994,"15min":9.9369999999999994},"last":1.2470000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.3689999999999998,"5min":2.363,"15min":1.5289999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":106.036,"5min":106.036,"15min":106.036},"last":1.284},{"interface":"front","average":{"1min":7.5259999999999998,"5min":1.9970000000000001,"15min":1.075},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":106.741,"5min":106.741,"15min":106.741},"last":0.88200000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:05:39 2023","interfaces":[{"interface":"back","average":{"1min":7.4710000000000001,"5min":1.9970000000000001,"15min":1.085},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":106.82599999999999,"5min":106.82599999999999,"15min":106.82599999999999},"last":0.91100000000000003},{"interface":"front","average":{"1min":7.3520000000000003,"5min":1.9510000000000001,"15min":1.0509999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":106.084,"5min":106.084,"15min":106.084},"last":6.1079999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986356,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2387220,"kb_used_data":2081740,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91357932,"statfs":{"total":95995035648,"available":93550522368,"internally_reserved":0,"allocated":2131701760,"data_stored":2131224864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":0.69299999999999995,"15min":0.58199999999999996},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.9900000000000002,"5min":8.9900000000000002,"15min":8.9900000000000002},"last":0.29099999999999998},{"interface":"front","average":{"1min":2.1200000000000001,"5min":0.86099999999999999,"15min":0.66900000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.1370000000000005,"5min":8.1370000000000005,"15min":8.1370000000000005},"last":0.40699999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.56699999999999995,"15min":0.55600000000000005},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.877},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.58099999999999996,"15min":0.55000000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.6339999999999999,"5min":2.6339999999999999,"15min":2.6339999999999999},"last":4.0499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":0.85199999999999998,"15min":0.67100000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":8.8680000000000003,"5min":8.8680000000000003,"15min":8.8680000000000003},"last":0.312},{"interface":"front","average":{"1min":1.331,"5min":0.71999999999999997,"15min":0.61399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.855,"5min":3.855,"15min":3.855},"last":0.34599999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:05:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.64700000000000002,"15min":0.57599999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.2869999999999999,"5min":3.2869999999999999,"15min":3.2869999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.196,"5min":0.68500000000000005,"15min":0.59999999999999998},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.8470000000000004,"5min":4.8470000000000004,"15min":4.8470000000000004},"last":3.02}]},{"osd":5,"last update":"Mon Oct 9 04:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.45700000000000002},"min":{"1min":0.26900000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70699999999999996,"5min":0.753,"15min":1.038},"last":0.376},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.52300000000000002,"15min":0.48899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.85499999999999998},"last":0.50800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.55700000000000005,"15min":0.53000000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.27900000000000003},"max":{"1min":0.70099999999999996,"5min":0.82099999999999995,"15min":0.92700000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.54700000000000004,"15min":0.52900000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66800000000000004,"5min":0.78200000000000003,"15min":0.93000000000000005},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.58899999999999997,"15min":0.55300000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.435},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.69499999999999995,"15min":0.629},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":5.5819999999999999,"5min":5.5819999999999999,"15min":5.5819999999999999},"last":0.64500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724720,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2403844,"kb_used_data":2094268,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91341308,"statfs":{"total":95995035648,"available":93533499392,"internally_reserved":0,"allocated":2144530432,"data_stored":2144065606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.64700000000000002,"15min":0.61399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5339999999999998,"5min":4.5339999999999998,"15min":4.5339999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.575,"5min":0.75,"15min":0.629},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":3.3620000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:05:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.59299999999999997,"15min":0.55900000000000005},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.9470000000000001,"5min":2.9470000000000001,"15min":2.9470000000000001},"last":4.0010000000000003},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.1739999999999999,"5min":2.1739999999999999,"15min":2.1739999999999999},"last":3.4180000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":0.90500000000000003,"15min":0.68700000000000006},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.6880000000000006,"5min":9.6880000000000006,"15min":9.6880000000000006},"last":3.4409999999999998},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.81100000000000005,"15min":0.629},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.484,"5min":9.484,"15min":9.484},"last":6.9420000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:05:42 2023","interfaces":[{"interface":"back","average":{"1min":4.3920000000000003,"5min":1.347,"15min":0.83899999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":60.396999999999998,"5min":60.396999999999998,"15min":60.396999999999998},"last":6.8550000000000004},{"interface":"front","average":{"1min":4.399,"5min":1.365,"15min":0.85899999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":60.383000000000003,"5min":60.383000000000003,"15min":60.383000000000003},"last":4.0919999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.61399999999999999,"15min":0.52000000000000002},"min":{"1min":0.27000000000000002,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":4.9539999999999997,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":3.4630000000000001},{"interface":"front","average":{"1min":1.327,"5min":0.68999999999999995,"15min":0.56599999999999995},"min":{"1min":0.30199999999999999,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":5.2000000000000002,"5min":5.2000000000000002,"15min":5.2000000000000002},"last":6.7709999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:05:43 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.55200000000000005,"15min":0.52600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":1.756,"5min":1.756,"15min":1.756},"last":4.056},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.59899999999999998,"15min":0.55800000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":2.145,"5min":2.145,"15min":2.145},"last":3.5129999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":0.78600000000000003,"15min":0.64600000000000002},"min":{"1min":0.371,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":8.3130000000000006,"5min":8.3130000000000006,"15min":8.3130000000000006},"last":4.0330000000000004},{"interface":"front","average":{"1min":1.8740000000000001,"5min":0.90800000000000003,"15min":0.70499999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34599999999999997},"max":{"1min":8.5329999999999995,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":3.4830000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1683750912,"data_stored":1683750912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2110914560,"data_stored":2110914560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3402366976,"data_stored":3402366976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4281139200,"data_stored":4281139200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2134507520,"data_stored":2134507520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2153447424,"data_stored":2153447424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2149580800,"data_stored":2149580800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2493710336,"data_stored":2493710336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:06:27.466 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:06:27.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-10-09T04:06:27.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:27.568 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:27.592 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:27.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:27.634 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:27.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:27.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:27.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:27.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:27 smithi163 ceph-mon[213841]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:27.821 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:27.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:27 smithi163 ceph-mon[221067]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:06:28.025 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 106787 106771 75.7922 47.1875 0.0173845 0.0131887 2023-10-09T04:06:28.050 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:28.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:28.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:28.243 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:28.328 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:28.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:28.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:28.463 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:28.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:28.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:28.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:28.675 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:28.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:28 smithi181 ceph-mon[196979]: pgmap v303: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T04:06:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:28 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:28 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:28 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:28.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[213841]: pgmap v303: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[221067]: pgmap v303: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:28.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:28 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:28.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:28.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:29.001 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:29.025 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 107589 107573 75.5039 50.125 0.00980064 0.0132398 2023-10-09T04:06:29.135 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:29.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:29.188 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:29.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:06:29.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:29.350 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:107867 6.a 6:5a4732d6:::benchmark_data_smithi163_287034_object107866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:29.383948+0000", 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.01317404, 2023-10-09T04:06:29.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013208298, 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 107867 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:29.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.383948+0000", 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.383948+0000", 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.383946+0000", 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:06:29.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384011+0000", 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5302000000000006e-05 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384039+0000", 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7903999999999999e-05 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384048+0000", 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9500000000000007e-06 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384117+0000", 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9436000000000004e-05 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384278+0000", 2023-10-09T04:06:29.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016065099999999999 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384395+0000", 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011760300000000001 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.390785+0000", 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0063893739999999997 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.394538+0000", 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037530570000000002 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:107868 6.a 6:5e968589:::benchmark_data_smithi163_287034_object107867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:29.384063+0000", 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.013058422, 2023-10-09T04:06:29.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013152192, 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 107868 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384063+0000", 2023-10-09T04:06:29.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384063+0000", 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384063+0000", 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384095+0000", 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2187999999999997e-05 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384107+0000", 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2694000000000001e-05 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.384113+0000", 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7559999999999996e-06 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.387209+0000", 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030954049999999999 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.387335+0000", 2023-10-09T04:06:29.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012668600000000001 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.387510+0000", 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000174482 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.394472+0000", 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069622909999999998 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.394544+0000", 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1965999999999995e-05 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:107874 6.7 6:e2141866:::benchmark_data_smithi163_287034_object107873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:29.388317+0000", 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0088048099999999997, 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089499539999999995, 2023-10-09T04:06:29.410 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 107874 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388317+0000", 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388317+0000", 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388316+0000", 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388419+0000", 2023-10-09T04:06:29.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103544 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388435+0000", 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6056e-05 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388441+0000", 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2049999999999999e-06 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388497+0000", 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5946999999999998e-05 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388603+0000", 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010547 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:06:29.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388730+0000", 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127391 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.392146+0000", 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034152470000000002 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.396186+0000", 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040404259999999997 2023-10-09T04:06:29.415 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:107877 6.a 6:5305a5d4:::benchmark_data_smithi163_287034_object107876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:29.388454+0000", 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0086674119999999993, 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0088633649999999998, 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:29.416 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 107877 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388454+0000", 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388454+0000", 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388454+0000", 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388489+0000", 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5051e-05 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388497+0000", 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1110000000000008e-06 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.388502+0000", 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0000000000000004e-06 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.390805+0000", 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023029499999999998 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.390935+0000", 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013029200000000001 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.391072+0000", 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013658999999999999 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.394523+0000", 2023-10-09T04:06:29.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003451698 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:29.394598+0000", 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4600999999999999e-05 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 4 2023-10-09T04:06:29.422 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:06:29.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:29 smithi163 ceph-mon[221067]: pgmap v304: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T04:06:29.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:29 smithi163 ceph-mon[213841]: pgmap v304: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T04:06:29.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:29.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:29.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:29.711 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:29 smithi181 ceph-mon[196979]: pgmap v304: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T04:06:29.715 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:29.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:29.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:29.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:29.865 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:29.907 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0058 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 108260 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 75.1757 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 34.5867 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 208 2023-10-09T04:06:30.026 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 37.6875 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1202 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 553.388 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3328 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 603 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0132986 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0147688 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 0.296599 2023-10-09T04:06:30.027 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00118809 2023-10-09T04:06:30.031 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:06:30.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:30.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:06:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:30.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:30.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:30.312 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:30.374 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:30.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:30.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:30.509 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/433084359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:06:30.509 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:06:30.534 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:06:30.562 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:30.700 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:30.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/433084359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:06:30.810 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:06:30.813 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:06:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:06:30] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-10-09T04:06:30.814 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/433084359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:06:30.814 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:06:30.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:30.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:30.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:30.968 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:31.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:31.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:31.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:31.440 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:31.464 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:31.518 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:31.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:31.600 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:31.601 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:06:31.601 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[221067]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T04:06:31.601 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[221067]: failed to encode map e122 with expected crc 2023-10-09T04:06:31.601 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[221067]: failed to encode map e122 with expected crc 2023-10-09T04:06:31.601 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[221067]: pgmap v306: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 114 B/s rd, 47 MiB/s wr, 768 op/s 2023-10-09T04:06:31.602 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:31.602 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:06:31.602 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[213841]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T04:06:31.602 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[213841]: failed to encode map e122 with expected crc 2023-10-09T04:06:31.602 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[213841]: failed to encode map e122 with expected crc 2023-10-09T04:06:31.603 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:31 smithi163 ceph-mon[213841]: pgmap v306: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 114 B/s rd, 47 MiB/s wr, 768 op/s 2023-10-09T04:06:31.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:31.698 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:31 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:06:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:31 smithi181 ceph-mon[196979]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T04:06:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:31 smithi181 ceph-mon[196979]: failed to encode map e122 with expected crc 2023-10-09T04:06:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:31 smithi181 ceph-mon[196979]: failed to encode map e122 with expected crc 2023-10-09T04:06:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:31 smithi181 ceph-mon[196979]: pgmap v306: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 114 B/s rd, 47 MiB/s wr, 768 op/s 2023-10-09T04:06:31.926 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:31.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:31.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:32.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:32.069 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:32.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:32.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:32.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:32.281 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:32.342 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:32.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:32.373 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:06:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:06:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:06:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:06:32.375 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:06:32.448 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:32.500 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:32.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:32.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:32.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:32.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:32.812 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:32.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:32.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:33.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:33.135 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:33.204 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 6 pg_num to 26 2023-10-09T04:06:33.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:33.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:33.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:33.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:06:33.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:33.403 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3195865316' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:06:33.403 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:33 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:06:33.411 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:33.437 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:06:33.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3195865316' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:06:33.449 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:33 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:06:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3195865316' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:06:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:33 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:06:33.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:33.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:33.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:33.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:33.650 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:33.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:33.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:33.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:33.825 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:06:33.836 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:06:33.836 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:06:33.837 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:33.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:33.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:06:33.887 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:06:33.918 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:33.944 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:06:33.975 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:06:34.028 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:06:34.338 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:34.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:34.398 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.398 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:06:34.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: osdmap e123: 8 total, 8 up, 8 in 2023-10-09T04:06:34.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: pgmap v308: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 37 MiB/s wr, 667 op/s 2023-10-09T04:06:34.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:06:34.399 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3507191228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:34.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:34.483 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:34.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:34.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:06:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: osdmap e123: 8 total, 8 up, 8 in 2023-10-09T04:06:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: pgmap v308: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 37 MiB/s wr, 667 op/s 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3507191228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: osdmap e123: 8 total, 8 up, 8 in 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: failed to encode map e123 with expected crc 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: pgmap v308: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 37 MiB/s wr, 667 op/s 2023-10-09T04:06:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:06:34.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3507191228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:34.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:34.657 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:34.700 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:34.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:34.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:34.830 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:34.983 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:35.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:35.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:35.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:35.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:35.171 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:35 smithi181 ceph-mon[196979]: failed to encode map e124 with expected crc 2023-10-09T04:06:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:06:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:35 smithi181 ceph-mon[196979]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T04:06:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:35 smithi181 ceph-mon[196979]: failed to encode map e124 with expected crc 2023-10-09T04:06:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:35.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:35.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:35.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:35.391 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:35.445 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:35.517 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:35.517 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:35.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:35.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:35.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[213841]: failed to encode map e124 with expected crc 2023-10-09T04:06:35.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:06:35.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[213841]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T04:06:35.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[213841]: failed to encode map e124 with expected crc 2023-10-09T04:06:35.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:35.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[221067]: failed to encode map e124 with expected crc 2023-10-09T04:06:35.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:06:35.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[221067]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T04:06:35.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[221067]: failed to encode map e124 with expected crc 2023-10-09T04:06:35.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:35.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:35.685 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:35.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:35.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:35.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:35.887 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:35.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:35.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:36.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:36.101 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:36.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[213841]: pgmap v311: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail 2023-10-09T04:06:36.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:36 smithi163 ceph-mon[221067]: pgmap v311: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail 2023-10-09T04:06:36.283 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:36.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:06:36.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:36.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:36.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:36.452 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:36.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T04:06:36.525 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: failed to encode map e125 with expected crc 2023-10-09T04:06:36.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:36 smithi181 ceph-mon[196979]: pgmap v311: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail 2023-10-09T04:06:36.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:36.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:36.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:36.720 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:36.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:36.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:36.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:36.877 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:36.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:36.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:06:36.959 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:37.063 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:37.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:37.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:37.255 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[213841]: 6.7 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[213841]: 6.9 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[213841]: 6.3 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[221067]: 6.7 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[221067]: 6.9 scrub starts 2023-10-09T04:06:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:37 smithi163 ceph-mon[221067]: 6.3 scrub starts 2023-10-09T04:06:37.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:37.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:37.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:37.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:37 smithi181 ceph-mon[196979]: 6.7 scrub starts 2023-10-09T04:06:37.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:37 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:06:37.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:37 smithi181 ceph-mon[196979]: 6.9 scrub starts 2023-10-09T04:06:37.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:37 smithi181 ceph-mon[196979]: 6.3 scrub starts 2023-10-09T04:06:37.551 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:37.619 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:37.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:37.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:37.752 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:37.804 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:37.921 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:37.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:38.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:38.053 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:38.119 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:38.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:38.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:38.262 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:38.290 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:06:38.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: 6.9 scrub ok 2023-10-09T04:06:38.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: 6.7 scrub ok 2023-10-09T04:06:38.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:06:38.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: 6.0 scrub starts 2023-10-09T04:06:38.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: pgmap v312: 99 pgs: 1 active+clean+scrubbing, 2 peering, 4 unknown, 92 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4010179142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[213841]: 6.3 scrub ok 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: 6.9 scrub ok 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: 6.7 scrub ok 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: 6.0 scrub starts 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: pgmap v312: 99 pgs: 1 active+clean+scrubbing, 2 peering, 4 unknown, 92 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4010179142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:38.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:38.322 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:38 smithi163 ceph-mon[221067]: 6.3 scrub ok 2023-10-09T04:06:38.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:38.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:38.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:38.480 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:38.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: 6.9 scrub ok 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: 6.7 scrub ok 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: 6.0 scrub starts 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: pgmap v312: 99 pgs: 1 active+clean+scrubbing, 2 peering, 4 unknown, 92 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4010179142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:38.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:38 smithi181 ceph-mon[196979]: 6.3 scrub ok 2023-10-09T04:06:38.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:38.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:38.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:38.736 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:39.210 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:39.210 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:39.435 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:06:39.435 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:06:39.435 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:06:39.435 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.237351 2023-10-09T04:06:39.436 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0.2373508764272274 2023-10-09T04:06:39.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: 6.17 scrub starts 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: 6.17 scrub ok 2023-10-09T04:06:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:39 smithi181 ceph-mon[196979]: 6.8 scrub starts 2023-10-09T04:06:39.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: 6.17 scrub starts 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: 6.17 scrub ok 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[213841]: 6.8 scrub starts 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: 6.17 scrub starts 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T04:06:39.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: failed to encode map e126 with expected crc 2023-10-09T04:06:39.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: 6.17 scrub ok 2023-10-09T04:06:39.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:39 smithi163 ceph-mon[221067]: 6.8 scrub starts 2023-10-09T04:06:40.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:40.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:40.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:06:40.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:40.283 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:40.352 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:40.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:40.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:40.504 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:40 smithi181 ceph-mon[196979]: pgmap v314: 99 pgs: 1 active+clean+scrubbing, 2 peering, 2 unknown, 94 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:40.512 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:40 smithi163 ceph-mon[221067]: pgmap v314: 99 pgs: 1 active+clean+scrubbing, 2 peering, 2 unknown, 94 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:40.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:40 smithi163 ceph-mon[213841]: pgmap v314: 99 pgs: 1 active+clean+scrubbing, 2 peering, 2 unknown, 94 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:40.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:40.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:40.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:40.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:40.720 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:40.785 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:40.870 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:40.932 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:40.943 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:06:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:06:40] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T04:06:40.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:40.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:41.039 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:06:41.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:41.072 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:06:41.115 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:41.116 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:06:41.151 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:06:41.193 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:06:41.237 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:06:41.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:41.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:41.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:41.286 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:06:41.338 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:06:41.344 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.345 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:06:41.362 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:41.437 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:41.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:41 smithi181 ceph-mon[196979]: 6.8 scrub ok 2023-10-09T04:06:41.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:41 smithi181 ceph-mon[196979]: 6.4 scrub starts 2023-10-09T04:06:41.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1778662106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:41 smithi163 ceph-mon[221067]: 6.8 scrub ok 2023-10-09T04:06:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:41 smithi163 ceph-mon[221067]: 6.4 scrub starts 2023-10-09T04:06:41.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1778662106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:41 smithi163 ceph-mon[213841]: 6.8 scrub ok 2023-10-09T04:06:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:41 smithi163 ceph-mon[213841]: 6.4 scrub starts 2023-10-09T04:06:41.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1778662106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:41.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:41.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:41.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:41.802 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:42.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:42.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:42.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:42.272 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:42.311 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:06:42.405 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:42 smithi181 ceph-mon[196979]: 6.4 scrub ok 2023-10-09T04:06:42.406 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:42 smithi181 ceph-mon[196979]: 6.14 scrub starts 2023-10-09T04:06:42.406 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:42 smithi181 ceph-mon[196979]: 6.10 deep-scrub starts 2023-10-09T04:06:42.406 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:42 smithi181 ceph-mon[196979]: pgmap v315: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:42.406 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:06:42.406 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:42 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:06:42.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:42.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:42.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[221067]: 6.4 scrub ok 2023-10-09T04:06:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[221067]: 6.14 scrub starts 2023-10-09T04:06:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[221067]: 6.10 deep-scrub starts 2023-10-09T04:06:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[221067]: pgmap v315: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:06:42.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:06:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[213841]: 6.4 scrub ok 2023-10-09T04:06:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[213841]: 6.14 scrub starts 2023-10-09T04:06:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[213841]: 6.10 deep-scrub starts 2023-10-09T04:06:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[213841]: pgmap v315: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:06:42.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:42 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:06:42.590 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:42.617 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:06:42.618 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:06:42.618 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:06:42.618 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:06:42.618 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:06:42.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:42.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:42.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:42.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:42.786 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:42.845 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:42.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:42.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:42.980 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:43.029 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:43.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:43.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:43.159 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:43.199 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:43.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:43.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:43.335 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:43.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:43.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:43.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:43.524 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:43.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:06:43.568 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:43.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: 6.14 scrub ok 2023-10-09T04:06:43.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: 6.19 scrub starts 2023-10-09T04:06:43.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: 6.11 deep-scrub starts 2023-10-09T04:06:43.573 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: failed to encode map e127 with expected crc 2023-10-09T04:06:43.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:06:43.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T04:06:43.574 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:43 smithi181 ceph-mon[196979]: failed to encode map e127 with expected crc 2023-10-09T04:06:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:43.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: 6.14 scrub ok 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: 6.19 scrub starts 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: 6.11 deep-scrub starts 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: failed to encode map e127 with expected crc 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T04:06:43.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[221067]: failed to encode map e127 with expected crc 2023-10-09T04:06:43.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: 6.14 scrub ok 2023-10-09T04:06:43.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: 6.19 scrub starts 2023-10-09T04:06:43.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: 6.11 deep-scrub starts 2023-10-09T04:06:43.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: failed to encode map e127 with expected crc 2023-10-09T04:06:43.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:06:43.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T04:06:43.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:43 smithi163 ceph-mon[213841]: failed to encode map e127 with expected crc 2023-10-09T04:06:43.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:43.711 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:43.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:43.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:43.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:43.908 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:43.942 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:43.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:44.025 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:44.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:44.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:44.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:44.250 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:44.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: 6.19 scrub ok 2023-10-09T04:06:44.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: 6.0 scrub starts 2023-10-09T04:06:44.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:06:44.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: pgmap v317: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:44.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: 6.2 scrub starts 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3613715889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]: dispatch 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]: dispatch 2023-10-09T04:06:44.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: 6.19 scrub ok 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: 6.0 scrub starts 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: pgmap v317: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: 6.2 scrub starts 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3613715889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]: dispatch 2023-10-09T04:06:44.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]: dispatch 2023-10-09T04:06:44.341 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.3 primary-affinity to 0.237351 (8155552) 2023-10-09T04:06:44.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:44.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:44.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:44.489 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: 6.19 scrub ok 2023-10-09T04:06:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: 6.0 scrub starts 2023-10-09T04:06:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:06:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: pgmap v317: 99 pgs: 99 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:06:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T04:06:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:06:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: failed to encode map e128 with expected crc 2023-10-09T04:06:44.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: 6.2 scrub starts 2023-10-09T04:06:44.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3613715889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]: dispatch 2023-10-09T04:06:44.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]: dispatch 2023-10-09T04:06:44.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:44.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:45.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:45.073 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:45.122 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:45.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:45.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:45.255 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: 6.0 scrub ok 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: 6.2 scrub ok 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]': finished 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T04:06:45.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.340 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.340 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:45.340 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:45 smithi181 ceph-mon[196979]: 6.12 scrub starts 2023-10-09T04:06:45.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: 6.0 scrub ok 2023-10-09T04:06:45.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: 6.2 scrub ok 2023-10-09T04:06:45.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:06:45.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]': finished 2023-10-09T04:06:45.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T04:06:45.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:45.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[213841]: 6.12 scrub starts 2023-10-09T04:06:45.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: 6.0 scrub ok 2023-10-09T04:06:45.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: 6.2 scrub ok 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.2373508764272274}]': finished 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: failed to encode map e129 with expected crc 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:45.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:45 smithi163 ceph-mon[221067]: 6.12 scrub starts 2023-10-09T04:06:45.600 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:06:45.600 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:06:45.600 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:06:45.601 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:06:45.601 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.601 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:71356 6.a e114/102 6:5b463fa6:::benchmark_data_smithi163_287034_object71355:head v 114'4528, mlcod=114'4526)", 2023-10-09T04:06:45.601 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:46.099088+0000", 2023-10-09T04:06:45.601 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 59.497715272999997, 2023-10-09T04:06:45.601 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016258093000000001, 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099088+0000", 2023-10-09T04:06:45.602 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099088+0000", 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.603 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.604 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.604 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099087+0000", 2023-10-09T04:06:45.604 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.604 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.604 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.604 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.605 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099116+0000", 2023-10-09T04:06:45.605 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9286999999999999e-05 2023-10-09T04:06:45.605 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.605 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.605 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.605 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099134+0000", 2023-10-09T04:06:45.606 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7490999999999999e-05 2023-10-09T04:06:45.606 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.606 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.606 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.606 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099140+0000", 2023-10-09T04:06:45.606 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0580000000000003e-06 2023-10-09T04:06:45.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.607 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.607 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.607 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099190+0000", 2023-10-09T04:06:45.607 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0640999999999998e-05 2023-10-09T04:06:45.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.608 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.608 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.608 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.099217+0000", 2023-10-09T04:06:45.608 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6486999999999999e-05 2023-10-09T04:06:45.608 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.608 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.609 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.115291+0000", 2023-10-09T04:06:45.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016074628000000001 2023-10-09T04:06:45.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.609 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:46.115346+0000", 2023-10-09T04:06:45.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4418e-05 2023-10-09T04:06:45.610 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.610 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.610 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.611 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.611 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:93745 6.3 e119/102 6:c539e149:::benchmark_data_smithi163_287034_object93744:head v 119'5898, mlcod=119'5896)", 2023-10-09T04:06:45.611 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:11.175162+0000", 2023-10-09T04:06:45.611 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.421641082000001, 2023-10-09T04:06:45.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01756597, 2023-10-09T04:06:45.611 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.612 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.612 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.612 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.612 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.612 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175162+0000", 2023-10-09T04:06:45.612 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175162+0000", 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.613 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.614 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.614 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175161+0000", 2023-10-09T04:06:45.614 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.614 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.614 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.614 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175182+0000", 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0973000000000001e-05 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175190+0000", 2023-10-09T04:06:45.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4789999999999999e-06 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175194+0000", 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1679999999999997e-06 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175240+0000", 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5849000000000001e-05 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.175261+0000", 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1472999999999999e-05 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.192673+0000", 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017411743 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.192728+0000", 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4768999999999997e-05 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:93986 6.5 e119/102 6:a257525a:::benchmark_data_smithi163_287034_object93985:head v 119'5845, mlcod=119'5842)", 2023-10-09T04:06:45.619 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:11.431617+0000", 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.165186255000002, 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015695363, 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431617+0000", 2023-10-09T04:06:45.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431617+0000", 2023-10-09T04:06:45.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431616+0000", 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431652+0000", 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6582999999999999e-05 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431671+0000", 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8473000000000001e-05 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431678+0000", 2023-10-09T04:06:45.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0859999999999996e-06 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431712+0000", 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4409000000000002e-05 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.431734+0000", 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1634e-05 2023-10-09T04:06:45.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.447269+0000", 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015534788000000001 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:11.447312+0000", 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3343000000000002e-05 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.626 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:95742 6.3 e120/102 6:cc7750fa:::benchmark_data_smithi163_287034_object95741:head v 120'6017, mlcod=120'6015)", 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:13.622625+0000", 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 31.974178393999999, 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015232589, 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.627 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622625+0000", 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622625+0000", 2023-10-09T04:06:45.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622624+0000", 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622643+0000", 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9162999999999999e-05 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622650+0000", 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.951e-06 2023-10-09T04:06:45.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622654+0000", 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0239999999999999e-06 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622697+0000", 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2904999999999997e-05 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.622720+0000", 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2342000000000001e-05 2023-10-09T04:06:45.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.637815+0000", 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015095307000000001 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:13.637857+0000", 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2179999999999999e-05 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.634 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96654 6.5 e120/102 6:a77b8512:::benchmark_data_smithi163_287034_object96653:head v 120'6004, mlcod=120'6002)", 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:14.735422+0000", 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.861381090999998, 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016752092, 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.635 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735422+0000", 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735422+0000", 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735421+0000", 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735443+0000", 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2344000000000002e-05 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735457+0000", 2023-10-09T04:06:45.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3801999999999999e-05 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735464+0000", 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7719999999999997e-06 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735531+0000", 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6747000000000003e-05 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735556+0000", 2023-10-09T04:06:45.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5358e-05 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.752124+0000", 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016568362999999999 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.752174+0000", 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9696e-05 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96653 6.c e120/102 6:3307209b:::benchmark_data_smithi163_287034_object96652:head v 120'6026, mlcod=120'6024)", 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:14.735490+0000", 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.861312893000001, 2023-10-09T04:06:45.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016594495000000001, 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735490+0000", 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735490+0000", 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735490+0000", 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735525+0000", 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5070999999999997e-05 2023-10-09T04:06:45.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735532+0000", 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.735537+0000", 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5630000000000002e-06 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.737565+0000", 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002028569 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.737587+0000", 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1497000000000001e-05 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.752027+0000", 2023-10-09T04:06:45.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014440741 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.752085+0000", 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7105999999999998e-05 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96651 6.a e120/102 6:571e852f:::benchmark_data_smithi163_287034_object96650:head v 120'6129, mlcod=120'6126)", 2023-10-09T04:06:45.647 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:14.738195+0000", 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.858608163, 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.019325416000000002, 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738195+0000", 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738195+0000", 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738194+0000", 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738222+0000", 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8498999999999999e-05 2023-10-09T04:06:45.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738236+0000", 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3869999999999999e-05 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738244+0000", 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3359999999999997e-06 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738644+0000", 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00040072699999999997 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738663+0000", 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.836e-05 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.757467+0000", 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018804213 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.757520+0000", 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3319000000000002e-05 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.652 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96655 6.a e120/102 6:5a4d1a63:::benchmark_data_smithi163_287034_object96654:head v 120'6130, mlcod=120'6126)", 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:14.738809+0000", 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.857994101999999, 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018734227999999999, 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738809+0000", 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738809+0000", 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738808+0000", 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738830+0000", 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1841999999999999e-05 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738841+0000", 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0747000000000001e-05 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.738847+0000", 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1990000000000003e-06 2023-10-09T04:06:45.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.741619+0000", 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027721740000000001 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.741642+0000", 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3045999999999999e-05 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.757524+0000", 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015881314000000001 2023-10-09T04:06:45.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.757543+0000", 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9511e-05 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96658 6.5 e120/102 6:a1252306:::benchmark_data_smithi163_287034_object96657:head v 120'6005, mlcod=120'6002)", 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:14.742612+0000", 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.854191266000001, 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016216544999999999, 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.659 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.742612+0000", 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.742612+0000", 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.742611+0000", 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.742635+0000", 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4294000000000002e-05 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.742647+0000", 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1344e-05 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.742652+0000", 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8069999999999996e-06 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.744809+0000", 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021570840000000001 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.744831+0000", 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1951999999999999e-05 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.758785+0000", 2023-10-09T04:06:45.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013953719999999999 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.758828+0000", 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3139999999999997e-05 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96704 6.3 e120/102 6:c19d5af3:::benchmark_data_smithi163_287034_object96703:head v 120'6077, mlcod=120'6073)", 2023-10-09T04:06:45.665 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:14.791738+0000", 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.805064643000001, 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015762831000000001, 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.791738+0000", 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.791738+0000", 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.791738+0000", 2023-10-09T04:06:45.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.791760+0000", 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1662999999999999e-05 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.791765+0000", 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6790000000000002e-06 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.791770+0000", 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0000000000000004e-06 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.794732+0000", 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029612029999999999 2023-10-09T04:06:45.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.794753+0000", 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1834000000000001e-05 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.807445+0000", 2023-10-09T04:06:45.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012691577000000001 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:14.807501+0000", 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6104999999999999e-05 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:96965 6.3 e120/102 6:c8c22fb5:::benchmark_data_smithi163_287034_object96964:head v 120'6091, mlcod=120'6088)", 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:15.093825+0000", 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.502978047999999, 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015692332, 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.093825+0000", 2023-10-09T04:06:45.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.093825+0000", 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.093825+0000", 2023-10-09T04:06:45.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.093848+0000", 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3269e-05 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.093855+0000", 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8430000000000004e-06 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.093859+0000", 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3579999999999996e-06 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.097053+0000", 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031935639999999999 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.097081+0000", 2023-10-09T04:06:45.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8935999999999999e-05 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.109436+0000", 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012354924999999999 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.109517+0000", 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0822999999999995e-05 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.677 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:97080 6.a e120/102 6:589ba547:::benchmark_data_smithi163_287034_object97079:head v 120'6155, mlcod=120'6153)", 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:15.208113+0000", 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.388690202999999, 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014938987000000001, 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.678 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208113+0000", 2023-10-09T04:06:45.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208113+0000", 2023-10-09T04:06:45.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208112+0000", 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208142+0000", 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0158000000000002e-05 2023-10-09T04:06:45.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208156+0000", 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4251e-05 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208163+0000", 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.25e-06 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208262+0000", 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8519999999999999e-05 2023-10-09T04:06:45.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208284+0000", 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2572999999999999e-05 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.222996+0000", 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014711838 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.223052+0000", 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5572000000000003e-05 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:97082 6.a e120/102 6:54e576b2:::benchmark_data_smithi163_287034_object97081:head v 120'6156, mlcod=120'6153)", 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:15.208181+0000", 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.388622016999999, 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014895647, 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208181+0000", 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208181+0000", 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208181+0000", 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208202+0000", 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1812999999999999e-05 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208209+0000", 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.917e-06 2023-10-09T04:06:45.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.208214+0000", 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5700000000000003e-06 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.210906+0000", 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026923699999999999 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.210930+0000", 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3306999999999999e-05 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.223056+0000", 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012126655 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.223077+0000", 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0329000000000001e-05 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:97126 6.c e120/102 6:31193c82:::benchmark_data_smithi163_287034_object97125:head v 120'6071, mlcod=120'6068)", 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:15.260975+0000", 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 30.335827684000002, 2023-10-09T04:06:45.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01681848, 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.260975+0000", 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.260975+0000", 2023-10-09T04:06:45.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.260975+0000", 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.264591+0000", 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036163620000000001 2023-10-09T04:06:45.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.264611+0000", 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9551999999999998e-05 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.264617+0000", 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0360000000000003e-06 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.264724+0000", 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107254 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.264741+0000", 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7657999999999999e-05 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.277755+0000", 2023-10-09T04:06:45.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013013943 2023-10-09T04:06:45.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:15.277794+0000", 2023-10-09T04:06:45.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8334999999999998e-05 2023-10-09T04:06:45.697 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:105329 6.5 e121/102 6:a1f21087:::benchmark_data_smithi163_287034_object105328:head v 121'6536, mlcod=121'6533)", 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:26.113546+0000", 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.483257126000002, 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015397157, 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.113546+0000", 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.113546+0000", 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.113545+0000", 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.113570+0000", 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4987999999999999e-05 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.113580+0000", 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.115e-06 2023-10-09T04:06:45.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.113584+0000", 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9259999999999999e-06 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.115814+0000", 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022293069999999998 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.115837+0000", 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2756e-05 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.128911+0000", 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013074217000000001 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:26.128943+0000", 2023-10-09T04:06:45.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2224000000000002e-05 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:107187 6.1 e121/102 6:86ecd37e:::benchmark_data_smithi163_287034_object107186:head v 121'6651, mlcod=121'6649)", 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:28.615504+0000", 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 16.981299022999998, 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014692485, 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.703 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615504+0000", 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615504+0000", 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615503+0000", 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615542+0000", 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9014999999999998e-05 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615558+0000", 2023-10-09T04:06:45.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5868000000000001e-05 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615564+0000", 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8159999999999999e-06 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615603+0000", 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9564000000000003e-05 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.615619+0000", 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5376999999999999e-05 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.630152+0000", 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014533595 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:28.630196+0000", 2023-10-09T04:06:45.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4298999999999997e-05 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:109519 6.3 e122/102 6:c47f2ae2:::benchmark_data_smithi163_287034_object1256:head v 122'6941, mlcod=122'6939)", 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:31.827098+0000", 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.769704549, 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015079781, 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.708 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.827098+0000", 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.827098+0000", 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.827097+0000", 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.827105+0000", 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3950000000000001e-06 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.827122+0000", 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7013000000000001e-05 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.827129+0000", 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5409999999999997e-06 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.830761+0000", 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036321349999999999 2023-10-09T04:06:45.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.830782+0000", 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0995000000000001e-05 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.842152+0000", 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011370465999999999 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:31.842178+0000", 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5704e-05 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:109845 6.c e122/102 6:39984094:::benchmark_data_smithi163_287034_object1582:head v 122'6881, mlcod=122'6878)", 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:32.270224+0000", 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.326578742000001, 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015043378, 2023-10-09T04:06:45.713 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270224+0000", 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270224+0000", 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270223+0000", 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270232+0000", 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8289999999999997e-06 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270247+0000", 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5806000000000001e-05 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270254+0000", 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4019999999999999e-06 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270311+0000", 2023-10-09T04:06:45.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7416000000000001e-05 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.270330+0000", 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9239999999999999e-05 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.285222+0000", 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014891391 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:32.285268+0000", 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5785000000000003e-05 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:111043 6.5 e123/102 6:a629e3dd:::benchmark_data_smithi163_287034_object2780:head v 123'6863, mlcod=123'6860)", 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:33.660830+0000", 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 11.935972722000001, 2023-10-09T04:06:45.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015313854999999999, 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660830+0000", 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660830+0000", 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660830+0000", 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660839+0000", 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8249999999999994e-06 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660845+0000", 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1650000000000003e-06 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660849+0000", 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8659999999999999e-06 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.664202+0000", 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033527140000000001 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.664222+0000", 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0630000000000001e-05 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.676107+0000", 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011884983 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.676144+0000", 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6897000000000003e-05 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.723 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:111047 6.5 e123/102 6:aa440585:::benchmark_data_smithi163_287034_object2784:head v 123'6864, mlcod=123'6860)", 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:33.660869+0000", 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 11.935933944, 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015296087999999999, 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660869+0000", 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660869+0000", 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660869+0000", 2023-10-09T04:06:45.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660872+0000", 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4790000000000001e-06 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660877+0000", 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0479999999999998e-06 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.660880+0000", 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1200000000000002e-06 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.667430+0000", 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065495830000000003 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:45.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.667449+0000", 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8828e-05 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.676147+0000", 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0086979880000000002 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:33.676165+0000", 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8240000000000002e-05 2023-10-09T04:06:45.728 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.729 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.729 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.729 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:45.729 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:45.729 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:06:45.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:06:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:45.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:45.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:45.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:45.860 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:46.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:06:46.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: 6.12 scrub ok 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: pgmap v320: 99 pgs: 6 peering, 93 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.691 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: 6.12 scrub ok 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: pgmap v320: 99 pgs: 6 peering, 93 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.693 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:46 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:06:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:46.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: 6.12 scrub ok 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: pgmap v320: 99 pgs: 6 peering, 93 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: failed to encode map e130 with expected crc 2023-10-09T04:06:46.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:46 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:06:46.779 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:46.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:46.828 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:46.891 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:46.954 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:46.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:47.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:47.137 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:47.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:47.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:47.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:47 smithi181 ceph-mon[196979]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:47 smithi181 ceph-mon[196979]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:47 smithi181 ceph-mon[196979]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T04:06:47.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:47 smithi181 ceph-mon[196979]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:47 smithi181 ceph-mon[196979]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[213841]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[213841]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[213841]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[213841]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[213841]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[221067]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[221067]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[221067]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T04:06:47.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[221067]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:47 smithi163 ceph-mon[221067]: failed to encode map e131 with expected crc 2023-10-09T04:06:47.621 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:47.840 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:47.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:47.948 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:48.005 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:48.132 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:48.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:48.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:48.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:48.288 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:48.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:48.375 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:48 smithi181 ceph-mon[196979]: pgmap v323: 99 pgs: 1 active+recovering+undersized+remapped, 4 activating, 6 peering, 88 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5938/292590 objects misplaced (2.029%); 2.5 MiB/s, 38 objects/s recovering 2023-10-09T04:06:48.375 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2160848872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:06:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:48.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:120527 6.18 6:1b75e722:::benchmark_data_smithi163_287034_object12264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.383076+0000", 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.016821372000000001, 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016845801000000001, 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.401 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120527 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383076+0000", 2023-10-09T04:06:48.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383076+0000", 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383075+0000", 2023-10-09T04:06:48.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383082+0000", 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8419999999999999e-06 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383098+0000", 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6167999999999998e-05 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383104+0000", 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5509999999999996e-06 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383159+0000", 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5136999999999997e-05 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383306+0000", 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014657 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.383420+0000", 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011431500000000001 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.384806+0000", 2023-10-09T04:06:48.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001385975 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.386308+0000", 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001501254 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:120534 6.8 6:16f8431b:::benchmark_data_smithi163_287034_object12271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T04:06:48.407 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.388536+0000", 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.011361682999999999, 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011444921, 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120534 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388536+0000", 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388536+0000", 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388535+0000", 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388539+0000", 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1749999999999998e-06 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388551+0000", 2023-10-09T04:06:48.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1758e-05 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388555+0000", 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7720000000000002e-06 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388609+0000", 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4001e-05 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388728+0000", 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118386 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.388846+0000", 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118446 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.390549+0000", 2023-10-09T04:06:48.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001703151 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.394580+0000", 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040307449999999996 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.413 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:120536 6.b 6:d655862f:::benchmark_data_smithi163_287034_object12273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.393323+0000", 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0065740499999999997, 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067070180000000004, 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120536 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393323+0000", 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393323+0000", 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393323+0000", 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393327+0000", 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5920000000000002e-06 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393340+0000", 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2935000000000001e-05 2023-10-09T04:06:48.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393346+0000", 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8900000000000004e-06 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393391+0000", 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5021000000000001e-05 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393513+0000", 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121907 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.393608+0000", 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5537000000000004e-05 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:06:48.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.396475+0000", 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028661440000000002 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:120538 6.8 6:10c7e440:::benchmark_data_smithi163_287034_object12275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.397144+0000", 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0027539190000000001, 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029331610000000001, 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:48.419 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120538 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397144+0000", 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397144+0000", 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397143+0000", 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397147+0000", 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4070000000000003e-06 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397158+0000", 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0667e-05 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397164+0000", 2023-10-09T04:06:48.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2960000000000004e-06 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397215+0000", 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1010000000000001e-05 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397335+0000", 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012016800000000001 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397446+0000", 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000110982 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:120540 6.b 6:d555e527:::benchmark_data_smithi163_287034_object12277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.397627+0000", 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0022701610000000001, 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002490857, 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.424 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120540 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397627+0000", 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397627+0000", 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397626+0000", 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397631+0000", 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0300000000000001e-06 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397644+0000", 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2663000000000001e-05 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397650+0000", 2023-10-09T04:06:48.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-06 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397693+0000", 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3306999999999997e-05 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397809+0000", 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115661 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.397919+0000", 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011000700000000001 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:120543 6.18 6:1c86ca9a:::benchmark_data_smithi163_287034_object12280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.398448+0000", 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.001449437, 2023-10-09T04:06:48.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001711101, 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120543 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398448+0000", 2023-10-09T04:06:48.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398448+0000", 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398447+0000", 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398452+0000", 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8289999999999998e-06 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398468+0000", 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5472999999999999e-05 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398474+0000", 2023-10-09T04:06:48.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8469999999999998e-06 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398524+0000", 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9203000000000003e-05 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398634+0000", 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000110356 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.398734+0000", 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9666000000000004e-05 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:120544 6.6 e132/129 6:63f4442f:::benchmark_data_smithi163_287034_object12281:head v 132'7329, mlcod=131'7327)", 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:48.399535+0000", 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.00036208300000000002, 2023-10-09T04:06:48.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00066583699999999996, 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399535+0000", 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399535+0000", 2023-10-09T04:06:48.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399534+0000", 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399542+0000", 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1529999999999997e-06 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399554+0000", 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1282e-05 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399560+0000", 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2099999999999998e-06 2023-10-09T04:06:48.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399599+0000", 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9422000000000003e-05 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:48.399617+0000", 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8122e-05 2023-10-09T04:06:48.438 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 7 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:06:48.439 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:06:48.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:48.546 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:48.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:48.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:48.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:48.768 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:48.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:48.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:48.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:48.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:48 smithi163 ceph-mon[221067]: pgmap v323: 99 pgs: 1 active+recovering+undersized+remapped, 4 activating, 6 peering, 88 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5938/292590 objects misplaced (2.029%); 2.5 MiB/s, 38 objects/s recovering 2023-10-09T04:06:48.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2160848872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:06:48.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:48 smithi163 ceph-mon[213841]: pgmap v323: 99 pgs: 1 active+recovering+undersized+remapped, 4 activating, 6 peering, 88 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5938/292590 objects misplaced (2.029%); 2.5 MiB/s, 38 objects/s recovering 2023-10-09T04:06:48.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2160848872' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:06:48.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:48.978 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:49.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: failed to encode map e132 with expected crc 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2160848872' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: failed to encode map e132 with expected crc 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:49.825 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:49.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: failed to encode map e132 with expected crc 2023-10-09T04:06:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2160848872' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:06:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T04:06:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: failed to encode map e132 with expected crc 2023-10-09T04:06:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:49.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: failed to encode map e132 with expected crc 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2160848872' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: failed to encode map e132 with expected crc 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:50.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:50.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:50.380 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:50.384 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:50 smithi163 ceph-mon[213841]: pgmap v325: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 activating, 6 peering, 87 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6636/292077 objects degraded (2.272%); 9256/292077 objects misplaced (3.169%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:06:50.384 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:50 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 6636/292077 objects degraded (2.272%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:06:50.385 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:50 smithi163 ceph-mon[221067]: pgmap v325: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 activating, 6 peering, 87 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6636/292077 objects degraded (2.272%); 9256/292077 objects misplaced (3.169%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:06:50.386 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:50 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 6636/292077 objects degraded (2.272%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:06:50.416 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:50.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:50.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:50.598 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:50.651 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:06:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:06:50] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-10-09T04:06:50.712 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:50 smithi181 ceph-mon[196979]: pgmap v325: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 activating, 6 peering, 87 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6636/292077 objects degraded (2.272%); 9256/292077 objects misplaced (3.169%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:06:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:50 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 6636/292077 objects degraded (2.272%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:06:50.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:06:50.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:50.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:06:50.936 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:06:50.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:50.970 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:06:50.997 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:51.003 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:06:51.041 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:06:51.081 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:06:51.084 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:51.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:51.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:51.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:06:51.164 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:06:51.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:51.220 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:06:51.220 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.220 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.220 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.221 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.221 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.221 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.221 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.221 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:06:51.290 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:51.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:51.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:51.505 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:51.505 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2121468074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:51.561 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:06:51.561 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:06:51.561 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:06:51.562 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.116657 2023-10-09T04:06:51.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.11665733042560278 2023-10-09T04:06:51.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:51.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2121468074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:51.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2121468074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:06:51.631 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:51.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:51.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:51.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:51.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:51.903 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:52.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:52.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:52.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:52.197 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:52.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:06:52.449 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:52 smithi181 ceph-mon[196979]: pgmap v326: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6636/284913 objects degraded (2.329%); 9256/284913 objects misplaced (3.249%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:06:52.449 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:52 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 3 pgs peering) 2023-10-09T04:06:52.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:06:52.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:52.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:52.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:52.628 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:52.653 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:52 smithi163 ceph-mon[213841]: pgmap v326: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6636/284913 objects degraded (2.329%); 9256/284913 objects misplaced (3.249%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:06:52.654 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:52 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 3 pgs peering) 2023-10-09T04:06:52.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:52 smithi163 ceph-mon[221067]: pgmap v326: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6636/284913 objects degraded (2.329%); 9256/284913 objects misplaced (3.249%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:06:52.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:52 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 3 pgs peering) 2023-10-09T04:06:52.709 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:52.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:52.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:52.840 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:52.853 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:52.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:52.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:52.965 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:53.023 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:53.059 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:53.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:53.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:53.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:53.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:53.318 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:53.438 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:53.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:53.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:53.561 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:06:53.597 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:53.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:06:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:53.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:06:53.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:53.816 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:53.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:53.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:53.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:54.036 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:54.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:54.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:54.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:54.594 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:54 smithi181 ceph-mon[196979]: pgmap v327: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6378/281733 objects degraded (2.264%); 8797/281733 objects misplaced (3.122%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:06:54.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:54.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:06:54.712 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:54 smithi163 ceph-mon[213841]: pgmap v327: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6378/281733 objects degraded (2.264%); 8797/281733 objects misplaced (3.122%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:06:54.712 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:54 smithi163 ceph-mon[221067]: pgmap v327: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 6378/281733 objects degraded (2.264%); 8797/281733 objects misplaced (3.122%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:06:54.745 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:54.828 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:06:54.828 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:06:54.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:54.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:54.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:55.024 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:55.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:55.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:55.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:55.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:55.460 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:55.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:55.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:55.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:06:55.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:55.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:55 smithi163 ceph-mon[221067]: pgmap v328: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 6378/278010 objects degraded (2.294%); 8797/278010 objects misplaced (3.164%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:06:55.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:55 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6378/278010 objects degraded (2.294%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:06:55.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:55 smithi163 ceph-mon[213841]: pgmap v328: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 6378/278010 objects degraded (2.294%); 8797/278010 objects misplaced (3.164%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:06:55.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:55 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6378/278010 objects degraded (2.294%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:06:55.717 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:55.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:55 smithi181 ceph-mon[196979]: pgmap v328: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 6378/278010 objects degraded (2.294%); 8797/278010 objects misplaced (3.164%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:06:55.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:55 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6378/278010 objects degraded (2.294%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:06:55.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:55.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:55.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:06:55.919 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:56.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:06:56.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:56.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:06:56.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:56.409 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:56.471 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:56.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:56.509 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.6 primary-affinity to 0.116657 (876452) 2023-10-09T04:06:56.538 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4093708852' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]: dispatch 2023-10-09T04:06:56.538 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:56 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]: dispatch 2023-10-09T04:06:56.538 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4093708852' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]: dispatch 2023-10-09T04:06:56.538 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:56 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]: dispatch 2023-10-09T04:06:56.559 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:56.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:56.687 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4093708852' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]: dispatch 2023-10-09T04:06:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:56 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]: dispatch 2023-10-09T04:06:56.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:56.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:56.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:06:56.986 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:57.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:06:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:57.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:06:57.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:06:57.244 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:57.687 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:06:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:57.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:06:57.749 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:57 smithi181 ceph-mon[196979]: failed to encode map e133 with expected crc 2023-10-09T04:06:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:57 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]': finished 2023-10-09T04:06:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:57 smithi181 ceph-mon[196979]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T04:06:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:57 smithi181 ceph-mon[196979]: pgmap v330: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 6378/271470 objects degraded (2.349%); 8513/271470 objects misplaced (3.136%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:06:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:57 smithi181 ceph-mon[196979]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T04:06:57.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:57.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[221067]: failed to encode map e133 with expected crc 2023-10-09T04:06:57.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]': finished 2023-10-09T04:06:57.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[221067]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T04:06:57.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[221067]: pgmap v330: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 6378/271470 objects degraded (2.349%); 8513/271470 objects misplaced (3.136%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:06:57.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[221067]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T04:06:57.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[213841]: failed to encode map e133 with expected crc 2023-10-09T04:06:57.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.11665733042560278}]': finished 2023-10-09T04:06:57.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[213841]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T04:06:57.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[213841]: pgmap v330: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 6378/271470 objects degraded (2.349%); 8513/271470 objects misplaced (3.136%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:06:57.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:57 smithi163 ceph-mon[213841]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T04:06:57.833 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:57.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:57.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:57.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:58.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:06:58.094 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:58.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:06:58.540 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3153647548' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: failed to encode map e134 with expected crc 2023-10-09T04:06:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:06:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3153647548' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: failed to encode map e134 with expected crc 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: failed to encode map e134 with expected crc 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: failed to encode map e134 with expected crc 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: failed to encode map e134 with expected crc 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: failed to encode map e134 with expected crc 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:06:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3153647548' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:06:59.029 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:06:59.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:59.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:06:59.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:06:59.184 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:59.211 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:06:59.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:06:59.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:59.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:06:59.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:06:59.391 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:59.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:06:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:06:59.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:06:59.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:06:59.671 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:06:59.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:59.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: failed to encode map e135 with expected crc 2023-10-09T04:06:59.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3153647548' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:59.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T04:06:59.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: failed to encode map e135 with expected crc 2023-10-09T04:06:59.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: pgmap v333: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 6168/270102 objects degraded (2.284%); 8408/270102 objects misplaced (3.113%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:06:59.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:06:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: failed to encode map e135 with expected crc 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3153647548' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: failed to encode map e135 with expected crc 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: pgmap v333: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 6168/270102 objects degraded (2.284%); 8408/270102 objects misplaced (3.113%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:06:59.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:06:59.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:06:59.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: failed to encode map e135 with expected crc 2023-10-09T04:06:59.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3153647548' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:06:59.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T04:06:59.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: failed to encode map e135 with expected crc 2023-10-09T04:06:59.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: pgmap v333: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 6168/270102 objects degraded (2.284%); 8408/270102 objects misplaced (3.113%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:06:59.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:06:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:00.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:00.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:00.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:00.613 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:00.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:00.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:00.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:00.798 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:07:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:07:00] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-10-09T04:07:00.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:00 smithi163 ceph-mon[221067]: failed to encode map e135 with expected crc 2023-10-09T04:07:00.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:00 smithi163 ceph-mon[213841]: failed to encode map e135 with expected crc 2023-10-09T04:07:00.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:00 smithi181 ceph-mon[196979]: failed to encode map e135 with expected crc 2023-10-09T04:07:00.834 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:07:00 smithi181 conmon[175350]: logger=sqlstore.transactions t=2023-10-09T04:07:00.540012082Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:07:00.835 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:07:00 smithi181 conmon[175350]: logger=cleanup t=2023-10-09T04:07:00.595833339Z level=info msg="Completed cleanup jobs" duration=66.196721ms 2023-10-09T04:07:00.837 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:00.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:00.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:01.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:01.061 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:01.753 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:01.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:01.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:01 smithi163 ceph-mon[213841]: pgmap v334: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 6168/260430 objects degraded (2.368%); 8408/260430 objects misplaced (3.229%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:07:01.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:01 smithi163 ceph-mon[221067]: pgmap v334: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 6168/260430 objects degraded (2.368%); 8408/260430 objects misplaced (3.229%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:07:01.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:01.960 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:01 smithi181 ceph-mon[196979]: pgmap v334: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 6168/260430 objects degraded (2.368%); 8408/260430 objects misplaced (3.229%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:07:02.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:02.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:02.244 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:07:02.263 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:02.290 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:07:02.306 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:132083 6.0 6:01263ffb:::benchmark_data_smithi163_287034_object23820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:02.295483+0000", 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0056623860000000002, 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056985220000000001, 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:02.312 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 132083 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295483+0000", 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295483+0000", 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295482+0000", 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:02.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295490+0000", 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0919999999999998e-06 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295507+0000", 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.736e-05 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295516+0000", 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9039999999999998e-06 2023-10-09T04:07:02.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.296073+0000", 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055720100000000001 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.296192+0000", 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011899400000000001 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:07:02.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.296291+0000", 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9047000000000002e-05 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:132088 6.10 6:0d07c5d2:::benchmark_data_smithi163_287034_object23825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:02.295523+0000", 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0056221830000000002, 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057605410000000001, 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:02.317 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 132088 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295523+0000", 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295523+0000", 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295523+0000", 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295526+0000", 2023-10-09T04:07:02.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1310000000000002e-06 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295533+0000", 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4389999999999997e-06 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.295545+0000", 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2544000000000001e-05 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298181+0000", 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026357260000000001 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298322+0000", 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014080599999999999 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298462+0000", 2023-10-09T04:07:02.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014057699999999999 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:132082 6.d e135/102 6:b58b29c8:::benchmark_data_smithi163_287034_object23819:head v 135'8331, mlcod=135'8329)", 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:02.298550+0000", 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0025960969999999999, 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028415850000000002, 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:07:02.322 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298550+0000", 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298550+0000", 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298549+0000", 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298554+0000", 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1830000000000002e-06 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298561+0000", 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8739999999999997e-06 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.298566+0000", 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.203e-06 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.300477+0000", 2023-10-09T04:07:02.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001911039 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:02.300497+0000", 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9947999999999999e-05 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 3 2023-10-09T04:07:02.326 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:07:02.332 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:07:02.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:02.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:02.396 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:07:02.438 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:07:02.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:02.482 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:07:02.514 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:07:02.544 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:02.560 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:07:02.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.563 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:07:02.644 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:07:02.645 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:07:02.645 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T04:07:02.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:07:02.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:02.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:02.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:02.767 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6168/260430 objects degraded (2.368%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:02.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3782985478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6168/260430 objects degraded (2.368%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3782985478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:02.853 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:02.943 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:02.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:02.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6168/260430 objects degraded (2.368%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3782985478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:03.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:03.089 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:03.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:03.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:03.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:03.313 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:03.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:03.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:03.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:03.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:03.601 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:03.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:03 smithi181 ceph-mon[196979]: pgmap v335: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5910/257115 objects degraded (2.299%); 7917/257115 objects misplaced (3.079%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:07:03.830 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:03 smithi163 ceph-mon[213841]: pgmap v335: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5910/257115 objects degraded (2.299%); 7917/257115 objects misplaced (3.079%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:07:03.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:03 smithi163 ceph-mon[221067]: pgmap v335: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5910/257115 objects degraded (2.299%); 7917/257115 objects misplaced (3.079%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:07:04.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:07:04.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:04.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:04.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:04.225 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:04.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:07:04.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:04.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:04.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:04.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:04.653 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:04 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:07:04.653 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:04.653 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:04.662 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:04.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:04 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:07:04.665 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:04.665 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:04.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:04.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:04.818 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:04.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:04.853 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:04.856 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:04.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:04.914 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:04 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:07:04.914 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:04.914 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:04.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:05.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:05.092 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:05.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:05.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:05.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:05.335 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:05.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:05.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:05.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:05.623 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:05 smithi181 ceph-mon[196979]: pgmap v336: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 5910/253380 objects degraded (2.332%); 7917/253380 objects misplaced (3.125%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:07:06.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:05 smithi163 ceph-mon[221067]: pgmap v336: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 5910/253380 objects degraded (2.332%); 7917/253380 objects misplaced (3.125%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:07:06.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:05 smithi163 ceph-mon[213841]: pgmap v336: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 5910/253380 objects degraded (2.332%); 7917/253380 objects misplaced (3.125%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:07:06.364 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:06.377 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:06.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:06.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:06.522 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:06.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:06.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:06.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:06.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:07:06.785 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:07.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:07.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:07.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:07.339 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:07.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:07.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:07.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:07.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5910/253380 objects degraded (2.332%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:07.543 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5910/253380 objects degraded (2.332%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:07.543 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5910/253380 objects degraded (2.332%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:07.546 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:08.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:08.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:08.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:08.282 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:08.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:08 smithi163 ceph-mon[213841]: pgmap v337: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 5910/246441 objects degraded (2.398%); 7565/246441 objects misplaced (3.070%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:07:08.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:08 smithi163 ceph-mon[221067]: pgmap v337: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 5910/246441 objects degraded (2.398%); 7565/246441 objects misplaced (3.070%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:07:08.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:08.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:08.449 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:08.486 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:08.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:08 smithi181 ceph-mon[196979]: pgmap v337: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 5910/246441 objects degraded (2.398%); 7565/246441 objects misplaced (3.070%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:07:08.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:08.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:08.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:08.698 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:08.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:08.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:08.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:08.922 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:08.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:08.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:09.100 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:09.159 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:09.243 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1097004677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:09.257 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:07:09.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:09.538 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1097004677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:09.539 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1097004677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:09.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:09.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:09.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:09.743 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:10.478 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1097004677' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: failed to encode map e136 with expected crc 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: failed to encode map e136 with expected crc 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: pgmap v339: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 5671/244827 objects degraded (2.316%); 7446/244827 objects misplaced (3.041%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:10 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:07:10.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1097004677' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: failed to encode map e136 with expected crc 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: failed to encode map e136 with expected crc 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: pgmap v339: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 5671/244827 objects degraded (2.316%); 7446/244827 objects misplaced (3.041%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:07:10.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1097004677' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: failed to encode map e136 with expected crc 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: failed to encode map e136 with expected crc 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: pgmap v339: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 5671/244827 objects degraded (2.316%); 7446/244827 objects misplaced (3.041%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:10.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:10 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:07:10.639 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:10.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:10.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:10.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:10.810 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:10.878 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:10.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:10.983 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:10.984 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:07:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:07:10] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-10-09T04:07:11.102 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:11.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:11.329 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:07:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:07:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:07:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:07:11.329 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:07:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:07:11.335 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:07:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:11.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:11.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:11.473 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:11.533 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:11.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:11.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:11.641 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:11.736 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:12.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:07:12.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[221067]: from='client.54679 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:12.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[221067]: pgmap v340: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 5671/235689 objects degraded (2.406%); 7446/235689 objects misplaced (3.159%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:07:12.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:07:12.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5671/235689 objects degraded (2.406%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:12.731 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[213841]: from='client.54679 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:12.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[213841]: pgmap v340: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 5671/235689 objects degraded (2.406%); 7446/235689 objects misplaced (3.159%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:07:12.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:07:12.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5671/235689 objects degraded (2.406%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:12 smithi181 ceph-mon[196979]: from='client.54679 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:12 smithi181 ceph-mon[196979]: pgmap v340: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 5671/235689 objects degraded (2.406%); 7446/235689 objects misplaced (3.159%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:07:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:12 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:07:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5671/235689 objects degraded (2.406%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:12.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:12.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:12.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:12.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:12.959 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:13.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:13.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:13.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:13.200 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:13.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:13.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:13.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:13.558 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:13.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:13.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:13.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:13.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:07:13.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:13.900 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":339,"stamp":"2023-10-09T04:07:09.340454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5347186616,"num_objects":81609,"num_object_clones":0,"num_object_copies":244827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5671,"num_objects_misplaced":7446,"num_objects_unfound":0,"num_objects_dirty":81609,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":95180,"num_write_kb":4545678,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1354,"num_bytes_recovered":89982560,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":201772,"ondisk_log_size":201772,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17547244,"kb_used_data":15937644,"kb_used_omap":126,"kb_used_meta":1609025,"kb_avail":732413972,"statfs":{"total":767960285184,"available":749991907328,"internally_reserved":0,"allocated":16320147456,"data_stored":16316018497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1647642235},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,8,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-552140800,"num_objects":-8425,"num_object_clones":0,"num_object_copies":-25275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-497,"num_objects_misplaced":-962,"num_objects_unfound":0,"num_objects_dirty":-8425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8425,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":605,"num_bytes_recovered":39649280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004644"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":112,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492668+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:06:57.492668+0000","last_peered":"2023-10-09T04:06:57.492668+0000","last_clean":"2023-10-09T04:06:57.492668+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:06:57.492668+0000","last_undegraded":"2023-10-09T04:06:57.492668+0000","last_fullsized":"2023-10-09T04:06:57.492668+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.495018+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:06:57.495018+0000","last_peered":"2023-10-09T04:06:57.495018+0000","last_clean":"2023-10-09T04:06:57.495018+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:06:57.495018+0000","last_undegraded":"2023-10-09T04:06:57.495018+0000","last_fullsized":"2023-10-09T04:06:57.495018+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491563+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:06:57.491563+0000","last_peered":"2023-10-09T04:06:57.491563+0000","last_clean":"2023-10-09T04:06:57.491563+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:57.491563+0000","last_undegraded":"2023-10-09T04:06:57.491563+0000","last_fullsized":"2023-10-09T04:06:57.491563+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491501+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:06:57.491501+0000","last_peered":"2023-10-09T04:06:57.491501+0000","last_clean":"2023-10-09T04:06:57.491501+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:06:57.491501+0000","last_undegraded":"2023-10-09T04:06:57.491501+0000","last_fullsized":"2023-10-09T04:06:57.491501+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":87,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.487673+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:06:57.487673+0000","last_peered":"2023-10-09T04:06:57.487673+0000","last_clean":"2023-10-09T04:06:57.487673+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:06:57.487673+0000","last_undegraded":"2023-10-09T04:06:57.487673+0000","last_fullsized":"2023-10-09T04:06:57.487673+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.498556+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:06:57.498556+0000","last_peered":"2023-10-09T04:06:57.498556+0000","last_clean":"2023-10-09T04:06:57.498556+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:06:57.498556+0000","last_undegraded":"2023-10-09T04:06:57.498556+0000","last_fullsized":"2023-10-09T04:06:57.498556+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"135'7708","reported_seq":7891,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.203666+0000","last_change":"2023-10-09T04:06:45.049045+0000","last_active":"2023-10-09T04:07:05.203666+0000","last_peered":"2023-10-09T04:07:05.203666+0000","last_clean":"2023-10-09T04:07:05.203666+0000","last_became_active":"2023-10-09T04:06:35.315954+0000","last_became_peered":"2023-10-09T04:06:35.315954+0000","last_unstale":"2023-10-09T04:07:05.203666+0000","last_undegraded":"2023-10-09T04:07:05.203666+0000","last_fullsized":"2023-10-09T04:07:05.203666+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"129'7222","last_scrub_stamp":"2023-10-09T04:06:45.048975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:45.048975+0000","objects_scrubbed":3069,"log_size":7708,"log_dups_size":0,"ondisk_log_size":7708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:42.361813+0000","scrub_duration":0.31357458100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169279488,"num_objects":2583,"num_object_clones":0,"num_object_copies":7749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":486,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491311+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:06:57.491311+0000","last_peered":"2023-10-09T04:06:57.491311+0000","last_clean":"2023-10-09T04:06:57.491311+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:06:57.491311+0000","last_undegraded":"2023-10-09T04:06:57.491311+0000","last_fullsized":"2023-10-09T04:06:57.491311+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.494033+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:06:57.494033+0000","last_peered":"2023-10-09T04:06:57.494033+0000","last_clean":"2023-10-09T04:06:57.494033+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:06:57.494033+0000","last_undegraded":"2023-10-09T04:06:57.494033+0000","last_fullsized":"2023-10-09T04:06:57.494033+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"135'7814","reported_seq":7865,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.385117+0000","last_change":"2023-10-09T04:06:35.323227+0000","last_active":"2023-10-09T04:07:05.385117+0000","last_peered":"2023-10-09T04:07:05.385117+0000","last_clean":"2023-10-09T04:07:05.385117+0000","last_became_active":"2023-10-09T04:06:35.319491+0000","last_became_peered":"2023-10-09T04:06:35.319491+0000","last_unstale":"2023-10-09T04:07:05.385117+0000","last_undegraded":"2023-10-09T04:07:05.385117+0000","last_fullsized":"2023-10-09T04:07:05.385117+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7814,"log_dups_size":0,"ondisk_log_size":7814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168984576,"num_objects":2579,"num_object_clones":0,"num_object_copies":7737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4271,"num_write_kb":219168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493570+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:06:57.493570+0000","last_peered":"2023-10-09T04:06:57.493570+0000","last_clean":"2023-10-09T04:06:57.493570+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:06:57.493570+0000","last_undegraded":"2023-10-09T04:06:57.493570+0000","last_fullsized":"2023-10-09T04:06:57.493570+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":31,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491556+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:06:57.491556+0000","last_peered":"2023-10-09T04:06:57.491556+0000","last_clean":"2023-10-09T04:06:57.491556+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:06:57.491556+0000","last_undegraded":"2023-10-09T04:06:57.491556+0000","last_fullsized":"2023-10-09T04:06:57.491556+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"135'7642","reported_seq":10168,"reported_epoch":135,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:07:06.818088+0000","last_change":"2023-10-09T04:06:44.537468+0000","last_active":"2023-10-09T04:07:06.818088+0000","last_peered":"2023-10-09T04:07:06.818088+0000","last_clean":"2023-10-09T04:06:42.296564+0000","last_became_active":"2023-10-09T04:06:44.463231+0000","last_became_peered":"2023-10-09T04:06:44.463231+0000","last_unstale":"2023-10-09T04:07:06.818088+0000","last_undegraded":"2023-10-09T04:07:06.818088+0000","last_fullsized":"2023-10-09T04:06:44.331372+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7642,"log_dups_size":0,"ondisk_log_size":7642,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163872768,"num_objects":2501,"num_object_clones":0,"num_object_copies":7503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4610,"num_objects_unfound":0,"num_objects_dirty":2501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4189,"num_write_kb":214048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":821,"num_bytes_recovered":53805056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,1","objects":196},{"shards":"0,2,3","objects":2305}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":97,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.487107+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:06:57.487107+0000","last_peered":"2023-10-09T04:06:57.487107+0000","last_clean":"2023-10-09T04:06:57.487107+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:06:57.487107+0000","last_undegraded":"2023-10-09T04:06:57.487107+0000","last_fullsized":"2023-10-09T04:06:57.487107+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492033+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:06:57.492033+0000","last_peered":"2023-10-09T04:06:57.492033+0000","last_clean":"2023-10-09T04:06:57.492033+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:06:57.492033+0000","last_undegraded":"2023-10-09T04:06:57.492033+0000","last_fullsized":"2023-10-09T04:06:57.492033+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"135'7678","reported_seq":9841,"reported_epoch":135,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:07:08.183751+0000","last_change":"2023-10-09T04:06:46.863766+0000","last_active":"2023-10-09T04:07:08.183751+0000","last_peered":"2023-10-09T04:07:08.183751+0000","last_clean":"2023-10-09T04:06:43.467386+0000","last_became_active":"2023-10-09T04:06:46.852798+0000","last_became_peered":"2023-10-09T04:06:46.852798+0000","last_unstale":"2023-10-09T04:07:08.183751+0000","last_undegraded":"2023-10-09T04:06:46.482499+0000","last_fullsized":"2023-10-09T04:06:46.344954+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7678,"log_dups_size":0,"ondisk_log_size":7678,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163676160,"num_objects":2498,"num_object_clones":0,"num_object_copies":7494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5671,"num_objects_misplaced":2836,"num_objects_unfound":0,"num_objects_dirty":2498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4213,"num_write_kb":214720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":526,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[1,0],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"0,1","objects":-7},{"shards":"1,2,5,6","objects":1},{"shards":"1,2,6","objects":2504}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":99,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492976+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:06:57.492976+0000","last_peered":"2023-10-09T04:06:57.492976+0000","last_clean":"2023-10-09T04:06:57.492976+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:06:57.492976+0000","last_undegraded":"2023-10-09T04:06:57.492976+0000","last_fullsized":"2023-10-09T04:06:57.492976+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":93,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492884+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:06:57.492884+0000","last_peered":"2023-10-09T04:06:57.492884+0000","last_clean":"2023-10-09T04:06:57.492884+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:06:57.492884+0000","last_undegraded":"2023-10-09T04:06:57.492884+0000","last_fullsized":"2023-10-09T04:06:57.492884+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"135'7782","reported_seq":7841,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.210440+0000","last_change":"2023-10-09T04:06:45.477720+0000","last_active":"2023-10-09T04:07:05.210440+0000","last_peered":"2023-10-09T04:07:05.210440+0000","last_clean":"2023-10-09T04:07:05.210440+0000","last_became_active":"2023-10-09T04:06:45.471218+0000","last_became_peered":"2023-10-09T04:06:45.471218+0000","last_unstale":"2023-10-09T04:07:05.210440+0000","last_undegraded":"2023-10-09T04:07:05.210440+0000","last_fullsized":"2023-10-09T04:07:05.210440+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4256,"num_write_kb":218912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":96,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486506+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:06:57.486506+0000","last_peered":"2023-10-09T04:06:57.486506+0000","last_clean":"2023-10-09T04:06:57.486506+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:06:57.486506+0000","last_undegraded":"2023-10-09T04:06:57.486506+0000","last_fullsized":"2023-10-09T04:06:57.486506+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492822+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:06:57.492822+0000","last_peered":"2023-10-09T04:06:57.492822+0000","last_clean":"2023-10-09T04:06:57.492822+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:06:57.492822+0000","last_undegraded":"2023-10-09T04:06:57.492822+0000","last_fullsized":"2023-10-09T04:06:57.492822+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.17","version":"135'7635","reported_seq":7826,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.288805+0000","last_change":"2023-10-09T04:06:38.312336+0000","last_active":"2023-10-09T04:07:05.288805+0000","last_peered":"2023-10-09T04:07:05.288805+0000","last_clean":"2023-10-09T04:07:05.288805+0000","last_became_active":"2023-10-09T04:06:35.331160+0000","last_became_peered":"2023-10-09T04:06:35.331160+0000","last_unstale":"2023-10-09T04:07:05.288805+0000","last_undegraded":"2023-10-09T04:07:05.288805+0000","last_fullsized":"2023-10-09T04:07:05.288805+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"126'6981","last_scrub_stamp":"2023-10-09T04:06:38.312243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:38.312243+0000","objects_scrubbed":3182,"log_size":7635,"log_dups_size":0,"ondisk_log_size":7635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:27.105520+0000","scrub_duration":0.56485964200000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":656,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491200+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:06:57.491200+0000","last_peered":"2023-10-09T04:06:57.491200+0000","last_clean":"2023-10-09T04:06:57.491200+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:06:57.491200+0000","last_undegraded":"2023-10-09T04:06:57.491200+0000","last_fullsized":"2023-10-09T04:06:57.491200+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493344+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:06:57.493344+0000","last_peered":"2023-10-09T04:06:57.493344+0000","last_clean":"2023-10-09T04:06:57.493344+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:06:57.493344+0000","last_undegraded":"2023-10-09T04:06:57.493344+0000","last_fullsized":"2023-10-09T04:06:57.493344+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"135'7797","reported_seq":7922,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:08.139710+0000","last_change":"2023-10-09T04:06:41.794687+0000","last_active":"2023-10-09T04:07:08.139710+0000","last_peered":"2023-10-09T04:07:08.139710+0000","last_clean":"2023-10-09T04:07:08.139710+0000","last_became_active":"2023-10-09T04:06:35.296891+0000","last_became_peered":"2023-10-09T04:06:35.296891+0000","last_unstale":"2023-10-09T04:07:08.139710+0000","last_undegraded":"2023-10-09T04:07:08.139710+0000","last_fullsized":"2023-10-09T04:07:08.139710+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"126'7207","last_scrub_stamp":"2023-10-09T04:06:41.794514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:41.794514+0000","objects_scrubbed":3122,"log_size":7797,"log_dups_size":0,"ondisk_log_size":7797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:31.793050+0000","scrub_duration":0.64420732400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165740544,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":593,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491479+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:06:57.491479+0000","last_peered":"2023-10-09T04:06:57.491479+0000","last_clean":"2023-10-09T04:06:57.491479+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:06:57.491479+0000","last_undegraded":"2023-10-09T04:06:57.491479+0000","last_fullsized":"2023-10-09T04:06:57.491479+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491388+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:06:57.491388+0000","last_peered":"2023-10-09T04:06:57.491388+0000","last_clean":"2023-10-09T04:06:57.491388+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:06:57.491388+0000","last_undegraded":"2023-10-09T04:06:57.491388+0000","last_fullsized":"2023-10-09T04:06:57.491388+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"135'7605","reported_seq":7656,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.394755+0000","last_change":"2023-10-09T04:06:35.308870+0000","last_active":"2023-10-09T04:07:05.394755+0000","last_peered":"2023-10-09T04:07:05.394755+0000","last_clean":"2023-10-09T04:07:05.394755+0000","last_became_active":"2023-10-09T04:06:35.304634+0000","last_became_peered":"2023-10-09T04:06:35.304634+0000","last_unstale":"2023-10-09T04:07:05.394755+0000","last_undegraded":"2023-10-09T04:07:05.394755+0000","last_fullsized":"2023-10-09T04:07:05.394755+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7605,"log_dups_size":0,"ondisk_log_size":7605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167608320,"num_objects":2558,"num_object_clones":0,"num_object_copies":7674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4162,"num_write_kb":215008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":24,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.533531+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:06:57.533531+0000","last_peered":"2023-10-09T04:06:57.533531+0000","last_clean":"2023-10-09T04:06:57.533531+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:06:57.533531+0000","last_undegraded":"2023-10-09T04:06:57.533531+0000","last_fullsized":"2023-10-09T04:06:57.533531+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":97,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.494928+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:06:57.494928+0000","last_peered":"2023-10-09T04:06:57.494928+0000","last_clean":"2023-10-09T04:06:57.494928+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:06:57.494928+0000","last_undegraded":"2023-10-09T04:06:57.494928+0000","last_fullsized":"2023-10-09T04:06:57.494928+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"135'8507","reported_seq":8548,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.371901+0000","last_change":"2023-10-09T04:04:51.281106+0000","last_active":"2023-10-09T04:07:05.371901+0000","last_peered":"2023-10-09T04:07:05.371901+0000","last_clean":"2023-10-09T04:07:05.371901+0000","last_became_active":"2023-10-09T04:04:51.280862+0000","last_became_peered":"2023-10-09T04:04:51.280862+0000","last_unstale":"2023-10-09T04:07:05.371901+0000","last_undegraded":"2023-10-09T04:07:05.371901+0000","last_fullsized":"2023-10-09T04:07:05.371901+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":8507,"log_dups_size":0,"ondisk_log_size":8507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:47.717190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340459520,"num_objects":5195,"num_object_clones":0,"num_object_copies":15585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8507,"num_write_kb":438464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":95,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.490109+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:06:57.490109+0000","last_peered":"2023-10-09T04:06:57.490109+0000","last_clean":"2023-10-09T04:06:57.490109+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:06:57.490109+0000","last_undegraded":"2023-10-09T04:06:57.490109+0000","last_fullsized":"2023-10-09T04:06:57.490109+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493384+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:06:57.493384+0000","last_peered":"2023-10-09T04:06:57.493384+0000","last_clean":"2023-10-09T04:06:57.493384+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:06:57.493384+0000","last_undegraded":"2023-10-09T04:06:57.493384+0000","last_fullsized":"2023-10-09T04:06:57.493384+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"135'8415","reported_seq":8466,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.388773+0000","last_change":"2023-10-09T04:06:45.485649+0000","last_active":"2023-10-09T04:07:05.388773+0000","last_peered":"2023-10-09T04:07:05.388773+0000","last_clean":"2023-10-09T04:07:05.388773+0000","last_became_active":"2023-10-09T04:06:45.471499+0000","last_became_peered":"2023-10-09T04:06:45.471499+0000","last_unstale":"2023-10-09T04:07:05.388773+0000","last_undegraded":"2023-10-09T04:07:05.388773+0000","last_fullsized":"2023-10-09T04:07:05.388773+0000","mapping_epoch":129,"log_start":"108'1107","ondisk_log_start":"108'1107","created":102,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7308,"log_dups_size":0,"ondisk_log_size":7308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.604485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299136,"num_objects":5101,"num_object_clones":0,"num_object_copies":15303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8415,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":96,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486839+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:06:57.486839+0000","last_peered":"2023-10-09T04:06:57.486839+0000","last_clean":"2023-10-09T04:06:57.486839+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:06:57.486839+0000","last_undegraded":"2023-10-09T04:06:57.486839+0000","last_fullsized":"2023-10-09T04:06:57.486839+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492639+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:06:57.492639+0000","last_peered":"2023-10-09T04:06:57.492639+0000","last_clean":"2023-10-09T04:06:57.492639+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:06:57.492639+0000","last_undegraded":"2023-10-09T04:06:57.492639+0000","last_fullsized":"2023-10-09T04:06:57.492639+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"135'7660","reported_seq":7850,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.302462+0000","last_change":"2023-10-09T04:06:40.080819+0000","last_active":"2023-10-09T04:07:05.302462+0000","last_peered":"2023-10-09T04:07:05.302462+0000","last_clean":"2023-10-09T04:07:05.302462+0000","last_became_active":"2023-10-09T04:06:35.281212+0000","last_became_peered":"2023-10-09T04:06:35.281212+0000","last_unstale":"2023-10-09T04:07:05.302462+0000","last_undegraded":"2023-10-09T04:07:05.302462+0000","last_fullsized":"2023-10-09T04:07:05.302462+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'7097","last_scrub_stamp":"2023-10-09T04:06:40.080754+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:40.080754+0000","objects_scrubbed":3223,"log_size":7660,"log_dups_size":0,"ondisk_log_size":7660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:12.223504+0000","scrub_duration":1.2295141649999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173146112,"num_objects":2642,"num_object_clones":0,"num_object_copies":7926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":581,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":78,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486717+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:06:57.486717+0000","last_peered":"2023-10-09T04:06:57.486717+0000","last_clean":"2023-10-09T04:06:57.486717+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:06:57.486717+0000","last_undegraded":"2023-10-09T04:06:57.486717+0000","last_fullsized":"2023-10-09T04:06:57.486717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":52,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:58.176725+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:06:58.176725+0000","last_peered":"2023-10-09T04:06:58.176725+0000","last_clean":"2023-10-09T04:06:58.176725+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:06:58.176725+0000","last_undegraded":"2023-10-09T04:06:58.176725+0000","last_fullsized":"2023-10-09T04:06:58.176725+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"135'7790","reported_seq":7919,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:06.774506+0000","last_change":"2023-10-09T04:06:36.749232+0000","last_active":"2023-10-09T04:07:06.774506+0000","last_peered":"2023-10-09T04:07:06.774506+0000","last_clean":"2023-10-09T04:07:06.774506+0000","last_became_active":"2023-10-09T04:06:35.284070+0000","last_became_peered":"2023-10-09T04:06:35.284070+0000","last_unstale":"2023-10-09T04:07:06.774506+0000","last_undegraded":"2023-10-09T04:07:06.774506+0000","last_fullsized":"2023-10-09T04:07:06.774506+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7102","last_scrub_stamp":"2023-10-09T04:06:36.748987+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:36.748987+0000","objects_scrubbed":3199,"log_size":7790,"log_dups_size":0,"ondisk_log_size":7790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:44.997815+0000","scrub_duration":0.68062300200000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164036608,"num_objects":2503,"num_object_clones":0,"num_object_copies":7509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":696,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493315+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:06:57.493315+0000","last_peered":"2023-10-09T04:06:57.493315+0000","last_clean":"2023-10-09T04:06:57.493315+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:06:57.493315+0000","last_undegraded":"2023-10-09T04:06:57.493315+0000","last_fullsized":"2023-10-09T04:06:57.493315+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491196+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:06:57.491196+0000","last_peered":"2023-10-09T04:06:57.491196+0000","last_clean":"2023-10-09T04:06:57.491196+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:06:57.491196+0000","last_undegraded":"2023-10-09T04:06:57.491196+0000","last_fullsized":"2023-10-09T04:06:57.491196+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"135'8308","reported_seq":8349,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.220957+0000","last_change":"2023-10-09T04:04:51.279219+0000","last_active":"2023-10-09T04:07:05.220957+0000","last_peered":"2023-10-09T04:07:05.220957+0000","last_clean":"2023-10-09T04:07:05.220957+0000","last_became_active":"2023-10-09T04:04:51.278902+0000","last_became_peered":"2023-10-09T04:04:51.278902+0000","last_unstale":"2023-10-09T04:07:05.220957+0000","last_undegraded":"2023-10-09T04:07:05.220957+0000","last_fullsized":"2023-10-09T04:07:05.220957+0000","mapping_epoch":102,"log_start":"107'900","ondisk_log_start":"107'900","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:37.687116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328073216,"num_objects":5006,"num_object_clones":0,"num_object_copies":15018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8308,"num_write_kb":426048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":37,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491125+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:06:57.491125+0000","last_peered":"2023-10-09T04:06:57.491125+0000","last_clean":"2023-10-09T04:06:57.491125+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:06:57.491125+0000","last_undegraded":"2023-10-09T04:06:57.491125+0000","last_fullsized":"2023-10-09T04:06:57.491125+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":93,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492890+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:06:57.492890+0000","last_peered":"2023-10-09T04:06:57.492890+0000","last_clean":"2023-10-09T04:06:57.492890+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:06:57.492890+0000","last_undegraded":"2023-10-09T04:06:57.492890+0000","last_fullsized":"2023-10-09T04:06:57.492890+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"135'8219","reported_seq":8261,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:04.572119+0000","last_change":"2023-10-09T04:04:51.275697+0000","last_active":"2023-10-09T04:07:04.572119+0000","last_peered":"2023-10-09T04:07:04.572119+0000","last_clean":"2023-10-09T04:07:04.572119+0000","last_became_active":"2023-10-09T04:04:51.275402+0000","last_became_peered":"2023-10-09T04:04:51.275402+0000","last_unstale":"2023-10-09T04:07:04.572119+0000","last_undegraded":"2023-10-09T04:07:04.572119+0000","last_fullsized":"2023-10-09T04:07:04.572119+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":8219,"log_dups_size":0,"ondisk_log_size":8219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:26.490300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339935232,"num_objects":5187,"num_object_clones":0,"num_object_copies":15561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8219,"num_write_kb":428992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":54,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.496319+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:06:57.496319+0000","last_peered":"2023-10-09T04:06:57.496319+0000","last_clean":"2023-10-09T04:06:57.496319+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:06:57.496319+0000","last_undegraded":"2023-10-09T04:06:57.496319+0000","last_fullsized":"2023-10-09T04:06:57.496319+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493832+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:06:57.493832+0000","last_peered":"2023-10-09T04:06:57.493832+0000","last_clean":"2023-10-09T04:06:57.493832+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:06:57.493832+0000","last_undegraded":"2023-10-09T04:06:57.493832+0000","last_fullsized":"2023-10-09T04:06:57.493832+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"135'8424","reported_seq":8475,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.186538+0000","last_change":"2023-10-09T04:06:45.497813+0000","last_active":"2023-10-09T04:07:05.186538+0000","last_peered":"2023-10-09T04:07:05.186538+0000","last_clean":"2023-10-09T04:07:05.186538+0000","last_became_active":"2023-10-09T04:06:45.470723+0000","last_became_peered":"2023-10-09T04:06:45.470723+0000","last_unstale":"2023-10-09T04:07:05.186538+0000","last_undegraded":"2023-10-09T04:07:05.186538+0000","last_fullsized":"2023-10-09T04:07:05.186538+0000","mapping_epoch":129,"log_start":"108'1142","ondisk_log_start":"108'1142","created":102,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7282,"log_dups_size":0,"ondisk_log_size":7282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:31.235972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337641472,"num_objects":5152,"num_object_clones":0,"num_object_copies":15456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8424,"num_write_kb":434432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492469+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:06:57.492469+0000","last_peered":"2023-10-09T04:06:57.492469+0000","last_clean":"2023-10-09T04:06:57.492469+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:06:57.492469+0000","last_undegraded":"2023-10-09T04:06:57.492469+0000","last_fullsized":"2023-10-09T04:06:57.492469+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":86,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.494093+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:06:57.494093+0000","last_peered":"2023-10-09T04:06:57.494093+0000","last_clean":"2023-10-09T04:06:57.494093+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:06:57.494093+0000","last_undegraded":"2023-10-09T04:06:57.494093+0000","last_fullsized":"2023-10-09T04:06:57.494093+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"135'8487","reported_seq":8529,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.258667+0000","last_change":"2023-10-09T04:04:51.279867+0000","last_active":"2023-10-09T04:07:05.258667+0000","last_peered":"2023-10-09T04:07:05.258667+0000","last_clean":"2023-10-09T04:07:05.258667+0000","last_became_active":"2023-10-09T04:04:51.279588+0000","last_became_peered":"2023-10-09T04:04:51.279588+0000","last_unstale":"2023-10-09T04:07:05.258667+0000","last_undegraded":"2023-10-09T04:07:05.258667+0000","last_fullsized":"2023-10-09T04:07:05.258667+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":8487,"log_dups_size":0,"ondisk_log_size":8487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.265205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336986136,"num_objects":5143,"num_object_clones":0,"num_object_copies":15429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5143,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8487,"num_write_kb":436097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.494263+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:06:57.494263+0000","last_peered":"2023-10-09T04:06:57.494263+0000","last_clean":"2023-10-09T04:06:57.494263+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:06:57.494263+0000","last_undegraded":"2023-10-09T04:06:57.494263+0000","last_fullsized":"2023-10-09T04:06:57.494263+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:06:59.185097+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:06:59.185097+0000","last_peered":"2023-10-09T04:06:59.185097+0000","last_clean":"2023-10-09T04:06:59.185097+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:06:59.185097+0000","last_undegraded":"2023-10-09T04:06:59.185097+0000","last_fullsized":"2023-10-09T04:06:59.185097+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":108,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492948+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:06:57.492948+0000","last_peered":"2023-10-09T04:06:57.492948+0000","last_clean":"2023-10-09T04:06:57.492948+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:06:57.492948+0000","last_undegraded":"2023-10-09T04:06:57.492948+0000","last_fullsized":"2023-10-09T04:06:57.492948+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":111,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493446+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:06:57.493446+0000","last_peered":"2023-10-09T04:06:57.493446+0000","last_clean":"2023-10-09T04:06:57.493446+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:06:57.493446+0000","last_undegraded":"2023-10-09T04:06:57.493446+0000","last_fullsized":"2023-10-09T04:06:57.493446+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492097+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:06:57.492097+0000","last_peered":"2023-10-09T04:06:57.492097+0000","last_clean":"2023-10-09T04:06:57.492097+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:06:57.492097+0000","last_undegraded":"2023-10-09T04:06:57.492097+0000","last_fullsized":"2023-10-09T04:06:57.492097+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"135'7647","reported_seq":7771,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:06.668311+0000","last_change":"2023-10-09T04:06:43.612195+0000","last_active":"2023-10-09T04:07:06.668311+0000","last_peered":"2023-10-09T04:07:06.668311+0000","last_clean":"2023-10-09T04:07:06.668311+0000","last_became_active":"2023-10-09T04:06:35.280772+0000","last_became_peered":"2023-10-09T04:06:35.280772+0000","last_unstale":"2023-10-09T04:07:06.668311+0000","last_undegraded":"2023-10-09T04:07:06.668311+0000","last_fullsized":"2023-10-09T04:07:06.668311+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"127'7121","last_scrub_stamp":"2023-10-09T04:06:43.612051+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:43.612051+0000","objects_scrubbed":2930,"log_size":7647,"log_dups_size":0,"ondisk_log_size":7647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:34.878053+0000","scrub_duration":0.51436748099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157548544,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":526,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"135'7751","reported_seq":7815,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.165607+0000","last_change":"2023-10-09T04:06:45.484861+0000","last_active":"2023-10-09T04:07:05.165607+0000","last_peered":"2023-10-09T04:07:05.165607+0000","last_clean":"2023-10-09T04:07:05.165607+0000","last_became_active":"2023-10-09T04:06:45.477025+0000","last_became_peered":"2023-10-09T04:06:45.477025+0000","last_unstale":"2023-10-09T04:07:05.165607+0000","last_undegraded":"2023-10-09T04:07:05.165607+0000","last_fullsized":"2023-10-09T04:07:05.165607+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7751,"log_dups_size":0,"ondisk_log_size":7751,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171442176,"num_objects":2616,"num_object_clones":0,"num_object_copies":7848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4225,"num_write_kb":218912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493282+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:06:57.493282+0000","last_peered":"2023-10-09T04:06:57.493282+0000","last_clean":"2023-10-09T04:06:57.493282+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:06:57.493282+0000","last_undegraded":"2023-10-09T04:06:57.493282+0000","last_fullsized":"2023-10-09T04:06:57.493282+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492967+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:06:57.492967+0000","last_peered":"2023-10-09T04:06:57.492967+0000","last_clean":"2023-10-09T04:06:57.492967+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:06:57.492967+0000","last_undegraded":"2023-10-09T04:06:57.492967+0000","last_fullsized":"2023-10-09T04:06:57.492967+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492885+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:06:57.492885+0000","last_peered":"2023-10-09T04:06:57.492885+0000","last_clean":"2023-10-09T04:06:57.492885+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:06:57.492885+0000","last_undegraded":"2023-10-09T04:06:57.492885+0000","last_fullsized":"2023-10-09T04:06:57.492885+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"135'7653","reported_seq":7708,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.308950+0000","last_change":"2023-10-09T04:06:35.940338+0000","last_active":"2023-10-09T04:07:05.308950+0000","last_peered":"2023-10-09T04:07:05.308950+0000","last_clean":"2023-10-09T04:07:05.308950+0000","last_became_active":"2023-10-09T04:06:35.267902+0000","last_became_peered":"2023-10-09T04:06:35.267902+0000","last_unstale":"2023-10-09T04:07:05.308950+0000","last_undegraded":"2023-10-09T04:07:05.308950+0000","last_fullsized":"2023-10-09T04:07:05.308950+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7653,"log_dups_size":0,"ondisk_log_size":7653,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164462592,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4209,"num_write_kb":215008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.502805+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:06:57.502805+0000","last_peered":"2023-10-09T04:06:57.502805+0000","last_clean":"2023-10-09T04:06:57.502805+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:06:57.502805+0000","last_undegraded":"2023-10-09T04:06:57.502805+0000","last_fullsized":"2023-10-09T04:06:57.502805+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491847+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:06:57.491847+0000","last_peered":"2023-10-09T04:06:57.491847+0000","last_clean":"2023-10-09T04:06:57.491847+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:06:57.491847+0000","last_undegraded":"2023-10-09T04:06:57.491847+0000","last_fullsized":"2023-10-09T04:06:57.491847+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":103,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.495236+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:06:57.495236+0000","last_peered":"2023-10-09T04:06:57.495236+0000","last_clean":"2023-10-09T04:06:57.495236+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:06:57.495236+0000","last_undegraded":"2023-10-09T04:06:57.495236+0000","last_fullsized":"2023-10-09T04:06:57.495236+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"135'7606","reported_seq":7793,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.355542+0000","last_change":"2023-10-09T04:06:36.855957+0000","last_active":"2023-10-09T04:07:05.355542+0000","last_peered":"2023-10-09T04:07:05.355542+0000","last_clean":"2023-10-09T04:07:05.355542+0000","last_became_active":"2023-10-09T04:06:35.290034+0000","last_became_peered":"2023-10-09T04:06:35.290034+0000","last_unstale":"2023-10-09T04:07:05.355542+0000","last_undegraded":"2023-10-09T04:07:05.355542+0000","last_fullsized":"2023-10-09T04:07:05.355542+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6925","last_scrub_stamp":"2023-10-09T04:06:36.855876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:36.855876+0000","objects_scrubbed":3147,"log_size":7606,"log_dups_size":0,"ondisk_log_size":7606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:54.835217+0000","scrub_duration":1.131125129,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160169984,"num_objects":2444,"num_object_clones":0,"num_object_copies":7332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":703,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493370+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:06:57.493370+0000","last_peered":"2023-10-09T04:06:57.493370+0000","last_clean":"2023-10-09T04:06:57.493370+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:06:57.493370+0000","last_undegraded":"2023-10-09T04:06:57.493370+0000","last_fullsized":"2023-10-09T04:06:57.493370+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":475,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486271+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:06:57.486271+0000","last_peered":"2023-10-09T04:06:57.486271+0000","last_clean":"2023-10-09T04:06:57.486271+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:06:57.486271+0000","last_undegraded":"2023-10-09T04:06:57.486271+0000","last_fullsized":"2023-10-09T04:06:57.486271+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":31,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493191+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:06:57.493191+0000","last_peered":"2023-10-09T04:06:57.493191+0000","last_clean":"2023-10-09T04:06:57.493191+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:06:57.493191+0000","last_undegraded":"2023-10-09T04:06:57.493191+0000","last_fullsized":"2023-10-09T04:06:57.493191+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.494742+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:06:57.494742+0000","last_peered":"2023-10-09T04:06:57.494742+0000","last_clean":"2023-10-09T04:06:57.494742+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:06:57.494742+0000","last_undegraded":"2023-10-09T04:06:57.494742+0000","last_fullsized":"2023-10-09T04:06:57.494742+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":115,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486187+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:06:57.486187+0000","last_peered":"2023-10-09T04:06:57.486187+0000","last_clean":"2023-10-09T04:06:57.486187+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:06:57.486187+0000","last_undegraded":"2023-10-09T04:06:57.486187+0000","last_fullsized":"2023-10-09T04:06:57.486187+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491112+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:06:57.491112+0000","last_peered":"2023-10-09T04:06:57.491112+0000","last_clean":"2023-10-09T04:06:57.491112+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:06:57.491112+0000","last_undegraded":"2023-10-09T04:06:57.491112+0000","last_fullsized":"2023-10-09T04:06:57.491112+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491658+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:06:57.491658+0000","last_peered":"2023-10-09T04:06:57.491658+0000","last_clean":"2023-10-09T04:06:57.491658+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:06:57.491658+0000","last_undegraded":"2023-10-09T04:06:57.491658+0000","last_fullsized":"2023-10-09T04:06:57.491658+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"135'7897","reported_seq":8023,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:08.139735+0000","last_change":"2023-10-09T04:06:40.914492+0000","last_active":"2023-10-09T04:07:08.139735+0000","last_peered":"2023-10-09T04:07:08.139735+0000","last_clean":"2023-10-09T04:07:08.139735+0000","last_became_active":"2023-10-09T04:06:35.285472+0000","last_became_peered":"2023-10-09T04:06:35.285472+0000","last_unstale":"2023-10-09T04:07:08.139735+0000","last_undegraded":"2023-10-09T04:07:08.139735+0000","last_fullsized":"2023-10-09T04:07:08.139735+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'7221","last_scrub_stamp":"2023-10-09T04:06:40.914305+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:40.914305+0000","objects_scrubbed":3121,"log_size":7897,"log_dups_size":0,"ondisk_log_size":7897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.920132+0000","scrub_duration":0.80758971499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159252480,"num_objects":2430,"num_object_clones":0,"num_object_copies":7290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":691,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":98,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486737+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:06:57.486737+0000","last_peered":"2023-10-09T04:06:57.486737+0000","last_clean":"2023-10-09T04:06:57.486737+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:06:57.486737+0000","last_undegraded":"2023-10-09T04:06:57.486737+0000","last_fullsized":"2023-10-09T04:06:57.486737+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493017+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:06:57.493017+0000","last_peered":"2023-10-09T04:06:57.493017+0000","last_clean":"2023-10-09T04:06:57.493017+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:06:57.493017+0000","last_undegraded":"2023-10-09T04:06:57.493017+0000","last_fullsized":"2023-10-09T04:06:57.493017+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491034+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:06:57.491034+0000","last_peered":"2023-10-09T04:06:57.491034+0000","last_clean":"2023-10-09T04:06:57.491034+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:06:57.491034+0000","last_undegraded":"2023-10-09T04:06:57.491034+0000","last_fullsized":"2023-10-09T04:06:57.491034+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"135'7705","reported_seq":7758,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:08.134997+0000","last_change":"2023-10-09T04:06:43.330354+0000","last_active":"2023-10-09T04:07:08.134997+0000","last_peered":"2023-10-09T04:07:08.134997+0000","last_clean":"2023-10-09T04:07:08.134997+0000","last_became_active":"2023-10-09T04:06:35.290191+0000","last_became_peered":"2023-10-09T04:06:35.290191+0000","last_unstale":"2023-10-09T04:07:08.134997+0000","last_undegraded":"2023-10-09T04:07:08.134997+0000","last_fullsized":"2023-10-09T04:07:08.134997+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161906688,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4239,"num_write_kb":214720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":39,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.489952+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:06:57.489952+0000","last_peered":"2023-10-09T04:06:57.489952+0000","last_clean":"2023-10-09T04:06:57.489952+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:06:57.489952+0000","last_undegraded":"2023-10-09T04:06:57.489952+0000","last_fullsized":"2023-10-09T04:06:57.489952+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":69,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.495974+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:06:57.495974+0000","last_peered":"2023-10-09T04:06:57.495974+0000","last_clean":"2023-10-09T04:06:57.495974+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:06:57.495974+0000","last_undegraded":"2023-10-09T04:06:57.495974+0000","last_fullsized":"2023-10-09T04:06:57.495974+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.486349+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:06:57.486349+0000","last_peered":"2023-10-09T04:06:57.486349+0000","last_clean":"2023-10-09T04:06:57.486349+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:06:57.486349+0000","last_undegraded":"2023-10-09T04:06:57.486349+0000","last_fullsized":"2023-10-09T04:06:57.486349+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"135'7683","reported_seq":7866,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.168827+0000","last_change":"2023-10-09T04:06:43.998989+0000","last_active":"2023-10-09T04:07:05.168827+0000","last_peered":"2023-10-09T04:07:05.168827+0000","last_clean":"2023-10-09T04:07:05.168827+0000","last_became_active":"2023-10-09T04:06:35.309761+0000","last_became_peered":"2023-10-09T04:06:35.309761+0000","last_unstale":"2023-10-09T04:07:05.168827+0000","last_undegraded":"2023-10-09T04:07:05.168827+0000","last_fullsized":"2023-10-09T04:07:05.168827+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"128'7225","last_scrub_stamp":"2023-10-09T04:06:43.998914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:43.998914+0000","objects_scrubbed":3035,"log_size":7683,"log_dups_size":0,"ondisk_log_size":7683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:01.096310+0000","scrub_duration":0.32511040200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168886272,"num_objects":2577,"num_object_clones":0,"num_object_copies":7731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":458,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.498435+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:06:57.498435+0000","last_peered":"2023-10-09T04:06:57.498435+0000","last_clean":"2023-10-09T04:06:57.498435+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:06:57.498435+0000","last_undegraded":"2023-10-09T04:06:57.498435+0000","last_fullsized":"2023-10-09T04:06:57.498435+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":92,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491584+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:06:57.491584+0000","last_peered":"2023-10-09T04:06:57.491584+0000","last_clean":"2023-10-09T04:06:57.491584+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:06:57.491584+0000","last_undegraded":"2023-10-09T04:06:57.491584+0000","last_fullsized":"2023-10-09T04:06:57.491584+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":76,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492588+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:06:57.492588+0000","last_peered":"2023-10-09T04:06:57.492588+0000","last_clean":"2023-10-09T04:06:57.492588+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:06:57.492588+0000","last_undegraded":"2023-10-09T04:06:57.492588+0000","last_fullsized":"2023-10-09T04:06:57.492588+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"135'7835","reported_seq":8028,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.362275+0000","last_change":"2023-10-09T04:06:37.747828+0000","last_active":"2023-10-09T04:07:05.362275+0000","last_peered":"2023-10-09T04:07:05.362275+0000","last_clean":"2023-10-09T04:07:05.362275+0000","last_became_active":"2023-10-09T04:06:35.269921+0000","last_became_peered":"2023-10-09T04:06:35.269921+0000","last_unstale":"2023-10-09T04:07:05.362275+0000","last_undegraded":"2023-10-09T04:07:05.362275+0000","last_fullsized":"2023-10-09T04:07:05.362275+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7180","last_scrub_stamp":"2023-10-09T04:06:37.747725+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:37.747725+0000","objects_scrubbed":3259,"log_size":7835,"log_dups_size":0,"ondisk_log_size":7835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:22.963882+0000","scrub_duration":0.83890256799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170000384,"num_objects":2594,"num_object_clones":0,"num_object_copies":7782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":665,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:06:59.192205+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:06:59.192205+0000","last_peered":"2023-10-09T04:06:59.192205+0000","last_clean":"2023-10-09T04:06:59.192205+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:06:59.192205+0000","last_undegraded":"2023-10-09T04:06:59.192205+0000","last_fullsized":"2023-10-09T04:06:59.192205+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.503094+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:06:57.503094+0000","last_peered":"2023-10-09T04:06:57.503094+0000","last_clean":"2023-10-09T04:06:57.503094+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:06:57.503094+0000","last_undegraded":"2023-10-09T04:06:57.503094+0000","last_fullsized":"2023-10-09T04:06:57.503094+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"135'7830","reported_seq":7960,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:06.741748+0000","last_change":"2023-10-09T04:06:42.872402+0000","last_active":"2023-10-09T04:07:06.741748+0000","last_peered":"2023-10-09T04:07:06.741748+0000","last_clean":"2023-10-09T04:07:06.741748+0000","last_became_active":"2023-10-09T04:06:35.306098+0000","last_became_peered":"2023-10-09T04:06:35.306098+0000","last_unstale":"2023-10-09T04:07:06.741748+0000","last_undegraded":"2023-10-09T04:07:06.741748+0000","last_fullsized":"2023-10-09T04:07:06.741748+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"127'7283","last_scrub_stamp":"2023-10-09T04:06:42.872198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:42.872198+0000","objects_scrubbed":3197,"log_size":7830,"log_dups_size":0,"ondisk_log_size":7830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:26.895471+0000","scrub_duration":0.738535999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173408256,"num_objects":2646,"num_object_clones":0,"num_object_copies":7938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":551,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":96,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493603+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:06:57.493603+0000","last_peered":"2023-10-09T04:06:57.493603+0000","last_clean":"2023-10-09T04:06:57.493603+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:06:57.493603+0000","last_undegraded":"2023-10-09T04:06:57.493603+0000","last_fullsized":"2023-10-09T04:06:57.493603+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493052+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:06:57.493052+0000","last_peered":"2023-10-09T04:06:57.493052+0000","last_clean":"2023-10-09T04:06:57.493052+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:06:57.493052+0000","last_undegraded":"2023-10-09T04:06:57.493052+0000","last_fullsized":"2023-10-09T04:06:57.493052+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"135'7627","reported_seq":7681,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:07:05.388334+0000","last_change":"2023-10-09T04:06:41.773617+0000","last_active":"2023-10-09T04:07:05.388334+0000","last_peered":"2023-10-09T04:07:05.388334+0000","last_clean":"2023-10-09T04:07:05.388334+0000","last_became_active":"2023-10-09T04:06:35.303590+0000","last_became_peered":"2023-10-09T04:06:35.303590+0000","last_unstale":"2023-10-09T04:07:05.388334+0000","last_undegraded":"2023-10-09T04:07:05.388334+0000","last_fullsized":"2023-10-09T04:07:05.388334+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7627,"log_dups_size":0,"ondisk_log_size":7627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170819584,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4150,"num_write_kb":216192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":79,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.493550+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:06:57.493550+0000","last_peered":"2023-10-09T04:06:57.493550+0000","last_clean":"2023-10-09T04:06:57.493550+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:06:57.493550+0000","last_undegraded":"2023-10-09T04:06:57.493550+0000","last_fullsized":"2023-10-09T04:06:57.493550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492737+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:06:57.492737+0000","last_peered":"2023-10-09T04:06:57.492737+0000","last_clean":"2023-10-09T04:06:57.492737+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:06:57.492737+0000","last_undegraded":"2023-10-09T04:06:57.492737+0000","last_fullsized":"2023-10-09T04:06:57.492737+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":13,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-10-09T04:06:59.183564+0000","last_change":"2023-10-09T04:06:57.499785+0000","last_active":"2023-10-09T04:06:59.183564+0000","last_peered":"2023-10-09T04:06:59.183564+0000","last_clean":"2023-10-09T04:06:59.183564+0000","last_became_active":"2023-10-09T04:06:57.499419+0000","last_became_peered":"2023-10-09T04:06:57.499419+0000","last_unstale":"2023-10-09T04:06:59.183564+0000","last_undegraded":"2023-10-09T04:06:59.183564+0000","last_fullsized":"2023-10-09T04:06:59.183564+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:14.270664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.502892+0000","last_change":"2023-10-09T04:06:45.454363+0000","last_active":"2023-10-09T04:06:57.502892+0000","last_peered":"2023-10-09T04:06:57.502892+0000","last_clean":"2023-10-09T04:06:57.502892+0000","last_became_active":"2023-10-09T04:06:45.454065+0000","last_became_peered":"2023-10-09T04:06:45.454065+0000","last_unstale":"2023-10-09T04:06:57.502892+0000","last_undegraded":"2023-10-09T04:06:57.502892+0000","last_fullsized":"2023-10-09T04:06:57.502892+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:21.580027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":92,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.491635+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:06:57.491635+0000","last_peered":"2023-10-09T04:06:57.491635+0000","last_clean":"2023-10-09T04:06:57.491635+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:06:57.491635+0000","last_undegraded":"2023-10-09T04:06:57.491635+0000","last_fullsized":"2023-10-09T04:06:57.491635+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":93,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-10-09T04:06:57.492400+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:06:57.492400+0000","last_peered":"2023-10-09T04:06:57.492400+0000","last_clean":"2023-10-09T04:06:57.492400+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:06:57.492400+0000","last_undegraded":"2023-10-09T04:06:57.492400+0000","last_fullsized":"2023-10-09T04:06:57.492400+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5346590744,"num_objects":81585,"num_object_clones":0,"num_object_copies":244755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5671,"num_objects_misplaced":7446,"num_objects_unfound":0,"num_objects_dirty":81585,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":94880,"num_write_kb":4543233,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1348,"num_bytes_recovered":88211456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16228691968,"data_stored":16228679752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":201556,"ondisk_log_size":201556,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201455,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2300560,"kb_used_data":1990984,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91444592,"statfs":{"total":95995035648,"available":93639262208,"internally_reserved":0,"allocated":2038767616,"data_stored":2038278633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":0.69499999999999995,"15min":0.57499999999999996},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"last":9.4580000000000002},{"interface":"front","average":{"1min":1.389,"5min":0.69199999999999995,"15min":0.57599999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.1479999999999997,"5min":4.1479999999999997,"15min":4.1479999999999997},"last":0.32100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":0.754,"15min":0.67000000000000004},"min":{"1min":0.248,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6669999999999998,"5min":5.6669999999999998,"15min":5.6669999999999998},"last":2.343},{"interface":"front","average":{"1min":2.3999999999999999,"5min":1.0649999999999999,"15min":0.82299999999999995},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.1989999999999998,"5min":9.1989999999999998,"15min":9.1989999999999998},"last":0.28699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":4.7030000000000003,"5min":1.3779999999999999,"15min":0.82399999999999995},"min":{"1min":0.32500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":58.615000000000002,"5min":58.615000000000002,"15min":58.615000000000002},"last":0.73899999999999999},{"interface":"front","average":{"1min":5.1870000000000003,"5min":1.52,"15min":0.91000000000000003},"min":{"1min":0.38200000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":58.798000000000002,"5min":58.798000000000002,"15min":58.798000000000002},"last":0.72799999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:06:38 2023","interfaces":[{"interface":"back","average":{"1min":8.2929999999999993,"5min":2.5510000000000002,"15min":1.2370000000000001},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":99.134,"5min":99.134,"15min":99.134},"last":0.61599999999999999},{"interface":"front","average":{"1min":8.4909999999999997,"5min":2.6459999999999999,"15min":1.292},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":99.117999999999995,"5min":99.117999999999995,"15min":99.117999999999995},"last":0.63300000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.2650000000000001,"5min":2.2650000000000001,"15min":6.3129999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.61499999999999999,"15min":0.55600000000000005},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":2.3180000000000001,"5min":2.3180000000000001,"15min":3.8769999999999998},"last":0.34699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53400000000000003,"15min":0.52400000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.4059999999999999,"5min":1.4330000000000001,"15min":1.464},"last":0.38},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.63800000000000001,"15min":0.57999999999999996},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":5.1399999999999997,"5min":5.1399999999999997,"15min":5.1399999999999997},"last":0.33300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.59399999999999997,"15min":0.55500000000000005},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.1030000000000002,"5min":2.1030000000000002,"15min":2.1030000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.63500000000000001,"15min":0.59799999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":3.8010000000000002,"5min":3.8010000000000002,"15min":3.8010000000000002},"last":0.49399999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463092,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1990400,"kb_used_data":1684920,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91754752,"statfs":{"total":95995035648,"available":93956866048,"internally_reserved":0,"allocated":1725358080,"data_stored":1724885012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.70499999999999996,"15min":0.59099999999999997},"min":{"1min":0.23000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":14.557,"5min":14.557,"15min":14.557},"last":0.249},{"interface":"front","average":{"1min":2.0960000000000001,"5min":0.84899999999999998,"15min":0.64000000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":14.632999999999999,"5min":14.632999999999999,"15min":14.632999999999999},"last":1.1279999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":0.90600000000000003,"15min":0.65000000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.7110000000000003,"5min":7.7110000000000003,"15min":7.7110000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.0089999999999999,"5min":1.0009999999999999,"15min":0.67200000000000004},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.1680000000000001,"5min":12.616,"15min":12.616},"last":0.74099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":1.125,"15min":0.748},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":42.195999999999998,"5min":42.195999999999998,"15min":42.195999999999998},"last":1.4370000000000001},{"interface":"front","average":{"1min":3.4180000000000001,"5min":1.177,"15min":0.80100000000000005},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":42.165999999999997,"5min":42.165999999999997,"15min":42.165999999999997},"last":1.47}]},{"osd":3,"last update":"Mon Oct 9 04:06:38 2023","interfaces":[{"interface":"back","average":{"1min":5.0519999999999996,"5min":1.9710000000000001,"15min":1.0669999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":37.679000000000002,"5min":38.606999999999999,"15min":38.606999999999999},"last":1.052},{"interface":"front","average":{"1min":4.1529999999999996,"5min":1.7569999999999999,"15min":1.022},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":37.432000000000002,"5min":37.914000000000001,"15min":37.914000000000001},"last":0.48899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.53300000000000003,"15min":0.47899999999999998},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":2.6909999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.444,"5min":0.72899999999999998,"15min":0.56699999999999995},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":6.1100000000000003,"5min":6.1100000000000003,"15min":6.1100000000000003},"last":0.373}]},{"osd":5,"last update":"Mon Oct 9 04:06:48 2023","interfaces":[{"interface":"back","average":{"1min":3.738,"5min":1.208,"15min":0.73499999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28899999999999998},"max":{"1min":54.773000000000003,"5min":54.773000000000003,"15min":54.773000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":4.1609999999999996,"5min":1.3380000000000001,"15min":0.81499999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":54.853999999999999,"5min":54.853999999999999,"15min":54.853999999999999},"last":0.40799999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.64500000000000002,"15min":0.58999999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32300000000000001},"max":{"1min":3.0760000000000001,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.74199999999999999,"15min":0.64700000000000002},"min":{"1min":0.32800000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0579999999999998,"5min":3.0579999999999998,"15min":3.0579999999999998},"last":0.42999999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089401,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1694496,"kb_used_data":1618648,"kb_used_omap":9,"kb_used_meta":75830,"kb_avail":92050656,"statfs":{"total":95995035648,"available":94259871744,"internally_reserved":0,"allocated":1657495552,"data_stored":1656934424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":77650921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":4.1900000000000004,"5min":1.365,"15min":0.70299999999999996},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":45.811999999999998,"5min":45.811999999999998,"15min":45.811999999999998},"last":0.24099999999999999},{"interface":"front","average":{"1min":9.2400000000000002,"5min":2.4100000000000001,"15min":1.0649999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":99.831999999999994,"5min":99.831999999999994,"15min":99.831999999999994},"last":1.3540000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":0.621,"15min":0.436},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.383,"5min":6.383,"15min":6.383},"last":0.372},{"interface":"front","average":{"1min":3.194,"5min":0.94899999999999995,"15min":0.56100000000000005},"min":{"1min":0.33700000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.833,"5min":10.833,"15min":10.833},"last":1.1719999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":6.0659999999999998,"5min":1.8069999999999999,"15min":0.85299999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":42.588999999999999,"5min":42.588999999999999,"15min":42.588999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":11.587999999999999,"5min":2.8650000000000002,"15min":1.21},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":100.02500000000001,"5min":100.02500000000001,"15min":100.02500000000001},"last":1.22}]},{"osd":4,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":6.3179999999999996,"5min":1.9510000000000001,"15min":1.1120000000000001},"min":{"1min":0.33900000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":99.989999999999995,"5min":99.989999999999995,"15min":99.989999999999995},"last":1.242},{"interface":"front","average":{"1min":6.2850000000000001,"5min":1.921,"15min":1.1080000000000001},"min":{"1min":0.41599999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":100.249,"5min":100.249,"15min":100.249},"last":0.82699999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":6.3079999999999998,"5min":1.966,"15min":1.1379999999999999},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":99.933999999999997,"5min":99.933999999999997,"15min":99.933999999999997},"last":1.198},{"interface":"front","average":{"1min":6.2329999999999997,"5min":1.929,"15min":1.127},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":100.34099999999999,"5min":100.34099999999999,"15min":100.34099999999999},"last":0.85799999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":7.6269999999999998,"5min":2.2229999999999999,"15min":1.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":100.309,"5min":100.309,"15min":100.309},"last":0.90900000000000003},{"interface":"front","average":{"1min":7.9160000000000004,"5min":2.3130000000000002,"15min":1.2649999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":100.143,"5min":100.143,"15min":100.143},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":6.3970000000000002,"5min":1.9990000000000001,"15min":1.163},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":100.214,"5min":100.214,"15min":100.214},"last":0.88100000000000001},{"interface":"front","average":{"1min":6.5469999999999997,"5min":2.0649999999999999,"15min":1.2010000000000001},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":100.113,"5min":100.113,"15min":100.113},"last":0.48599999999999999}]}]},{"osd":0,"up_from":84,"seq":360777252927,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1381480,"kb_used_data":1313760,"kb_used_omap":28,"kb_used_meta":67683,"kb_avail":92363672,"statfs":{"total":95995035648,"available":94580400128,"internally_reserved":0,"allocated":1345290240,"data_stored":1344719677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":69308240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:06:42 2023","interfaces":[{"interface":"back","average":{"1min":7.5899999999999999,"5min":1.968,"15min":0.90500000000000003},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":118.48,"5min":118.48,"15min":118.48},"last":6.5759999999999996},{"interface":"front","average":{"1min":8.016,"5min":2.0659999999999998,"15min":0.93100000000000005},"min":{"1min":0.23599999999999999,"5min":0.214,"15min":0.214},"max":{"1min":118.384,"5min":118.384,"15min":118.384},"last":0.64700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:06:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":0.81000000000000005,"15min":0.54800000000000004},"min":{"1min":0.255,"5min":0.224,"15min":0.224},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.1480000000000001,"5min":0.77000000000000002,"15min":0.53600000000000003},"min":{"1min":0.23499999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.0259999999999998,"5min":7.0259999999999998,"15min":7.0259999999999998},"last":4.7539999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:06:42 2023","interfaces":[{"interface":"back","average":{"1min":14.875999999999999,"5min":4.4790000000000001,"15min":1.754},"min":{"1min":0.29799999999999999,"5min":0.249,"15min":0.249},"max":{"1min":118.73999999999999,"5min":118.73999999999999,"15min":118.73999999999999},"last":6.5010000000000003},{"interface":"front","average":{"1min":15.391,"5min":4.7290000000000001,"15min":1.871},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":118.584,"5min":118.584,"15min":118.584},"last":0.61799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.149,"5min":1.9350000000000001,"15min":1.0369999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":118.633,"5min":118.633,"15min":118.633},"last":0.58199999999999996},{"interface":"front","average":{"1min":8.0259999999999998,"5min":2.0950000000000002,"15min":1.1040000000000001},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":118.913,"5min":118.913,"15min":118.913},"last":4.8200000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.9290000000000003,"5min":2.0790000000000002,"15min":1.109},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":118.943,"5min":118.943,"15min":118.943},"last":4.8520000000000003},{"interface":"front","average":{"1min":7.4009999999999998,"5min":1.9910000000000001,"15min":1.089},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":118.898,"5min":118.898,"15min":118.898},"last":6.5430000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.3680000000000003,"5min":1.9970000000000001,"15min":1.1060000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":118.864,"5min":118.864,"15min":118.864},"last":6.6029999999999998},{"interface":"front","average":{"1min":7.1779999999999999,"5min":1.9750000000000001,"15min":1.089},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":118.70399999999999,"5min":118.70399999999999,"15min":118.70399999999999},"last":0.84699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:06:19 2023","interfaces":[{"interface":"back","average":{"1min":7.9400000000000004,"5min":2.1120000000000001,"15min":1.1379999999999999},"min":{"1min":0.41799999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":118.834,"5min":118.834,"15min":118.834},"last":6.6289999999999996},{"interface":"front","average":{"1min":7.5739999999999998,"5min":2.077,"15min":1.137},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":118.66200000000001,"5min":118.66200000000001,"15min":118.66200000000001},"last":0.81499999999999995}]}]},{"osd":2,"up_from":94,"seq":403726925874,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2814420,"kb_used_data":2714636,"kb_used_omap":81,"kb_used_meta":99694,"kb_avail":90930732,"statfs":{"total":95995035648,"available":93113069568,"internally_reserved":0,"allocated":2779787264,"data_stored":2779236450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":102086874},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7310000000000001,"5min":0.63400000000000001,"15min":0.436},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.5700000000000003,"5min":6.5700000000000003,"15min":6.5700000000000003},"last":1.147},{"interface":"front","average":{"1min":2.9199999999999999,"5min":0.89900000000000002,"15min":0.56100000000000005},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":12.619,"5min":12.619,"15min":12.619},"last":1.6060000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":5.492,"5min":1.419,"15min":0.72299999999999998},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":68.009,"5min":68.009,"15min":68.009},"last":1.7050000000000001},{"interface":"front","average":{"1min":5.9630000000000001,"5min":1.518,"15min":0.75},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":66.975999999999999,"5min":66.975999999999999,"15min":66.975999999999999},"last":3.1789999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:06:40 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":1.4610000000000001,"15min":0.74199999999999999},"min":{"1min":0.39300000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.6269999999999998,"5min":27.562000000000001,"15min":27.562000000000001},"last":1.1160000000000001},{"interface":"front","average":{"1min":3.706,"5min":1.4570000000000001,"15min":0.73899999999999999},"min":{"1min":0.36699999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":12.662000000000001,"5min":27.710999999999999,"15min":27.710999999999999},"last":1.367}]},{"osd":4,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":0.97199999999999998,"15min":0.71999999999999997},"min":{"1min":0.41799999999999998,"5min":0.375,"15min":0.375},"max":{"1min":10.553000000000001,"5min":10.553000000000001,"15min":10.553000000000001},"last":3.1190000000000002},{"interface":"front","average":{"1min":1.377,"5min":0.75900000000000001,"15min":0.65000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.7930000000000001,"5min":3.7930000000000001,"15min":3.7930000000000001},"last":1.1779999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":0.86299999999999999,"15min":0.72399999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.9029999999999996,"5min":5.9029999999999996,"15min":5.9029999999999996},"last":1.45},{"interface":"front","average":{"1min":2.4409999999999998,"5min":0.99299999999999999,"15min":0.76100000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":10.507999999999999,"5min":10.507999999999999,"15min":10.507999999999999},"last":3.149}]},{"osd":6,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":0.79700000000000004,"15min":0.68500000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":3.746,"5min":3.746,"15min":3.746},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.8129999999999999,"5min":0.90700000000000003,"15min":0.755},"min":{"1min":0.38900000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.8780000000000001,"5min":5.8780000000000001,"15min":5.8780000000000001},"last":1.4279999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:09 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":0.82799999999999996,"15min":0.72199999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.714,"5min":3.714,"15min":3.714},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.873,"5min":0.94299999999999995,"15min":0.78800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.8300000000000001,"5min":5.8300000000000001,"15min":5.8300000000000001},"last":1.403}]}]},{"osd":3,"up_from":99,"seq":425201762348,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3448516,"kb_used_data":3308284,"kb_used_omap":7,"kb_used_meta":139832,"kb_avail":90296636,"statfs":{"total":95995035648,"available":92463755264,"internally_reserved":0,"allocated":3387682816,"data_stored":3387145563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":143188584},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":8.5969999999999995,"5min":4.4450000000000003,"15min":2.0910000000000002},"min":{"1min":0.27300000000000002,"5min":0.185,"15min":0.185},"max":{"1min":88.719999999999999,"5min":106.22,"15min":106.22},"last":4.1779999999999999},{"interface":"front","average":{"1min":7.3250000000000002,"5min":4.2430000000000003,"15min":2.0230000000000001},"min":{"1min":0.224,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":88.385000000000005,"5min":106.986,"15min":106.986},"last":4.0599999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":5.9180000000000001,"5min":2.8490000000000002,"15min":1.169},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.677000000000007,"5min":106.121,"15min":106.121},"last":2.8399999999999999},{"interface":"front","average":{"1min":8.2390000000000008,"5min":2.0390000000000001,"15min":0.91300000000000003},"min":{"1min":0.34100000000000003,"5min":0.22,"15min":0.22},"max":{"1min":65.015000000000001,"5min":65.015000000000001,"15min":65.015000000000001},"last":3.5009999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":8.032,"5min":3.3199999999999998,"15min":1.3520000000000001},"min":{"1min":0.25800000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":85.974000000000004,"5min":106.105,"15min":106.105},"last":2.4119999999999999},{"interface":"front","average":{"1min":8.859,"5min":3.472,"15min":1.3979999999999999},"min":{"1min":0.33900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":85.912999999999997,"5min":106.70399999999999,"15min":106.70399999999999},"last":2.2570000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":2.2810000000000001,"15min":1.155},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":6.9340000000000002,"5min":106.85899999999999,"15min":106.85899999999999},"last":2.0710000000000002},{"interface":"front","average":{"1min":1.946,"5min":0.93600000000000005,"15min":0.70899999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.1509999999999998,"5min":6.1509999999999998,"15min":6.1509999999999998},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.0209999999999999,"15min":0.73599999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":7.5179999999999998,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.1509999999999998,"5min":1.4470000000000001,"15min":1.2350000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":12.817,"5min":12.817,"15min":12.817},"last":2.3860000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0129999999999999,"5min":2.7429999999999999,"15min":1.655},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.762,"5min":106.036,"15min":106.036},"last":2.351},{"interface":"front","average":{"1min":2.4670000000000001,"5min":2.367,"15min":1.1990000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":6.8520000000000003,"5min":106.741,"15min":106.741},"last":2.1480000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.339,"5min":2.339,"15min":1.1990000000000001},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.758,"5min":106.82599999999999,"15min":106.82599999999999},"last":2.1099999999999999},{"interface":"front","average":{"1min":2.4870000000000001,"5min":2.3279999999999998,"15min":1.1759999999999999},"min":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":6.7519999999999998,"5min":106.084,"15min":106.084},"last":0.40999999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986364,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1947652,"kb_used_data":1642172,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91797500,"statfs":{"total":95995035648,"available":94000640000,"internally_reserved":0,"allocated":1681584128,"data_stored":1681106554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":0.69299999999999995,"15min":0.58199999999999996},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.9900000000000002,"5min":8.9900000000000002,"15min":8.9900000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.1200000000000001,"5min":0.86099999999999999,"15min":0.66900000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.1370000000000005,"5min":8.1370000000000005,"15min":8.1370000000000005},"last":2.8660000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7240000000000002,"5min":1.002,"15min":0.70099999999999996},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":10.105,"5min":10.105,"15min":10.105},"last":3.593},{"interface":"front","average":{"1min":2.1699999999999999,"5min":0.90800000000000003,"15min":0.65900000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.3849999999999998,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.315}]},{"osd":2,"last update":"Mon Oct 9 04:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":0.85199999999999998,"15min":0.67100000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":8.8680000000000003,"5min":8.8680000000000003,"15min":8.8680000000000003},"last":5.9900000000000002},{"interface":"front","average":{"1min":1.331,"5min":0.71999999999999997,"15min":0.61399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.855,"5min":3.855,"15min":3.855},"last":6.0060000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":0.88300000000000001,"15min":0.65500000000000003},"min":{"1min":0.36799999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.0170000000000003,"5min":5.0170000000000003,"15min":5.0170000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.1519999999999999,"15min":0.75600000000000001},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.0329999999999995,"5min":9.0329999999999995,"15min":9.0329999999999995},"last":9.0449999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.54700000000000004,"15min":0.48399999999999999},"min":{"1min":0.30299999999999999,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":2.8650000000000002,"5min":2.8650000000000002,"15min":2.8650000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.5,"5min":0.71699999999999997,"15min":0.56299999999999994},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.24399999999999999},"max":{"1min":5.032,"5min":5.032,"15min":5.032},"last":0.44400000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":0.73799999999999999,"15min":0.59899999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.9960000000000004,"5min":4.9960000000000004,"15min":4.9960000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.995,"5min":0.63200000000000001,"15min":0.56399999999999995},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":4.3280000000000003},"last":0.59599999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:06:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.58899999999999997,"15min":0.55300000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.504},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.69499999999999995,"15min":0.629},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.32500000000000001},"max":{"1min":5.5819999999999999,"5min":5.5819999999999999,"15min":5.5819999999999999},"last":0.48099999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724728,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969720,"kb_used_data":1664240,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91775432,"statfs":{"total":95995035648,"available":93978042368,"internally_reserved":0,"allocated":1704181760,"data_stored":1703712184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.64700000000000002,"15min":0.61399999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.5339999999999998,"5min":4.5339999999999998,"15min":4.5339999999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.575,"5min":0.75,"15min":0.629},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":1.1890000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:06:36 2023","interfaces":[{"interface":"back","average":{"1min":7.0309999999999997,"5min":1.8899999999999999,"15min":0.99199999999999999},"min":{"1min":0.28699999999999998,"5min":0.249,"15min":0.249},"max":{"1min":91.212999999999994,"5min":91.212999999999994,"15min":91.212999999999994},"last":0.40400000000000003},{"interface":"front","average":{"1min":8.5909999999999993,"5min":2.2109999999999999,"15min":1.1200000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":91.254000000000005,"5min":91.254000000000005,"15min":91.254000000000005},"last":1.028}]},{"osd":2,"last update":"Mon Oct 9 04:06:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":0.90500000000000003,"15min":0.68700000000000006},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.6880000000000006,"5min":9.6880000000000006,"15min":9.6880000000000006},"last":0.23499999999999999},{"interface":"front","average":{"1min":1.8859999999999999,"5min":0.81100000000000005,"15min":0.629},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.484,"5min":9.484,"15min":9.484},"last":0.28999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:06:43 2023","interfaces":[{"interface":"back","average":{"1min":7.5810000000000004,"5min":2.746,"15min":1.306},"min":{"1min":0.47299999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":91.834999999999994,"5min":91.834999999999994,"15min":91.834999999999994},"last":0.65500000000000003},{"interface":"front","average":{"1min":7.5039999999999996,"5min":2.7440000000000002,"15min":1.319},"min":{"1min":0.46300000000000002,"5min":0.376,"15min":0.376},"max":{"1min":91.802000000000007,"5min":91.802000000000007,"15min":91.802000000000007},"last":0.67300000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.61399999999999999,"15min":0.52000000000000002},"min":{"1min":0.27000000000000002,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":4.9539999999999997,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":0.317},{"interface":"front","average":{"1min":1.327,"5min":0.68999999999999995,"15min":0.56599999999999995},"min":{"1min":0.30199999999999999,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":5.2000000000000002,"5min":5.2000000000000002,"15min":5.2000000000000002},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":0.69899999999999995,"15min":0.57399999999999995},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":4.056,"5min":4.056,"15min":4.056},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.327,"5min":0.75800000000000001,"15min":0.61099999999999999},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":4.3099999999999996,"5min":4.3099999999999996,"15min":4.3099999999999996},"last":0.42199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":0.78600000000000003,"15min":0.64600000000000002},"min":{"1min":0.371,"5min":0.34599999999999997,"15min":0.317},"max":{"1min":8.3130000000000006,"5min":8.3130000000000006,"15min":8.3130000000000006},"last":0.442},{"interface":"front","average":{"1min":1.8740000000000001,"5min":0.90800000000000003,"15min":0.70499999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34599999999999997},"max":{"1min":8.5329999999999995,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":0.44900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1331433472,"data_stored":1331429400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1640042496,"data_stored":1640038424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2768568320,"data_stored":2768568320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3382378496,"data_stored":3382378496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1671626752,"data_stored":1671626752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1694699520,"data_stored":1694695448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1714552832,"data_stored":1714552832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2025390080,"data_stored":2025390080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:07:13.907 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 26 2023-10-09T04:07:13.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:13.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:13.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:14.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:14.087 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:14.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:14.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:14.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:14.203 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:14.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:14.369 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:14.553 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:14.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:14.604 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:14 smithi163 ceph-mon[213841]: pgmap v341: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 5671/234300 objects degraded (2.420%); 7068/234300 objects misplaced (3.017%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:07:14.604 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:14 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:07:14.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:14 smithi163 ceph-mon[221067]: pgmap v341: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 5671/234300 objects degraded (2.420%); 7068/234300 objects misplaced (3.017%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:07:14.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:14 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:07:14.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:14 smithi181 ceph-mon[196979]: pgmap v341: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 5671/234300 objects degraded (2.420%); 7068/234300 objects misplaced (3.017%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:07:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:14 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:07:14.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:14.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:07:14.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:14.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:14.986 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:07:15.026 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:07:15.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:07:15.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:15.100 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:07:15.142 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:07:15.162 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:15.201 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:07:15.255 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:07:15.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.270 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.271 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:07:15.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:15.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:15.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:15.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:15.415 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:15.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:15.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:15.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:15.592 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3033751426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:15.593 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3033751426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:15.669 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3033751426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:16.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:16.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:16.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:16.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:16 smithi163 ceph-mon[221067]: pgmap v342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 5380/229029 objects degraded (2.349%); 6922/229029 objects misplaced (3.022%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:07:16.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:16 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:07:16.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:16 smithi163 ceph-mon[213841]: pgmap v342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 5380/229029 objects degraded (2.349%); 6922/229029 objects misplaced (3.022%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:07:16.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:16 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:07:16.727 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:16.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:16 smithi181 ceph-mon[196979]: pgmap v342: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 5380/229029 objects degraded (2.349%); 6922/229029 objects misplaced (3.022%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:07:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:16 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:07:16.976 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:16.978 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:17.106 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:17.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:17.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:17.303 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:17.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:17.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:17.536 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5380/229029 objects degraded (2.349%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:17.537 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5380/229029 objects degraded (2.349%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:17.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:17.676 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:17.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:17.739 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5380/229029 objects degraded (2.349%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:17.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:17.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:17.909 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:18.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:18 smithi181 ceph-mon[196979]: pgmap v343: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 5380/222840 objects degraded (2.414%); 6596/222840 objects misplaced (2.960%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:07:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:18 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:07:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1949192305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[213841]: pgmap v343: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 5380/222840 objects degraded (2.414%); 6596/222840 objects misplaced (2.960%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:07:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:07:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1949192305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[221067]: pgmap v343: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 5380/222840 objects degraded (2.414%); 6596/222840 objects misplaced (2.960%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:07:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:07:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1949192305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:18.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:18.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:18.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:18.996 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:19.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:19.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:19.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:19.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:19.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:07:19.225 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:07:19.308 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:07:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:07:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[213841]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T04:07:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[213841]: failed to encode map e137 with expected crc 2023-10-09T04:07:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:19.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:07:19.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:07:19.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[221067]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T04:07:19.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[221067]: failed to encode map e137 with expected crc 2023-10-09T04:07:19.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:19.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:19 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:07:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:07:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:19 smithi181 ceph-mon[196979]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T04:07:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:19 smithi181 ceph-mon[196979]: failed to encode map e137 with expected crc 2023-10-09T04:07:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:20.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:20.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:20.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:20.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:20 smithi163 ceph-mon[221067]: failed to encode map e137 with expected crc 2023-10-09T04:07:20.721 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:20 smithi163 ceph-mon[221067]: pgmap v345: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5149/221382 objects degraded (2.326%); 6481/221382 objects misplaced (2.928%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:07:20.725 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:07:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:07:20] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-10-09T04:07:20.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:20 smithi163 ceph-mon[213841]: failed to encode map e137 with expected crc 2023-10-09T04:07:20.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:20 smithi163 ceph-mon[213841]: pgmap v345: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5149/221382 objects degraded (2.326%); 6481/221382 objects misplaced (2.928%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:07:20.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:20 smithi181 ceph-mon[196979]: failed to encode map e137 with expected crc 2023-10-09T04:07:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:20 smithi181 ceph-mon[196979]: pgmap v345: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 5149/221382 objects degraded (2.326%); 6481/221382 objects misplaced (2.928%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:07:20.772 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:20.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:20.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:20.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:20.974 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:21.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:21.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:21.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:21.141 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:21.181 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:21.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:21.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:21.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:21.349 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:21.438 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:07:21.438 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:07:21.465 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:21.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:21.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:21.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:21.891 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:22.481 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[213841]: pgmap v346: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 5149/212649 objects degraded (2.421%); 6481/212649 objects misplaced (3.048%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:07:22.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2948521612' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:07:22.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:07:22.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:07:22.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:07:22.482 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5149/212649 objects degraded (2.421%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:22.506 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 6 pgp_num to 26 2023-10-09T04:07:22.755 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:07:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:22 smithi181 ceph-mon[196979]: pgmap v346: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 5149/212649 objects degraded (2.421%); 6481/212649 objects misplaced (3.048%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:07:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2948521612' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:07:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:22 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:07:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:22 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:07:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:22 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:07:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5149/212649 objects degraded (2.421%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[221067]: pgmap v346: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 5149/212649 objects degraded (2.421%); 6481/212649 objects misplaced (3.048%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:07:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2948521612' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:07:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:07:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:07:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:07:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5149/212649 objects degraded (2.421%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:23.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:23.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:23.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:23.253 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:23.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:23.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:23.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:23.478 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:23.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:07:23.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T04:07:23.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.499 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.499 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.499 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:07:23.499 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[213841]: pgmap v348: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5149/211161 objects degraded (2.438%); 6093/211161 objects misplaced (2.885%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T04:07:23.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:07:23.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T04:07:23.526 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:07:23.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:23 smithi181 ceph-mon[196979]: pgmap v348: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5149/211161 objects degraded (2.438%); 6093/211161 objects misplaced (2.885%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T04:07:23.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:23.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:23.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:23.723 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:23.807 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:23.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:23.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:23.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:07:23.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T04:07:23.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: failed to encode map e138 with expected crc 2023-10-09T04:07:23.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:07:23.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:23 smithi163 ceph-mon[221067]: pgmap v348: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 5149/211161 objects degraded (2.438%); 6093/211161 objects misplaced (2.885%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T04:07:23.914 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:23.954 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:07:23.955 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:24.005 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:24.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:24.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:24.144 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:24.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:24.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:24.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:24.415 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:24.571 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:24.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:24.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:24.724 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:24.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:24.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:24.802 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:24.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:24.926 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:24.943 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:25.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:25.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:24 smithi181 ceph-mon[196979]: 6.16 deep-scrub starts 2023-10-09T04:07:25.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:25.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:25.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:25.160 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:24 smithi163 ceph-mon[213841]: 6.16 deep-scrub starts 2023-10-09T04:07:25.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:24 smithi163 ceph-mon[221067]: 6.16 deep-scrub starts 2023-10-09T04:07:25.212 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:25.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:25.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:25.538 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:07:25.554 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:25.600 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:07:25.649 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:07:25.655 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:25.717 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:07:25.767 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:07:25.799 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:07:25.873 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:07:25.915 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:07:25.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.932 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:07:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:25.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:26.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:26.071 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:26.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:25 smithi163 ceph-mon[213841]: 6.15 scrub starts 2023-10-09T04:07:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:25 smithi163 ceph-mon[213841]: pgmap v349: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4900/206094 objects degraded (2.378%); 5968/206094 objects misplaced (2.896%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:07:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3165826647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:25 smithi163 ceph-mon[221067]: 6.15 scrub starts 2023-10-09T04:07:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:25 smithi163 ceph-mon[221067]: pgmap v349: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4900/206094 objects degraded (2.378%); 5968/206094 objects misplaced (2.896%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:07:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3165826647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:26.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:26.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:26.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:25 smithi181 ceph-mon[196979]: 6.15 scrub starts 2023-10-09T04:07:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:25 smithi181 ceph-mon[196979]: pgmap v349: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4900/206094 objects degraded (2.378%); 5968/206094 objects misplaced (2.896%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:07:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3165826647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:26.285 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:26.352 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:26.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:26.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:26.507 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:26.585 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:26.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:26.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:26.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:26.723 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:27.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:27 smithi163 ceph-mon[213841]: 6.13 scrub starts 2023-10-09T04:07:27.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:27 smithi163 ceph-mon[221067]: 6.13 scrub starts 2023-10-09T04:07:27.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:27 smithi181 ceph-mon[196979]: 6.13 scrub starts 2023-10-09T04:07:28.085 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:28.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:28.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:28.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:28.259 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:28.329 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:28.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4900/206094 objects degraded (2.378%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[213841]: pgmap v350: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 4900/199497 objects degraded (2.456%); 5560/199497 objects misplaced (2.787%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[213841]: 6.13 scrub ok 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4900/206094 objects degraded (2.378%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[221067]: pgmap v350: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 4900/199497 objects degraded (2.456%); 5560/199497 objects misplaced (2.787%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[221067]: 6.13 scrub ok 2023-10-09T04:07:28.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:28 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:28.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:28 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4900/206094 objects degraded (2.378%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:28.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:28 smithi181 ceph-mon[196979]: pgmap v350: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 4900/199497 objects degraded (2.456%); 5560/199497 objects misplaced (2.787%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:07:28.340 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:28 smithi181 ceph-mon[196979]: 6.13 scrub ok 2023-10-09T04:07:28.340 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:28 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:28.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:28.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:28.481 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:28.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:28.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:28.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:28.732 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:28.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:28.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:28.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:29.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:29.090 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:29.386 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:29.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:29.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:29.540 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:29.591 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:07:29.591 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:07:29.591 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:07:29.591 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:07:29.591 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:07:29.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:07:29.655 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:29.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:29.748 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:07:29.748 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:07:29.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.748 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:153259 6.11 6:8d29d995:::benchmark_data_smithi163_287034_object44996:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:29.695520+0000", 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.051956385000000001, 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.051985709999999997, 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 153259 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:29.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.695520+0000", 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.695520+0000", 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:29.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.695519+0000", 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.695523+0000", 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9500000000000003e-06 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.695533+0000", 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4709999999999996e-06 2023-10-09T04:07:29.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.695539+0000", 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6989999999999997e-06 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.700130+0000", 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045902959999999998 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.701073+0000", 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00094374900000000002 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.720396+0000", 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019323152999999999 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.720508+0000", 2023-10-09T04:07:29.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011178700000000001 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.720905+0000", 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00039675700000000002 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.732083+0000", 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011177650000000001 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.742635+0000", 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010552182 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:153245 6.11 6:8ff5b12d:::benchmark_data_smithi163_287034_object44982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:29.678426+0000", 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.069050466000000005, 2023-10-09T04:07:29.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.069149778999999995, 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 153245 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.757 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678426+0000", 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678426+0000", 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678424+0000", 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678432+0000", 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1540000000000002e-06 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678446+0000", 2023-10-09T04:07:29.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.394e-05 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678455+0000", 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.081e-06 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.679249+0000", 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00079435699999999998 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.681131+0000", 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001881387 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.724282+0000", 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.043151564000000003 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:29.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.724385+0000", 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102524 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.724740+0000", 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00035499899999999999 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.732089+0000", 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0073495840000000002 2023-10-09T04:07:29.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.742669+0000", 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010579242000000001 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:153277 6.d e138/102 6:bfc09ebb:::benchmark_data_smithi163_287034_object45014:head v 138'9653, mlcod=138'9651)", 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:29.737223+0000", 2023-10-09T04:07:29.763 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.010252862999999999, 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010411729, 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.737223+0000", 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:29.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.737223+0000", 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.737217+0000", 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999938 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.737236+0000", 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8510999999999999e-05 2023-10-09T04:07:29.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.737253+0000", 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7487999999999999e-05 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.737263+0000", 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8390000000000002e-06 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.740579+0000", 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033164039999999998 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.740604+0000", 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4641e-05 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:29.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:153248 6.11 6:8ae94871:::benchmark_data_smithi163_287034_object44985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:29.678482+0000", 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.068994045000000004, 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.069190808000000006, 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:29.768 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 153248 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678482+0000", 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678482+0000", 2023-10-09T04:07:29.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678481+0000", 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678485+0000", 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0810000000000004e-06 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678493+0000", 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6190000000000002e-06 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.678499+0000", 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1589999999999998e-06 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.681245+0000", 2023-10-09T04:07:29.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002745333 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.691125+0000", 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0098802309999999997 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.715359+0000", 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024233958 2023-10-09T04:07:29.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.715463+0000", 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000104454 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.715843+0000", 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00037958700000000002 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:29.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.726631+0000", 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010787966 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.732073+0000", 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054424069999999998 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.774 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:153283 6.d e138/102 6:bef8d756:::benchmark_data_smithi163_287034_object45020:head v 138'9654, mlcod=138'9651)", 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:29.739831+0000", 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0076454950000000004, 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078995079999999995, 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.739831+0000", 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.739831+0000", 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.739829+0000", 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.739840+0000", 2023-10-09T04:07:29.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0360000000000001e-05 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.739859+0000", 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9595000000000001e-05 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.739868+0000", 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0229999999999998e-06 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.745584+0000", 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057162960000000001 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:29.745621+0000", 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6507000000000001e-05 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.778 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:29.779 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:07:29.779 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 5 2023-10-09T04:07:29.779 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:07:29.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:29.859 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:30.019 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:30.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:30.129 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:30.169 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:30.234 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:07:30.254 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:30.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:30.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:30.398 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:30.426 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:30 smithi181 ceph-mon[196979]: pgmap v351: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 4630/198087 objects degraded (2.337%); 5425/198087 objects misplaced (2.739%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:07:30.426 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2861903725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:07:30.426 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:30 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:07:30.426 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:07:30.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:30.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:30.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:07:30.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:30 smithi163 ceph-mon[221067]: pgmap v351: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 4630/198087 objects degraded (2.337%); 5425/198087 objects misplaced (2.739%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:07:30.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2861903725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:07:30.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:30 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:07:30.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:30 smithi163 ceph-mon[213841]: pgmap v351: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 4630/198087 objects degraded (2.337%); 5425/198087 objects misplaced (2.739%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:07:30.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2861903725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:07:30.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:30 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:07:30.683 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:07:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:07:30] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-10-09T04:07:30.685 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:30.713 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:31.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:31.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:31.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:31.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:07:31.677 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:31.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:31.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2861903725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:07:31.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[213841]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T04:07:31.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[213841]: failed to encode map e139 with expected crc 2023-10-09T04:07:31.729 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[213841]: failed to encode map e139 with expected crc 2023-10-09T04:07:31.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:31.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2861903725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:07:31.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[221067]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T04:07:31.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[221067]: failed to encode map e139 with expected crc 2023-10-09T04:07:31.729 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:31 smithi163 ceph-mon[221067]: failed to encode map e139 with expected crc 2023-10-09T04:07:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:31 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2861903725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:07:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:31 smithi181 ceph-mon[196979]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T04:07:31.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:31 smithi181 ceph-mon[196979]: failed to encode map e139 with expected crc 2023-10-09T04:07:31.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:31 smithi181 ceph-mon[196979]: failed to encode map e139 with expected crc 2023-10-09T04:07:31.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:31.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:31.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:32.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:32.164 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:32.280 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:32.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:32.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:32.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:32.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:32.485 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:32.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:32 smithi163 ceph-mon[213841]: pgmap v353: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 4630/189861 objects degraded (2.439%); 5425/189861 objects misplaced (2.857%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:07:32.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4630/189861 objects degraded (2.439%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:32 smithi163 ceph-mon[221067]: pgmap v353: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 4630/189861 objects degraded (2.439%); 5425/189861 objects misplaced (2.857%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:07:32.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4630/189861 objects degraded (2.439%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:32.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:32.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:32.697 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:32 smithi181 ceph-mon[196979]: pgmap v353: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 4630/189861 objects degraded (2.439%); 5425/189861 objects misplaced (2.857%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:07:32.698 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4630/189861 objects degraded (2.439%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:32.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:32.763 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:32.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:32.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:32.960 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:32.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:33.010 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:33.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:33.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:33.382 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:33.463 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:33.546 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:07:33.546 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:07:33.547 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:07:33.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:07:34.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:34.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:34.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:34.418 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:34.666 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[213841]: pgmap v354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 272 op/s; 4630/188520 objects degraded (2.456%); 5009/188520 objects misplaced (2.657%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:07:34.666 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[221067]: pgmap v354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 272 op/s; 4630/188520 objects degraded (2.456%); 5009/188520 objects misplaced (2.657%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:34 smithi181 ceph-mon[196979]: pgmap v354: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 272 op/s; 4630/188520 objects degraded (2.456%); 5009/188520 objects misplaced (2.657%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:07:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:34.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:34.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:35.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:35.090 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:35.634 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:07:35.634 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:07:35.682 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:07:35.717 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:07:35.757 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:07:35.801 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:07:35.861 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:07:35.862 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:07:36.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:36.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:36.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:36.317 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:36.395 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:36.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:36.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:36.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:36.542 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:36.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:36 smithi163 ceph-mon[221067]: pgmap v355: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 4388/183879 objects degraded (2.386%); 4888/183879 objects misplaced (2.658%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:07:36.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3625594978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:36 smithi163 ceph-mon[213841]: pgmap v355: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 4388/183879 objects degraded (2.386%); 4888/183879 objects misplaced (2.658%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:07:36.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3625594978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:36.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:36.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:36 smithi181 ceph-mon[196979]: pgmap v355: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 4388/183879 objects degraded (2.386%); 4888/183879 objects misplaced (2.658%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:07:36.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3625594978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:36.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:36.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:36.798 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:37.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:37.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:37.278 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:37.288 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:37.405 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:37.575 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4388/183879 objects degraded (2.386%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:37.575 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4388/183879 objects degraded (2.386%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:37.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:37.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:37.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:37.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4388/183879 objects degraded (2.386%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:37.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:37.940 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:07:37.940 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:07:37.992 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:38.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:38.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:38.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:38.281 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:38.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:38.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:38.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:38.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:38 smithi163 ceph-mon[213841]: pgmap v356: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 4388/177762 objects degraded (2.468%); 4394/177762 objects misplaced (2.472%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T04:07:38.475 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:38.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:38.540 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:38 smithi181 ceph-mon[196979]: pgmap v356: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 4388/177762 objects degraded (2.468%); 4394/177762 objects misplaced (2.472%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T04:07:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:38.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:38.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:38.679 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:38.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:38.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:38.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:38 smithi163 ceph-mon[221067]: pgmap v356: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 4388/177762 objects degraded (2.468%); 4394/177762 objects misplaced (2.472%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T04:07:38.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:38.884 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:38.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:38.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:38.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:39.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:39.143 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:39.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:07:39.373 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:39.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:39.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:39.520 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:39.525 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:07:39.526 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:07:39.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:39.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:39.686 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:39.788 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:39.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:07:39.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:40.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:40.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:40.135 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:40.189 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:40.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:40.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:40.327 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:40.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:40.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:40.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:40.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:40.599 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:40.702 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:40 smithi181 ceph-mon[196979]: pgmap v357: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 4157/176355 objects degraded (2.357%); 4279/176355 objects misplaced (2.426%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T04:07:40.838 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:07:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:07:40] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-10-09T04:07:40.841 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:40 smithi163 ceph-mon[213841]: pgmap v357: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 4157/176355 objects degraded (2.357%); 4279/176355 objects misplaced (2.426%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T04:07:40.841 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:40 smithi163 ceph-mon[221067]: pgmap v357: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 4157/176355 objects degraded (2.357%); 4279/176355 objects misplaced (2.426%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T04:07:40.870 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:40.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:40.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:41.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:41.047 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:41.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:41.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:41.208 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:41.328 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:41.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:41.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:41.572 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:41.620 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:41.686 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:41.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:41.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:41.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:41.998 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:42.011 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:07:42.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:42.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:42.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:42.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:42.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/832866325' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:42.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:42.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[213841]: pgmap v358: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 317 op/s; 4157/167913 objects degraded (2.476%); 4279/167913 objects misplaced (2.548%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T04:07:42.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:42.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:42.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/832866325' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:42.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:42.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:41 smithi163 ceph-mon[221067]: pgmap v358: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 317 op/s; 4157/167913 objects degraded (2.476%); 4279/167913 objects misplaced (2.548%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T04:07:42.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:42.221 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:42.221 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:42.221 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/832866325' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:42.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:41 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:07:42.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:41 smithi181 ceph-mon[196979]: pgmap v358: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 317 op/s; 4157/167913 objects degraded (2.476%); 4279/167913 objects misplaced (2.548%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T04:07:42.224 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:42.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:42.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:42.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:42.465 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4157/167913 objects degraded (2.476%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:43.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:43.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:43.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:43.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4157/167913 objects degraded (2.476%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:43.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: failed to encode map e140 with expected crc 2023-10-09T04:07:43.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4157/167913 objects degraded (2.476%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:07:44.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:44.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:44.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:43 smithi163 ceph-mon[213841]: failed to encode map e140 with expected crc 2023-10-09T04:07:44.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:43 smithi163 ceph-mon[213841]: failed to encode map e140 with expected crc 2023-10-09T04:07:44.214 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:43 smithi163 ceph-mon[213841]: pgmap v360: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 4157/166494 objects degraded (2.497%); 3775/166494 objects misplaced (2.267%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T04:07:44.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:43 smithi163 ceph-mon[221067]: failed to encode map e140 with expected crc 2023-10-09T04:07:44.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:43 smithi163 ceph-mon[221067]: failed to encode map e140 with expected crc 2023-10-09T04:07:44.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:43 smithi163 ceph-mon[221067]: pgmap v360: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 4157/166494 objects degraded (2.497%); 3775/166494 objects misplaced (2.267%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T04:07:44.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:43 smithi181 ceph-mon[196979]: failed to encode map e140 with expected crc 2023-10-09T04:07:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:43 smithi181 ceph-mon[196979]: failed to encode map e140 with expected crc 2023-10-09T04:07:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:43 smithi181 ceph-mon[196979]: pgmap v360: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 4157/166494 objects degraded (2.497%); 3775/166494 objects misplaced (2.267%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T04:07:44.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:44.386 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:44.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:44.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:44.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:44.737 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:45.075 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:45.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:45.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:45.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:45.262 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:46.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:07:46.088 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:46.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:46.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:46.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:46.272 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:46.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:46.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:46.446 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:07:46.449 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:07:46.484 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:07:46.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:46.524 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:07:46.543 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:46.576 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:07:46.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:46.619 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:07:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:46.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:46.663 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:07:46.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[213841]: pgmap v361: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 3936/161496 objects degraded (2.437%); 3664/161496 objects misplaced (2.269%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:07:46.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2194322270' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:07:46.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:07:46.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/722332815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:46.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[221067]: pgmap v361: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 3936/161496 objects degraded (2.437%); 3664/161496 objects misplaced (2.269%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:07:46.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2194322270' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:07:46.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:07:46.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/722332815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:46.699 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:07:46.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:46.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:46 smithi181 ceph-mon[196979]: pgmap v361: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 3936/161496 objects degraded (2.437%); 3664/161496 objects misplaced (2.269%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:07:46.755 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2194322270' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:07:46.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:46 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:07:46.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/722332815' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:46.756 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:07:46.758 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:07:46.824 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:46.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:46.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:47.026 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:47.058 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:07:47.058 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:07:47.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:07:47.058 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:07:47.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.058 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119029 6.11 6:89ce47bf:::benchmark_data_smithi163_287034_object10766:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.718667787000001, 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.554635572, 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119029 2023-10-09T04:07:47.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335398+0000", 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335406+0000", 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7929999999999998e-06 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.363872+0000", 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028466115 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.363891+0000", 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8961e-05 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364339+0000", 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000447935 2023-10-09T04:07:47.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364350+0000", 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0750999999999999e-05 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.862665+0000", 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.498315641 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.863639+0000", 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097311200000000004 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877198+0000", 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013559584 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877283+0000", 2023-10-09T04:07:47.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5289000000000003e-05 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877425+0000", 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000141619 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.881660+0000", 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042347840000000001 2023-10-09T04:07:47.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882652+0000", 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0009924109999999999 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882695+0000", 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2449e-05 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.887923+0000", 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052283570000000003 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.887979+0000", 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5566000000000003e-05 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.890035+0000", 2023-10-09T04:07:47.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002056745 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119035 6.11 6:88144e9f:::benchmark_data_smithi163_287034_object10772:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.689921468999998, 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.535802179, 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.068 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119035 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364145+0000", 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364152+0000", 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9800000000000001e-06 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364162+0000", 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5899999999999997e-06 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364169+0000", 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3880000000000002e-06 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364483+0000", 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031385899999999999 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364491+0000", 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.867872+0000", 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.503381112 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.871582+0000", 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037102459999999999 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.885871+0000", 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014288876000000001 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.886009+0000", 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013767400000000001 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.886197+0000", 2023-10-09T04:07:47.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188591 2023-10-09T04:07:47.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.890058+0000", 2023-10-09T04:07:47.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038603420000000001 2023-10-09T04:07:47.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897207+0000", 2023-10-09T04:07:47.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071496529999999997 2023-10-09T04:07:47.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897263+0000", 2023-10-09T04:07:47.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5933999999999997e-05 2023-10-09T04:07:47.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897307+0000", 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3574e-05 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897386+0000", 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.928e-05 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.899948+0000", 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002562174 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119089 6.11 6:8c811572:::benchmark_data_smithi163_287034_object10826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.524292690999999, 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.375641546, 2023-10-09T04:07:47.083 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119089 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:07:47.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:47.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529777+0000", 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.531e-06 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529784+0000", 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1779999999999997e-06 2023-10-09T04:07:47.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529789+0000", 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0509999999999996e-06 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529844+0000", 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4928e-05 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529854+0000", 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9369999999999999e-06 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.881692+0000", 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3518380210000001 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882588+0000", 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000895884 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895389+0000", 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012800420999999999 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895468+0000", 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9363000000000006e-05 2023-10-09T04:07:47.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895563+0000", 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5142000000000005e-05 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903262+0000", 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076987920000000003 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903275+0000", 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2758999999999999e-05 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903318+0000", 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3312000000000001e-05 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903378+0000", 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0386000000000001e-05 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903429+0000", 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0822000000000002e-05 2023-10-09T04:07:47.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.905416+0000", 2023-10-09T04:07:47.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019873859999999998 2023-10-09T04:07:47.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119092 6.11 6:883a032b:::benchmark_data_smithi163_287034_object10829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.524246490000003, 2023-10-09T04:07:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4310658030000001, 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119092 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:07:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529820+0000", 2023-10-09T04:07:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:07:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529825+0000", 2023-10-09T04:07:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7060000000000003e-06 2023-10-09T04:07:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529837+0000", 2023-10-09T04:07:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1731e-05 2023-10-09T04:07:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529844+0000", 2023-10-09T04:07:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6240000000000002e-06 2023-10-09T04:07:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529878+0000", 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.392e-05 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529887+0000", 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7059999999999993e-06 2023-10-09T04:07:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.900013+0000", 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.370126255 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.902419+0000", 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024062950000000001 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.927730+0000", 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.025310295999999999 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.927860+0000", 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012995199999999999 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.928026+0000", 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016576500000000001 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.945127+0000", 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017101775 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958766+0000", 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013638477 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958829+0000", 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3446000000000002e-05 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959082+0000", 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025267399999999999 2023-10-09T04:07:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959123+0000", 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0979999999999997e-05 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960887+0000", 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00176406 2023-10-09T04:07:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119103 6.11 6:8d75c45e:::benchmark_data_smithi163_287034_object10840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.512297036, 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4205783109999999, 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119103 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541769+0000", 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541776+0000", 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1169999999999998e-06 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541795+0000", 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9114999999999999e-05 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541807+0000", 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1369000000000001e-05 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541880+0000", 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3739000000000007e-05 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541891+0000", 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1107000000000001e-05 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.902472+0000", 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.36058053 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903197+0000", 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072483200000000002 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933631+0000", 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.030434745999999999 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933720+0000", 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8212000000000002e-05 2023-10-09T04:07:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933838+0000", 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011806 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.945137+0000", 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011299208 2023-10-09T04:07:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958878+0000", 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013741547 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958932+0000", 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3193999999999999e-05 2023-10-09T04:07:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960951+0000", 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020194599999999998 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960992+0000", 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1319000000000002e-05 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962349+0000", 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013564200000000001 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119123 6.11 6:892f37da:::benchmark_data_smithi163_287034_object10860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.494246871999998, 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.403942767, 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119123 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559819+0000", 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559825+0000", 2023-10-09T04:07:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2280000000000003e-06 2023-10-09T04:07:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559842+0000", 2023-10-09T04:07:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6141999999999999e-05 2023-10-09T04:07:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559850+0000", 2023-10-09T04:07:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9079999999999995e-06 2023-10-09T04:07:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559902+0000", 2023-10-09T04:07:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2420999999999998e-05 2023-10-09T04:07:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559917+0000", 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5003e-05 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.905461+0000", 2023-10-09T04:07:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.345543943 2023-10-09T04:07:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.906296+0000", 2023-10-09T04:07:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083475499999999996 2023-10-09T04:07:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.938986+0000", 2023-10-09T04:07:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032690364 2023-10-09T04:07:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.939133+0000", 2023-10-09T04:07:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014743800000000001 2023-10-09T04:07:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.939247+0000", 2023-10-09T04:07:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113915 2023-10-09T04:07:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958704+0000", 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019456505999999998 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958968+0000", 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026427900000000002 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959024+0000", 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5674000000000003e-05 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962395+0000", 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003370736 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962431+0000", 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6906000000000002e-05 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963764+0000", 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00133204 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119141 6.11 6:8c64562a:::benchmark_data_smithi163_287034_object10878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:07:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.478968252999998, 2023-10-09T04:07:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.396054788, 2023-10-09T04:07:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119141 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575102+0000", 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9490000000000001e-06 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575108+0000", 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1140000000000002e-06 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575113+0000", 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2360000000000004e-06 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575184+0000", 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0494000000000006e-05 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575195+0000", 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1283e-05 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.908246+0000", 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.333050761 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.909908+0000", 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001662455 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949276+0000", 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039368001 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949574+0000", 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029827500000000003 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949695+0000", 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120931 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958714+0000", 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0090190619999999996 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963822+0000", 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051071249999999997 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963923+0000", 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101548 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.969034+0000", 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051108899999999999 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.969082+0000", 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8161999999999997e-05 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971154+0000", 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020719359999999999 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119147 6.11 6:8f3284c4:::benchmark_data_smithi163_287034_object10884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.468726717999999, 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3874358629999999, 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119147 2023-10-09T04:07:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:07:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585339+0000", 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585345+0000", 2023-10-09T04:07:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.02e-06 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585357+0000", 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1371e-05 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585362+0000", 2023-10-09T04:07:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1259999999999997e-06 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585402+0000", 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0420999999999998e-05 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585413+0000", 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0242e-05 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.911820+0000", 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.326407552 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.912798+0000", 2023-10-09T04:07:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097778699999999993 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.954865+0000", 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.042066631 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.954986+0000", 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121303 2023-10-09T04:07:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.955142+0000", 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015607500000000001 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959037+0000", 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003894837 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963876+0000", 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048392319999999997 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963970+0000", 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4040000000000001e-05 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971217+0000", 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072466520000000001 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971271+0000", 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4704999999999999e-05 2023-10-09T04:07:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.972777+0000", 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001505337 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119163 6.11 6:8b083f99:::benchmark_data_smithi163_287034_object10900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.452557376000001, 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3956605739999999, 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119163 2023-10-09T04:07:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601509+0000", 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601516+0000", 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3549999999999999e-06 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601528+0000", 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1885000000000001e-05 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601537+0000", 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9060000000000008e-06 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601581+0000", 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4168000000000001e-05 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601589+0000", 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3240000000000003e-06 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.964003+0000", 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.362413941 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.965225+0000", 2023-10-09T04:07:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012218419999999999 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977192+0000", 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011966976000000001 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977278+0000", 2023-10-09T04:07:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5663999999999998e-05 2023-10-09T04:07:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977394+0000", 2023-10-09T04:07:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011626 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.982063+0000", 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046684430000000004 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995092+0000", 2023-10-09T04:07:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013029099000000001 2023-10-09T04:07:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995125+0000", 2023-10-09T04:07:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3803999999999997e-05 2023-10-09T04:07:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995155+0000", 2023-10-09T04:07:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9683999999999999e-05 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995201+0000", 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6093000000000001e-05 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997171+0000", 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019696169999999999 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119188 6.11 6:8a56da9b:::benchmark_data_smithi163_287034_object10925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.430188176000001, 2023-10-09T04:07:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3781419879999999, 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119188 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623878+0000", 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623884+0000", 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623900+0000", 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6373000000000001e-05 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623908+0000", 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1419999999999993e-06 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623965+0000", 2023-10-09T04:07:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6709000000000002e-05 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623977+0000", 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2267e-05 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.965277+0000", 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3412998730000001 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968073+0000", 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027958570000000001 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986363+0000", 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018290041 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986467+0000", 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010372199999999999 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986609+0000", 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014165400000000001 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997219+0000", 2023-10-09T04:07:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010610519000000001 2023-10-09T04:07:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997253+0000", 2023-10-09T04:07:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3803999999999997e-05 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997288+0000", 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5064999999999997e-05 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000032+0000", 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027444190000000001 2023-10-09T04:07:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000092+0000", 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9963000000000002e-05 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002021+0000", 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001929107 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119196 6.11 6:8fdab617:::benchmark_data_smithi163_287034_object10933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.422684408999999, 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3723983319999999, 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119196 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631382+0000", 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631387+0000", 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5260000000000004e-06 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631396+0000", 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0110000000000006e-06 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631402+0000", 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:07:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631439+0000", 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6520999999999998e-05 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631446+0000", 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4059999999999999e-06 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968116+0000", 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.33666961 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968985+0000", 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086886199999999998 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993266+0000", 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024281193999999999 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993347+0000", 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1148000000000003e-05 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993458+0000", 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111158 2023-10-09T04:07:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000020+0000", 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065613119999999997 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002036+0000", 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020161950000000001 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002091+0000", 2023-10-09T04:07:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5470999999999998e-05 2023-10-09T04:07:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002144+0000", 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2332e-05 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002203+0000", 2023-10-09T04:07:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9389999999999999e-05 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.003782+0000", 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015784429999999999 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119206 6.11 6:8f80b5b4:::benchmark_data_smithi163_287034_object10943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.413537746000003, 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.397280665, 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119206 2023-10-09T04:07:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640528+0000", 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640535+0000", 2023-10-09T04:07:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0119999999999999e-06 2023-10-09T04:07:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640550+0000", 2023-10-09T04:07:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4673e-05 2023-10-09T04:07:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640559+0000", 2023-10-09T04:07:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.935e-06 2023-10-09T04:07:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640615+0000", 2023-10-09T04:07:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6740999999999998e-05 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640627+0000", 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.189e-05 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997311+0000", 2023-10-09T04:07:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3566840010000001 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.998135+0000", 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082370499999999999 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012025+0000", 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013889755 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012139+0000", 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011416099999999999 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012292+0000", 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015364299999999999 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026026+0000", 2023-10-09T04:07:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013733176999999999 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026062+0000", 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6235999999999997e-05 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032222+0000", 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0061599669999999997 2023-10-09T04:07:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.036596+0000", 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043742829999999996 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.036661+0000", 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5092000000000006e-05 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037811+0000", 2023-10-09T04:07:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001149279 2023-10-09T04:07:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119231 6.11 6:8f1bfbf5:::benchmark_data_smithi163_287034_object10968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.382986582000001, 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3681440409999999, 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119231 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671079+0000", 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671087+0000", 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5410000000000003e-06 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671103+0000", 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6531000000000001e-05 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671113+0000", 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7170000000000003e-06 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671174+0000", 2023-10-09T04:07:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1267999999999997e-05 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671184+0000", 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9890000000000004e-06 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.998170+0000", 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.326985353 2023-10-09T04:07:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999028+0000", 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000858163 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018202+0000", 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019174388000000001 2023-10-09T04:07:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018330+0000", 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127269 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018487+0000", 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015788 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032196+0000", 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013708549 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032241+0000", 2023-10-09T04:07:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5118999999999999e-05 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037827+0000", 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055862890000000004 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037934+0000", 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106397 2023-10-09T04:07:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037980+0000", 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6433000000000001e-05 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.039225+0000", 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001244818 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119251 6.11 6:8bf159d8:::benchmark_data_smithi163_287034_object10988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.356006972000003, 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3497973839999999, 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119251 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698059+0000", 2023-10-09T04:07:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698065+0000", 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.908e-06 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698078+0000", 2023-10-09T04:07:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.296e-05 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698086+0000", 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7570000000000004e-06 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698139+0000", 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3701e-05 2023-10-09T04:07:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698150+0000", 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0548e-05 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999079+0000", 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.30092889 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999966+0000", 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088695600000000005 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024113+0000", 2023-10-09T04:07:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.02414738 2023-10-09T04:07:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024224+0000", 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011063799999999999 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024372+0000", 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000148123 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032262+0000", 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078901960000000004 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037837+0000", 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055749579999999996 2023-10-09T04:07:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.039272+0000", 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001435012 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.046029+0000", 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067568610000000003 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.046059+0000", 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9821000000000001e-05 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.047858+0000", 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00179923 2023-10-09T04:07:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119258 6.11 6:89e8583e:::benchmark_data_smithi163_287034_object10995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.346075939999999, 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3578773550000001, 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119258 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:07:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:07:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707990+0000", 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707997+0000", 2023-10-09T04:07:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8560000000000001e-06 2023-10-09T04:07:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708014+0000", 2023-10-09T04:07:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6804999999999999e-05 2023-10-09T04:07:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708021+0000", 2023-10-09T04:07:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0180000000000004e-06 2023-10-09T04:07:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708074+0000", 2023-10-09T04:07:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2914999999999997e-05 2023-10-09T04:07:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708088+0000", 2023-10-09T04:07:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3818999999999999e-05 2023-10-09T04:07:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026085+0000", 2023-10-09T04:07:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3179974219999999 2023-10-09T04:07:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026910+0000", 2023-10-09T04:07:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082452299999999997 2023-10-09T04:07:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050702+0000", 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023792316000000001 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050818+0000", 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116173 2023-10-09T04:07:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050970+0000", 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015151300000000001 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060167+0000", 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091971970000000007 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060243+0000", 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6215000000000001e-05 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060374+0000", 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000130894 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.063837+0000", 2023-10-09T04:07:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003463224 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.063891+0000", 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4011999999999997e-05 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.065869+0000", 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001977768 2023-10-09T04:07:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119295 6.11 6:8feb1e91:::benchmark_data_smithi163_287034_object11032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.228374379999998, 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2504799099999999, 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119295 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825692+0000", 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825700+0000", 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1559999999999995e-06 2023-10-09T04:07:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825719+0000", 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9177999999999998e-05 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825727+0000", 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4630000000000001e-06 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825788+0000", 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0730999999999998e-05 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825803+0000", 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4347e-05 2023-10-09T04:07:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026947+0000", 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2011442640000001 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.027704+0000", 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00075679299999999996 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.056255+0000", 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028550981 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.057403+0000", 2023-10-09T04:07:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011480500000000001 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.057552+0000", 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014943500000000001 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060380+0000", 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002827867 2023-10-09T04:07:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.068125+0000", 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0077448899999999999 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.069348+0000", 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001223285 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.074303+0000", 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049545429999999996 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.074377+0000", 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4524999999999995e-05 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.076173+0000", 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017959460000000001 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:137553 6.1 6:8091e241:::benchmark_data_smithi163_287034_object29290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:08.989652+0000", 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 38.064415437000001, 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23674761999999999, 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 137553 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989652+0000", 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989652+0000", 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989652+0000", 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989655+0000", 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.123e-06 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989663+0000", 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9100000000000005e-06 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989667+0000", 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3109999999999999e-06 2023-10-09T04:07:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989727+0000", 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9627999999999999e-05 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.989828+0000", 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010128699999999999 2023-10-09T04:07:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.990186+0000", 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00035818699999999999 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.992010+0000", 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001823585 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:08.993936+0000", 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019264779999999999 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:09.226310+0000", 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23237401999999999 2023-10-09T04:07:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:09.226362+0000", 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2133999999999999e-05 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:09.226400+0000", 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7478000000000003e-05 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:151829 6.11 6:8cd5107b:::benchmark_data_smithi163_287034_object43566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:27.591825+0000", 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 19.462242801999999, 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.266038636, 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151829 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.591825+0000", 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.591825+0000", 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.591824+0000", 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.591829+0000", 2023-10-09T04:07:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9200000000000003e-06 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.591840+0000", 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1474999999999999e-05 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.591848+0000", 2023-10-09T04:07:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3390000000000004e-06 2023-10-09T04:07:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.592106+0000", 2023-10-09T04:07:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025868499999999998 2023-10-09T04:07:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.593013+0000", 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090680399999999999 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.616347+0000", 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023334222000000002 2023-10-09T04:07:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.616447+0000", 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9080999999999995e-05 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.616776+0000", 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032989199999999998 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.847647+0000", 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23087048399999999 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.847709+0000", 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2068000000000003e-05 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.847720+0000", 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1069e-05 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.856285+0000", 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085644320000000003 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.856342+0000", 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7123999999999997e-05 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.857863+0000", 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015217970000000001 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:151838 6.11 6:894bda32:::benchmark_data_smithi163_287034_object43575:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:27.608923+0000", 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 19.445144392, 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26875176699999997, 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151838 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608923+0000", 2023-10-09T04:07:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608923+0000", 2023-10-09T04:07:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608921+0000", 2023-10-09T04:07:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608928+0000", 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0789999999999997e-06 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608938+0000", 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0461e-05 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608945+0000", 2023-10-09T04:07:47.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.725e-06 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.608996+0000", 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1233000000000002e-05 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.614006+0000", 2023-10-09T04:07:47.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050104169999999996 2023-10-09T04:07:47.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.853845+0000", 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23983874799999999 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.853970+0000", 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012492499999999999 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.854410+0000", 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043951299999999999 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.859227+0000", 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048179629999999998 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.863724+0000", 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044965259999999998 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.863822+0000", 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7973000000000005e-05 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.875599+0000", 2023-10-09T04:07:47.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011777219 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.875649+0000", 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9427e-05 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.877675+0000", 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020263500000000001 2023-10-09T04:07:47.284 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:151858 6.11 6:8fd37e05:::benchmark_data_smithi163_287034_object43595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:27.627388+0000", 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 19.426679503999999, 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25994620000000002, 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:07:47.285 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 151858 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.627388+0000", 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.627388+0000", 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.627387+0000", 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.627393+0000", 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9150000000000001e-06 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.627403+0000", 2023-10-09T04:07:47.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0694e-05 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.627411+0000", 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3760000000000002e-06 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.847752+0000", 2023-10-09T04:07:47.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22034108599999999 2023-10-09T04:07:47.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:07:47.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.850926+0000", 2023-10-09T04:07:47.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031737409999999999 2023-10-09T04:07:47.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.870054+0000", 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019128323999999999 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.870157+0000", 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103577 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.870560+0000", 2023-10-09T04:07:47.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00040306100000000002 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.881147+0000", 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010586541999999999 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.883172+0000", 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020245380000000002 2023-10-09T04:07:47.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.883186+0000", 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4868999999999999e-05 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.885962+0000", 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027757179999999999 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:07:47.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.886025+0000", 2023-10-09T04:07:47.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2367999999999996e-05 2023-10-09T04:07:47.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:47.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:47.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:07:47.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:27.887334+0000", 2023-10-09T04:07:47.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013097849999999999 2023-10-09T04:07:47.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.298 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:47.298 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:47.299 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:07:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:47 smithi181 ceph-mon[196979]: failed to encode map e141 with expected crc 2023-10-09T04:07:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:07:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:47 smithi181 ceph-mon[196979]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T04:07:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:47 smithi181 ceph-mon[196979]: failed to encode map e141 with expected crc 2023-10-09T04:07:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3936/161496 objects degraded (2.437%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:07:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[221067]: failed to encode map e141 with expected crc 2023-10-09T04:07:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:07:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[221067]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T04:07:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[221067]: failed to encode map e141 with expected crc 2023-10-09T04:07:47.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3936/161496 objects degraded (2.437%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:07:47.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[213841]: failed to encode map e141 with expected crc 2023-10-09T04:07:47.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:07:47.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[213841]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T04:07:47.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[213841]: failed to encode map e141 with expected crc 2023-10-09T04:07:47.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3936/161496 objects degraded (2.437%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:07:48.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:07:48.706 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:48 smithi163 ceph-mon[213841]: pgmap v363: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 3936/155025 objects degraded (2.539%); 3158/155025 objects misplaced (2.037%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T04:07:48.707 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:48 smithi163 ceph-mon[221067]: pgmap v363: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 3936/155025 objects degraded (2.539%); 3158/155025 objects misplaced (2.037%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T04:07:48.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:48.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:48 smithi181 ceph-mon[196979]: pgmap v363: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 3936/155025 objects degraded (2.539%); 3158/155025 objects misplaced (2.037%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T04:07:48.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:48.881 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:48.952 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:48.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:48.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:49.059 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:49.112 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:49.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:49.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:49.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:49.394 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:49.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:07:49.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:49.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:49.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:49.571 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:49.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:49.651 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:49.679 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:49.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:49.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:49.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:49.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:49.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:49.926 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:50.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:50.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:50.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:50.236 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:50.476 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:50 smithi163 ceph-mon[213841]: pgmap v364: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 3692/153489 objects degraded (2.405%); 3036/153489 objects misplaced (1.978%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:07:50.476 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:50 smithi163 ceph-mon[221067]: pgmap v364: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 3692/153489 objects degraded (2.405%); 3036/153489 objects misplaced (1.978%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:07:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:50 smithi181 ceph-mon[196979]: pgmap v364: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 3692/153489 objects degraded (2.405%); 3036/153489 objects misplaced (1.978%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T04:07:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:07:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:07:50] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:07:50.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:50.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:50.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:50.982 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:51.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:51.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:51.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:51.162 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:51.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:07:51.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:51.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:51.316 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:51.356 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:51.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:51.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:51.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:51.598 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:51.695 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:51.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:51.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:51.808 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:51.846 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:51.911 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:51.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:51.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:52.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:52.029 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:52.173 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:52.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:52.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:52.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:52.551 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:52.646 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:52.766 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:52 smithi163 ceph-mon[221067]: pgmap v365: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 3692/145191 objects degraded (2.543%); 3036/145191 objects misplaced (2.091%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:07:52.766 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3692/145191 objects degraded (2.543%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:07:52.766 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:52 smithi163 ceph-mon[213841]: pgmap v365: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 3692/145191 objects degraded (2.543%); 3036/145191 objects misplaced (2.091%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:07:52.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3692/145191 objects degraded (2.543%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:07:52.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:52 smithi181 ceph-mon[196979]: pgmap v365: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 3692/145191 objects degraded (2.543%); 3036/145191 objects misplaced (2.091%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:07:52.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3692/145191 objects degraded (2.543%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:07:52.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:52.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:52.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:07:53.040 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:53.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:07:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:53.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:07:53.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:53.428 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:53.507 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:07:53.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3730238101' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:53.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:53 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:53.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3730238101' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:53.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:53 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:53.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3730238101' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:53.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:53 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:07:54.518 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:07:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: pgmap v366: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3692/143676 objects degraded (2.570%); 2471/143676 objects misplaced (1.720%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: failed to encode map e142 with expected crc 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: failed to encode map e142 with expected crc 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1279492965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:07:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: pgmap v366: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3692/143676 objects degraded (2.570%); 2471/143676 objects misplaced (1.720%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: failed to encode map e142 with expected crc 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: failed to encode map e142 with expected crc 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1279492965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:07:54.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: pgmap v366: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 3692/143676 objects degraded (2.570%); 2471/143676 objects misplaced (1.720%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: failed to encode map e142 with expected crc 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: failed to encode map e142 with expected crc 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1279492965' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:07:54.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:07:55.467 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:55.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:55.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:07:55.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[221067]: failed to encode map e143 with expected crc 2023-10-09T04:07:55.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:07:55.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[221067]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T04:07:55.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[221067]: failed to encode map e143 with expected crc 2023-10-09T04:07:55.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:07:55.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[221067]: pgmap v369: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3472/138621 objects degraded (2.505%); 2361/138621 objects misplaced (1.703%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:07:55.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[213841]: failed to encode map e143 with expected crc 2023-10-09T04:07:55.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:07:55.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[213841]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T04:07:55.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[213841]: failed to encode map e143 with expected crc 2023-10-09T04:07:55.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:07:55.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:55 smithi163 ceph-mon[213841]: pgmap v369: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3472/138621 objects degraded (2.505%); 2361/138621 objects misplaced (1.703%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:07:55.642 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:56.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:55 smithi181 ceph-mon[196979]: failed to encode map e143 with expected crc 2023-10-09T04:07:56.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:55 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:07:56.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:55 smithi181 ceph-mon[196979]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T04:07:56.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:55 smithi181 ceph-mon[196979]: failed to encode map e143 with expected crc 2023-10-09T04:07:56.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:55 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:07:56.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:55 smithi181 ceph-mon[196979]: pgmap v369: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3472/138621 objects degraded (2.505%); 2361/138621 objects misplaced (1.703%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:07:56.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:07:56.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:56.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:07:56.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:56.524 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:56.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:56 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:07:56.600 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:56 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:07:56.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:56 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:07:56.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:56 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:07:56.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:07:56.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:56.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:56.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:56.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:56 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:07:56.939 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:56 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:07:56.941 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:57.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:57.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:57.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:57.141 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:57.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:57.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:173856 6.f 6:f9b4a05b:::benchmark_data_smithi163_287034_object65593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:57.239803+0000", 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.012148386000000001, 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012189320999999999, 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:57.255 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 173856 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239803+0000", 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:57.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239803+0000", 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239801+0000", 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239810+0000", 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6260000000000001e-06 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239825+0000", 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4899000000000001e-05 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:57.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239831+0000", 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3729999999999998e-06 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.239883+0000", 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1897999999999997e-05 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:57.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.240025+0000", 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014198800000000001 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.240166+0000", 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014129 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.248371+0000", 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082044579999999995 2023-10-09T04:07:57.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.248424+0000", 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3872000000000002e-05 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:173861 6.f 6:f88ebbb6:::benchmark_data_smithi163_287034_object65598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:57.249453+0000", 2023-10-09T04:07:57.261 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0024982339999999998, 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026334800000000001, 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 173861 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:57.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249453+0000", 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249453+0000", 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249451+0000", 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249460+0000", 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4989999999999994e-06 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249478+0000", 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7787000000000001e-05 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:57.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249490+0000", 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1936000000000001e-05 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249527+0000", 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7156000000000001e-05 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249684+0000", 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015719099999999999 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.249829+0000", 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145492 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:173860 6.0 e143/124 6:0424fbd5:::benchmark_data_smithi163_287034_object65597:head v 143'8803, mlcod=143'8801)", 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:07:57.251227+0000", 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.00072444000000000002, 2023-10-09T04:07:57.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00093294600000000004, 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251227+0000", 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251227+0000", 2023-10-09T04:07:57.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251226+0000", 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251233+0000", 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9519999999999997e-06 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251248+0000", 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.49e-05 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251256+0000", 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2150000000000001e-06 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251349+0000", 2023-10-09T04:07:57.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3111999999999999e-05 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:07:57.251369+0000", 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9941000000000001e-05 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 3 2023-10-09T04:07:57.270 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:07:57.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:57.362 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:57.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:57.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:07:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:57.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:57.486 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:07:57.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:07:57.543 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:07:57.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[213841]: 6.15 scrub starts 2023-10-09T04:07:57.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3472/138621 objects degraded (2.505%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:07:57.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[213841]: pgmap v370: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 3472/132405 objects degraded (2.622%); 1748/132405 objects misplaced (1.320%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:07:57.552 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1901409862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:57.580 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:57.586 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:07:57.633 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:07:57.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:07:57.663 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:07:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:57.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:07:57.700 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:07:57.736 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:07:57.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.750 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.750 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.750 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.750 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:07:57.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:07:57.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:57 smithi181 ceph-mon[196979]: 6.15 scrub starts 2023-10-09T04:07:57.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3472/138621 objects degraded (2.505%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:07:57.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:57 smithi181 ceph-mon[196979]: pgmap v370: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 3472/132405 objects degraded (2.622%); 1748/132405 objects misplaced (1.320%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:07:57.800 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1901409862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:57.802 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:57.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[221067]: 6.15 scrub starts 2023-10-09T04:07:57.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3472/138621 objects degraded (2.505%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:07:57.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[221067]: pgmap v370: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 3472/132405 objects degraded (2.622%); 1748/132405 objects misplaced (1.320%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:07:57.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1901409862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:07:57.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:07:57.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:57.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:07:58.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:07:58.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:07:58.096 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:58.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:07:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:58.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:07:58.385 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:07:58.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:58.565 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:58.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:58 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:07:58.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:58 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:07:58.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:58 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:07:58.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:58 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:07:58.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:58.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:59.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:07:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:58 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:07:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:58 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:07:59.039 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:59.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:07:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:59.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:07:59.162 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:59.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:07:59.240 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:59.308 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:59.314 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:59.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:07:59.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:59.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:07:59.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:07:59 smithi163 ceph-mon[213841]: pgmap v371: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3226/131001 objects degraded (2.463%); 1625/131001 objects misplaced (1.240%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:07:59.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:59.716 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:59.788 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:07:59.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:07:59.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:07:59.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:07:59 smithi163 ceph-mon[221067]: pgmap v371: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3226/131001 objects degraded (2.463%); 1625/131001 objects misplaced (1.240%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:07:59.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:07:59.933 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:07:59.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:07:59 smithi181 ceph-mon[196979]: pgmap v371: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3226/131001 objects degraded (2.463%); 1625/131001 objects misplaced (1.240%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:08:00.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:00.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:00.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:00.199 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:00.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:00 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:08:00.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:08:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:08:00] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:08:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:00 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:08:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:00 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:08:01.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:08:01.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:01.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:01.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:01.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:01.395 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:01.568 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:01.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:01.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:01.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:01 smithi163 ceph-mon[213841]: 6.15 scrub starts 2023-10-09T04:08:01.752 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:01 smithi163 ceph-mon[213841]: pgmap v372: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 3226/122322 objects degraded (2.637%); 1625/122322 objects misplaced (1.328%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:08:01.752 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:01 smithi163 ceph-mon[221067]: 6.15 scrub starts 2023-10-09T04:08:01.752 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:01 smithi163 ceph-mon[221067]: pgmap v372: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 3226/122322 objects degraded (2.637%); 1625/122322 objects misplaced (1.328%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:08:01.761 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:01.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:01.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:01.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:02.008 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:01 smithi181 ceph-mon[196979]: 6.15 scrub starts 2023-10-09T04:08:02.008 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:01 smithi181 ceph-mon[196979]: pgmap v372: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 97 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 3226/122322 objects degraded (2.637%); 1625/122322 objects misplaced (1.328%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:08:02.012 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:02.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:02.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:02.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:02.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:02.192 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:02.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:02.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:02.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:02.372 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:02.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:02.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:02.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[221067]: failed to encode map e144 with expected crc 2023-10-09T04:08:02.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[221067]: failed to encode map e144 with expected crc 2023-10-09T04:08:02.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[221067]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T04:08:02.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[221067]: failed to encode map e144 with expected crc 2023-10-09T04:08:02.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:08:02.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3226/122322 objects degraded (2.637%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:08:02.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:02.746 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:02.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[213841]: failed to encode map e144 with expected crc 2023-10-09T04:08:02.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[213841]: failed to encode map e144 with expected crc 2023-10-09T04:08:02.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[213841]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T04:08:02.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[213841]: failed to encode map e144 with expected crc 2023-10-09T04:08:02.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:08:02.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3226/122322 objects degraded (2.637%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:08:02.944 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:02 smithi181 ceph-mon[196979]: failed to encode map e144 with expected crc 2023-10-09T04:08:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:02 smithi181 ceph-mon[196979]: failed to encode map e144 with expected crc 2023-10-09T04:08:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:02 smithi181 ceph-mon[196979]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T04:08:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:02 smithi181 ceph-mon[196979]: failed to encode map e144 with expected crc 2023-10-09T04:08:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:02 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:08:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3226/122322 objects degraded (2.637%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:08:03.040 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:03.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:03.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:03.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:03.290 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:03.305 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:08:03.305 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:08:03.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:03.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:03.692 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:08:03.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:03.806 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: pgmap v375: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 3226/120921 objects degraded (2.668%); 1613/120921 objects misplaced (1.334%); 3.4 MiB/s, 93 objects/s recovering 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1273378070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:03.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: pgmap v375: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 3226/120921 objects degraded (2.668%); 1613/120921 objects misplaced (1.334%); 3.4 MiB/s, 93 objects/s recovering 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1273378070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:03.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:03 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:03.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:03.891 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.891 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T04:08:03.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: failed to encode map e145 with expected crc 2023-10-09T04:08:03.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: pgmap v375: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 3226/120921 objects degraded (2.668%); 1613/120921 objects misplaced (1.334%); 3.4 MiB/s, 93 objects/s recovering 2023-10-09T04:08:03.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1273378070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:03.892 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:03 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:03.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:03.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:04.027 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:04.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:04.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:04.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:04.259 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:04.691 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: 6.16 deep-scrub starts 2023-10-09T04:08:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/216415711' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:08:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:08:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:08:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: 6.16 deep-scrub starts 2023-10-09T04:08:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/216415711' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:08:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:08:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:08:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: failed to encode map e146 with expected crc 2023-10-09T04:08:05.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: 6.16 deep-scrub starts 2023-10-09T04:08:05.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/216415711' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:08:05.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:08:05.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:08:05.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:05.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[221067]: pgmap v378: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 3266/116661 objects misplaced (2.800%); 3.1 MiB/s, 128 objects/s recovering 2023-10-09T04:08:05.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:08:05.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T04:08:05.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: failed to encode map e147 with expected crc 2023-10-09T04:08:05.929 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:05 smithi163 ceph-mon[213841]: pgmap v378: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 3266/116661 objects misplaced (2.800%); 3.1 MiB/s, 128 objects/s recovering 2023-10-09T04:08:05.970 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:08:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: failed to encode map e147 with expected crc 2023-10-09T04:08:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:08:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T04:08:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: failed to encode map e147 with expected crc 2023-10-09T04:08:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: failed to encode map e147 with expected crc 2023-10-09T04:08:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: failed to encode map e147 with expected crc 2023-10-09T04:08:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:05 smithi181 ceph-mon[196979]: pgmap v378: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 97 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 3266/116661 objects misplaced (2.800%); 3.1 MiB/s, 128 objects/s recovering 2023-10-09T04:08:06.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:06.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:06 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:08:06.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:06 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:08:06.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:06.971 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:07.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:06 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:08:07.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:07.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:07.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:07.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:08:07.684 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:07.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:07.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:07 smithi163 ceph-mon[221067]: pgmap v379: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 3266/111420 objects misplaced (2.931%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T04:08:07.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:08:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:07.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:07 smithi163 ceph-mon[213841]: pgmap v379: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 3266/111420 objects misplaced (2.931%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T04:08:07.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:08:07.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:08.001 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:07 smithi181 ceph-mon[196979]: pgmap v379: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 3266/111420 objects misplaced (2.931%); 2.2 MiB/s, 89 objects/s recovering 2023-10-09T04:08:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:08:08.022 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:08.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:08.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:08.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:08.233 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:08:08.259 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:08.272 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:08:08.301 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:08.317 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:08:08.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:08.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:08.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:08.373 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:08:08.409 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:08:08.445 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:08:08.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:08.481 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:08:08.523 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:08:08.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:08:08.547 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:08.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:08:08.788 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[221067]: 6.15 scrub starts 2023-10-09T04:08:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/48328136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:08:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[221067]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T04:08:08.792 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[221067]: failed to encode map e148 with expected crc 2023-10-09T04:08:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:08.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:08.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[213841]: 6.15 scrub starts 2023-10-09T04:08:08.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/48328136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:08.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:08:08.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[213841]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T04:08:08.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:08 smithi163 ceph-mon[213841]: failed to encode map e148 with expected crc 2023-10-09T04:08:08.894 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:08.961 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:08 smithi181 ceph-mon[196979]: 6.15 scrub starts 2023-10-09T04:08:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/48328136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:08:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:08 smithi181 ceph-mon[196979]: osdmap e148: 8 total, 8 up, 8 in 2023-10-09T04:08:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:08 smithi181 ceph-mon[196979]: failed to encode map e148 with expected crc 2023-10-09T04:08:09.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:09.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:09.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:09.308 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:09.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:09.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:09.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:09.489 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:10.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:10.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:10.424 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: pgmap v382: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 2772/109989 objects misplaced (2.520%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:08:10.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:10 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:08:10.463 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:10.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:10.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T04:08:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: pgmap v382: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 2772/109989 objects misplaced (2.520%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:08:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: failed to encode map e149 with expected crc 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: pgmap v382: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 2772/109989 objects misplaced (2.520%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:08:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:10 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:08:10.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:10.711 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:10.784 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:10.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:10.870 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:10.894 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:10.923 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:10.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:10.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:11.075 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:11.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:11.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:08:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:08:10] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:08:11.133 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:11.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:11.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:11.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:11 smithi181 ceph-mon[196979]: failed to encode map e150 with expected crc 2023-10-09T04:08:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:08:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:11 smithi181 ceph-mon[196979]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T04:08:11.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:11.377 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:11 smithi163 ceph-mon[221067]: failed to encode map e150 with expected crc 2023-10-09T04:08:11.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:08:11.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:11 smithi163 ceph-mon[221067]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T04:08:11.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:11 smithi163 ceph-mon[213841]: failed to encode map e150 with expected crc 2023-10-09T04:08:11.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:08:11.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:11 smithi163 ceph-mon[213841]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T04:08:11.382 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49830 6.f 6:fdf7a709:::benchmark_data_smithi163_287034_object49829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.696408+0000", 2023-10-09T04:08:12.127 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 165.29432876999999, 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21095125200000001, 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49830 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696408+0000", 2023-10-09T04:08:12.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696408+0000", 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696407+0000", 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:08:12.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696429+0000", 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2234000000000001e-05 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696438+0000", 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4269999999999997e-06 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696444+0000", 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4190000000000001e-06 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696487+0000", 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2809999999999998e-05 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696633+0000", 2023-10-09T04:08:12.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014677200000000001 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696811+0000", 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000178133 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.700571+0000", 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037594880000000001 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.700618+0000", 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7077999999999999e-05 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907264+0000", 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.206646471 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907312+0000", 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7716999999999997e-05 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907360+0000", 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7426000000000003e-05 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.134 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49834 6.f 6:f21ebc5a:::benchmark_data_smithi163_287034_object49833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.696766+0000", 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 165.29397120900001, 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.210726, 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49834 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696766+0000", 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696766+0000", 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696765+0000", 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696787+0000", 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1902000000000001e-05 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696806+0000", 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8995999999999999e-05 2023-10-09T04:08:12.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.696813+0000", 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4939999999999997e-06 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.698613+0000", 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017995929999999999 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.698771+0000", 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015784499999999999 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.698925+0000", 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000154439 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.700710+0000", 2023-10-09T04:08:12.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001785247 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.702366+0000", 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001655414 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907411+0000", 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20504536600000001 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907456+0000", 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4634e-05 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907492+0000", 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6100000000000003e-05 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.141 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:74824 6.f 6:fbaaa0a0:::benchmark_data_smithi163_287034_object74823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.637333+0000", 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 142.353404383, 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20278981800000001, 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.142 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 74824 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637333+0000", 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637333+0000", 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637331+0000", 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637355+0000", 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3445999999999999e-05 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637373+0000", 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8567999999999999e-05 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637379+0000", 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6169999999999998e-06 2023-10-09T04:08:12.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637423+0000", 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4113000000000003e-05 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637546+0000", 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012319300000000001 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637676+0000", 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012949700000000001 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.641418+0000", 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037424340000000002 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.642370+0000", 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00095150900000000001 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.840025+0000", 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.197655624 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.840082+0000", 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7290000000000002e-05 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.840123+0000", 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0089000000000002e-05 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.149 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:74825 6.f 6:f516fca1:::benchmark_data_smithi163_287034_object74824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.637392+0000", 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 142.353344822, 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20560746799999999, 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 74825 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637392+0000", 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637392+0000", 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637392+0000", 2023-10-09T04:08:12.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637414+0000", 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2578999999999998e-05 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637421+0000", 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4339999999999997e-06 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637426+0000", 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1150000000000002e-06 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639491+0000", 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020647410000000001 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639611+0000", 2023-10-09T04:08:12.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012036699999999999 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639745+0000", 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013353799999999999 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.641595+0000", 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018506989999999999 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.642536+0000", 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00094095699999999997 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.842908+0000", 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20037137899999999 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.842959+0000", 2023-10-09T04:08:12.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0988e-05 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.843000+0000", 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1103000000000003e-05 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181485 6.12 6:4f267e19:::benchmark_data_smithi163_287034_object73222:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.156 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206125+0000", 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7846122639999997, 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1531547579999999, 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181485 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206125+0000", 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206125+0000", 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206124+0000", 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:12.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206131+0000", 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4989999999999997e-06 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206144+0000", 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2483000000000001e-05 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206153+0000", 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.149e-06 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206209+0000", 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6422000000000003e-05 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206217+0000", 2023-10-09T04:08:12.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8480000000000001e-06 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207601+0000", 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001383692 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207609+0000", 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3860000000000007e-06 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.350954+0000", 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1433449339999999 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.351067+0000", 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113324 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.351164+0000", 2023-10-09T04:08:12.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6971000000000004e-05 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.358467+0000", 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0073023610000000003 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.358481+0000", 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4358e-05 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.358611+0000", 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012959500000000001 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.358675+0000", 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4432999999999998e-05 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359280+0000", 2023-10-09T04:08:12.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060459800000000003 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181492 6.12 6:4d32a009:::benchmark_data_smithi163_287034_object73229:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206167+0000", 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7845705039999999, 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1592916090000001, 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.165 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181492 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206167+0000", 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206167+0000", 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206166+0000", 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206169+0000", 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.503e-06 2023-10-09T04:08:12.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206179+0000", 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.06e-05 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206185+0000", 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9989999999999997e-06 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206290+0000", 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000105043 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206297+0000", 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2339999999999997e-06 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207724+0000", 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001426535 2023-10-09T04:08:12.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207732+0000", 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4740000000000005e-06 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.353768+0000", 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1460353320000001 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.353936+0000", 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016834900000000001 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.354123+0000", 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018661099999999999 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359300+0000", 2023-10-09T04:08:12.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051775830000000004 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359345+0000", 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4243999999999999e-05 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.364798+0000", 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005453304 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.364854+0000", 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6319000000000001e-05 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.365458+0000", 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060406299999999995 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181504 6.12 6:4d91ac1a:::benchmark_data_smithi163_287034_object73241:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206229+0000", 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7845080289999999, 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.159823544, 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181504 2023-10-09T04:08:12.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206229+0000", 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206229+0000", 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206229+0000", 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206232+0000", 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9069999999999999e-06 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206238+0000", 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6309999999999997e-06 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206244+0000", 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8900000000000004e-06 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206354+0000", 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011011299999999999 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206359+0000", 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7450000000000003e-06 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207748+0000", 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013892659999999999 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207755+0000", 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4609999999999996e-06 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.356477+0000", 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.148722257 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.356645+0000", 2023-10-09T04:08:12.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000168095 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.356822+0000", 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017662600000000001 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359354+0000", 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002532416 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.364746+0000", 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005391524 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.365533+0000", 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078782600000000004 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.365592+0000", 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8717e-05 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366053+0000", 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046052700000000002 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.182 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181508 6.12 6:4a361a8a:::benchmark_data_smithi163_287034_object73245:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206249+0000", 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7844879860000002, 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1603319080000001, 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181508 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.183 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206249+0000", 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206249+0000", 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206249+0000", 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206251+0000", 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5730000000000002e-06 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206257+0000", 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.032e-06 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206263+0000", 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2959999999999998e-06 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206408+0000", 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014579 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206415+0000", 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3489999999999997e-06 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207777+0000", 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013623179999999999 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207784+0000", 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4880000000000004e-06 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359502+0000", 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1517182939999999 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359658+0000", 2023-10-09T04:08:12.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000155857 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.359826+0000", 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016826499999999999 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.365473+0000", 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056467389999999996 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366103+0000", 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00063074599999999995 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366155+0000", 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1545999999999997e-05 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366210+0000", 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5418999999999999e-05 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366581+0000", 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00037057099999999999 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.191 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181552 6.12 6:4f103d79:::benchmark_data_smithi163_287034_object73289:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206284+0000", 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.78445293, 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.167814972, 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.192 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181552 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206284+0000", 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206284+0000", 2023-10-09T04:08:12.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206283+0000", 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206288+0000", 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7190000000000001e-06 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206297+0000", 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6680000000000007e-06 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206304+0000", 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.34e-06 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206458+0000", 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015428100000000001 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206463+0000", 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0660000000000003e-06 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207814+0000", 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013509049999999999 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207821+0000", 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9779999999999999e-06 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.361986+0000", 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.154164556 2023-10-09T04:08:12.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.362161+0000", 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000175107 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.362356+0000", 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019547400000000001 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366111+0000", 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003754201 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366642+0000", 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053104499999999995 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.373536+0000", 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0068942719999999999 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.373598+0000", 2023-10-09T04:08:12.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2352000000000002e-05 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374099+0000", 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00050086800000000002 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.205 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181617 6.12 6:4f60f305:::benchmark_data_smithi163_287034_object73354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206321+0000", 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7844165739999998, 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1683574919999999, 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181617 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206321+0000", 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206321+0000", 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206320+0000", 2023-10-09T04:08:12.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206330+0000", 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7769999999999997e-06 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206336+0000", 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4999999999999996e-06 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206341+0000", 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9570000000000001e-06 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206509+0000", 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016712500000000001 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206514+0000", 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9869999999999996e-06 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207845+0000", 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013307670000000001 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207850+0000", 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3309999999999997e-06 2023-10-09T04:08:12.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366691+0000", 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.158841623 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366835+0000", 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014332599999999999 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.366983+0000", 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014826800000000001 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374138+0000", 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071556340000000001 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374179+0000", 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0438000000000001e-05 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374191+0000", 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2172e-05 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374245+0000", 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3804999999999997e-05 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374678+0000", 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043317200000000001 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181618 6.12 6:4ab85405:::benchmark_data_smithi163_287034_object73355:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206351+0000", 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7843860940000003, 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.168852295, 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181618 2023-10-09T04:08:12.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206351+0000", 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206351+0000", 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206351+0000", 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206353+0000", 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1440000000000001e-06 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206358+0000", 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3000000000000001e-06 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206363+0000", 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4950000000000002e-06 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206576+0000", 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000213661 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206581+0000", 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7269999999999998e-06 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207889+0000", 2023-10-09T04:08:12.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013077500000000001 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207892+0000", 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.523e-06 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.369615+0000", 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1617223839999999 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.369965+0000", 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000350331 2023-10-09T04:08:12.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.370107+0000", 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014204699999999999 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374690+0000", 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.004582811 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374722+0000", 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1986999999999997e-05 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374749+0000", 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7657e-05 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.374804+0000", 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4246000000000002e-05 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.375203+0000", 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00039959099999999998 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.224 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181655 6.12 6:4bea51b9:::benchmark_data_smithi163_287034_object73392:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206379+0000", 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7843582420000001, 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.182776646, 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181655 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206379+0000", 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206379+0000", 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206379+0000", 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206381+0000", 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2460000000000002e-06 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206389+0000", 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5739999999999996e-06 2023-10-09T04:08:12.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206395+0000", 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2449999999999998e-06 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206616+0000", 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00022133 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206621+0000", 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8289999999999998e-06 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207917+0000", 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001296422 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207923+0000", 2023-10-09T04:08:12.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1979999999999998e-06 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.375264+0000", 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1673406719999999 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.375441+0000", 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017729599999999999 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.375621+0000", 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017971799999999999 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388280+0000", 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012659016 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388347+0000", 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7183999999999999e-05 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388404+0000", 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6292999999999998e-05 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388467+0000", 2023-10-09T04:08:12.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3905999999999994e-05 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389156+0000", 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068804300000000005 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.233 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181687 6.12 6:4978badb:::benchmark_data_smithi163_287034_object73424:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206398+0000", 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7843387590000002, 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1833152739999999, 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181687 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206398+0000", 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206398+0000", 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206398+0000", 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206400+0000", 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.894e-06 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206405+0000", 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.826e-06 2023-10-09T04:08:12.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206409+0000", 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3749999999999996e-06 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207945+0000", 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015361249999999999 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207949+0000", 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4070000000000002e-06 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.377954+0000", 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1700055570000001 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.378132+0000", 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017737300000000001 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.378300+0000", 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016794800000000001 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389172+0000", 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010872254 2023-10-09T04:08:12.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389219+0000", 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6961e-05 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389266+0000", 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7357000000000001e-05 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389310+0000", 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4169999999999999e-05 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389714+0000", 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00040331200000000002 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181790 6.12 6:483ee06b:::benchmark_data_smithi163_287034_object73527:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7843246649999998, 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2147281759999999, 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181790 2023-10-09T04:08:12.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206414+0000", 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.855e-06 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206419+0000", 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7840000000000003e-06 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206423+0000", 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5449999999999997e-06 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207972+0000", 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001549031 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207978+0000", 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9889999999999997e-06 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388116+0000", 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.180137601 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for rw locks", 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388262+0000", 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145642 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408226+0000", 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019964064 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408317+0000", 2023-10-09T04:08:12.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1717000000000003e-05 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408461+0000", 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014404700000000001 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418786+0000", 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010325051 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420556+0000", 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017692529999999999 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420629+0000", 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2847000000000002e-05 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420693+0000", 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4324000000000003e-05 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.421141+0000", 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044771999999999998 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.250 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181801 6.12 6:4926c61f:::benchmark_data_smithi163_287034_object73538:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206457+0000", 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.784279942, 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.194131713, 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181801 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.251 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206457+0000", 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206457+0000", 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206456+0000", 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206461+0000", 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0479999999999998e-06 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206470+0000", 2023-10-09T04:08:12.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5129999999999996e-06 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206476+0000", 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208005+0000", 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00152842 2023-10-09T04:08:12.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208010+0000", 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8780000000000003e-06 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389754+0000", 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.181744014 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.389913+0000", 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015841600000000001 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.390065+0000", 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015229800000000001 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397287+0000", 2023-10-09T04:08:12.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072214810000000001 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397344+0000", 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7188000000000002e-05 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.399827+0000", 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024831179999999999 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.399888+0000", 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0550000000000001e-05 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400589+0000", 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070134500000000003 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181863 6.12 6:49f6e6d2:::benchmark_data_smithi163_287034_object73600:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206481+0000", 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7842565559999999, 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2005455380000001, 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181863 2023-10-09T04:08:12.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206481+0000", 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206481+0000", 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206480+0000", 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206486+0000", 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0349999999999998e-06 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206492+0000", 2023-10-09T04:08:12.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6099999999999997e-06 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206496+0000", 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5789999999999997e-06 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208041+0000", 2023-10-09T04:08:12.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015442170000000001 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208046+0000", 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6550000000000001e-06 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.392760+0000", 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.184713361 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.392902+0000", 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014215300000000001 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.393055+0000", 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000153144 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397353+0000", 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042977079999999999 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400655+0000", 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033022189999999999 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.406312+0000", 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056570409999999998 2023-10-09T04:08:12.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.406382+0000", 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9560999999999993e-05 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407026+0000", 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000644555 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181893 6.12 6:48514e0f:::benchmark_data_smithi163_287034_object73630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.784237053, 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2117616529999999, 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.267 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181893 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206502+0000", 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0099999999999998e-06 2023-10-09T04:08:12.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206513+0000", 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206517+0000", 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5499999999999996e-06 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208079+0000", 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015617529999999999 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208087+0000", 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5059999999999998e-06 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395090+0000", 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1870032690000001 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395194+0000", 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010448 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395313+0000", 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011805 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400612+0000", 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052990609999999999 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407098+0000", 2023-10-09T04:08:12.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064860760000000003 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417540+0000", 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010442168999999999 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417604+0000", 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4577000000000002e-05 2023-10-09T04:08:12.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418262+0000", 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00065729299999999998 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181924 6.12 6:4956f571:::benchmark_data_smithi163_287034_object73661:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:08:12.275 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7841959149999997, 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.212897227, 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181924 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:08:12.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206543+0000", 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0640000000000001e-06 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206549+0000", 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6949999999999997e-06 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206554+0000", 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208118+0000", 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015642570000000001 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208123+0000", 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6370000000000004e-06 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397380+0000", 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1892568610000001 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397556+0000", 2023-10-09T04:08:12.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017565199999999999 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397712+0000", 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015616 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400663+0000", 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002950664 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407160+0000", 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064973219999999998 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418821+0000", 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011661227 2023-10-09T04:08:12.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418878+0000", 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6666999999999998e-05 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419438+0000", 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056037000000000003 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.283 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181928 6.12 6:4a3131f6:::benchmark_data_smithi163_287034_object73665:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.784165185, 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2134125929999999, 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.284 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181928 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206574+0000", 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0389999999999999e-06 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206579+0000", 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206585+0000", 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6330000000000001e-06 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208144+0000", 2023-10-09T04:08:12.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015595439999999999 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208151+0000", 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400687+0000", 2023-10-09T04:08:12.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.192535828 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400844+0000", 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015750000000000001 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.401023+0000", 2023-10-09T04:08:12.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017885000000000001 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407172+0000", 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006149374 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407223+0000", 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0392999999999997e-05 2023-10-09T04:08:12.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419479+0000", 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01225652 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419533+0000", 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3735e-05 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419985+0000", 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045140000000000002 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181968 6.12 6:4a8027c1:::benchmark_data_smithi163_287034_object73705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.218359+0000", 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.7723776839999998, 2023-10-09T04:08:12.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.202148295, 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181968 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218359+0000", 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218359+0000", 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:12.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218358+0000", 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218365+0000", 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6529999999999994e-06 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218381+0000", 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5310000000000001e-05 2023-10-09T04:08:12.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218389+0000", 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2460000000000003e-06 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218430+0000", 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1372000000000003e-05 2023-10-09T04:08:12.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.218439+0000", 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3809999999999999e-06 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.403830+0000", 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.185390918 2023-10-09T04:08:12.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.404019+0000", 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018908600000000001 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.404207+0000", 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018809400000000001 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407182+0000", 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029752289999999998 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.411016+0000", 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038339340000000002 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420030+0000", 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0090135809999999997 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420082+0000", 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2849000000000002e-05 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:12.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420508+0000", 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00042525100000000003 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:12.301 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:08:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[221067]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[221067]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[221067]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T04:08:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[221067]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[221067]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[221067]: pgmap v385: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 2772/104592 objects misplaced (2.650%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:08:12.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[213841]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[213841]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[213841]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T04:08:12.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[213841]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[213841]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:12 smithi163 ceph-mon[213841]: pgmap v385: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 2772/104592 objects misplaced (2.650%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:08:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:12 smithi181 ceph-mon[196979]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:12 smithi181 ceph-mon[196979]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:12 smithi181 ceph-mon[196979]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T04:08:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:12 smithi181 ceph-mon[196979]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:12 smithi181 ceph-mon[196979]: failed to encode map e151 with expected crc 2023-10-09T04:08:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:12 smithi181 ceph-mon[196979]: pgmap v385: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 2772/104592 objects misplaced (2.650%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:08:12.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:08:13.229 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[221067]: failed to encode map e151 with expected crc 2023-10-09T04:08:13.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[221067]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[221067]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[221067]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T04:08:13.230 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[221067]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:13 smithi181 ceph-mon[196979]: failed to encode map e151 with expected crc 2023-10-09T04:08:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:13 smithi181 ceph-mon[196979]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:13 smithi181 ceph-mon[196979]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:13 smithi181 ceph-mon[196979]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T04:08:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:13 smithi181 ceph-mon[196979]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[213841]: failed to encode map e151 with expected crc 2023-10-09T04:08:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[213841]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[213841]: failed to encode map e152 with expected crc 2023-10-09T04:08:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[213841]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T04:08:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:13 smithi163 ceph-mon[213841]: failed to encode map e152 with expected crc 2023-10-09T04:08:14.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:14 smithi181 ceph-mon[196979]: failed to encode map e152 with expected crc 2023-10-09T04:08:14.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:14 smithi181 ceph-mon[196979]: pgmap v387: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 2772/103869 objects misplaced (2.669%) 2023-10-09T04:08:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:14 smithi163 ceph-mon[213841]: failed to encode map e152 with expected crc 2023-10-09T04:08:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:14 smithi163 ceph-mon[213841]: pgmap v387: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 2772/103869 objects misplaced (2.669%) 2023-10-09T04:08:14.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:14 smithi163 ceph-mon[221067]: failed to encode map e152 with expected crc 2023-10-09T04:08:14.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:14 smithi163 ceph-mon[221067]: pgmap v387: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 2772/103869 objects misplaced (2.669%) 2023-10-09T04:08:15.026 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:15.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:15.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:15.166 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:15.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:15.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:15.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:15.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:15.365 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:15.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:15.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:15.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:15.656 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:15 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/100581 objects degraded (3.380%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:15.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:15 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/100581 objects degraded (3.380%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:15.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:15 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/100581 objects degraded (3.380%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:16.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:16.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:16.236 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:16.268 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:16.327 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:16.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:16.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:16.443 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:08:16.467 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:16 smithi181 ceph-mon[196979]: pgmap v388: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 3400/100581 objects degraded (3.380%); 5389/100581 objects misplaced (5.358%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T04:08:16.528 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:16.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:16.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:16.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:16.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:16 smithi163 ceph-mon[213841]: pgmap v388: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 3400/100581 objects degraded (3.380%); 5389/100581 objects misplaced (5.358%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T04:08:16.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:16 smithi163 ceph-mon[221067]: pgmap v388: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 3400/100581 objects degraded (3.380%); 5389/100581 objects misplaced (5.358%); 9.5 MiB/s, 151 objects/s recovering 2023-10-09T04:08:16.772 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:17.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:17.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:17.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:17.017 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:17.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:17.241 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:17.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:17.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:17.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:17.570 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:17.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:17.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:17.714 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:17.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:17.842 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:17.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:17.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:18.083 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:18.126 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:18.196 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:18.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:18.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:18.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:18 smithi163 ceph-mon[213841]: pgmap v389: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 3400/96387 objects degraded (3.527%); 5389/96387 objects misplaced (5.591%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:08:18.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4240356876' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:08:18.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2129580715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:08:18.254 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:08:18.273 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:08:18.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:18.360 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:18.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:18.427 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:18 smithi181 ceph-mon[196979]: pgmap v389: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 3400/96387 objects degraded (3.527%); 5389/96387 objects misplaced (5.591%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:08:18.427 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4240356876' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:08:18.427 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2129580715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:08:18.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:18.528 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:18.591 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:18 smithi163 ceph-mon[221067]: pgmap v389: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 3400/96387 objects degraded (3.527%); 5389/96387 objects misplaced (5.591%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:08:18.591 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4240356876' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:08:18.591 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2129580715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:08:18.614 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4240356876' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:08:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2129580715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:08:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:08:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:19 smithi181 ceph-mon[196979]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4240356876' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:08:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2129580715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:08:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[221067]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4240356876' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2129580715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: failed to encode map e153 with expected crc 2023-10-09T04:08:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:19 smithi163 ceph-mon[213841]: failed to encode map e153 with expected crc 2023-10-09T04:08:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:20 smithi163 ceph-mon[213841]: pgmap v391: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 3400/95079 objects degraded (3.576%); 4784/95079 objects misplaced (5.032%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T04:08:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:20 smithi163 ceph-mon[221067]: pgmap v391: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 3400/95079 objects degraded (3.576%); 4784/95079 objects misplaced (5.032%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T04:08:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:20 smithi181 ceph-mon[196979]: pgmap v391: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 3400/95079 objects degraded (3.576%); 4784/95079 objects misplaced (5.032%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T04:08:21.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:21.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:21.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:21.014 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:08:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:08:20] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:08:21.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:21.143 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:21.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:08:21.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:21 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/87573 objects degraded (3.882%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:21.593 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:21 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/87573 objects degraded (3.882%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:21.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:21.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:21.740 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:21 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/87573 objects degraded (3.882%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:21.776 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:21.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:21.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:21.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:22.015 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:22.189 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:22.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:22.311 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:22.362 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:22.417 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:08:22.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:22.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:22.467 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:08:22.503 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:22 smithi163 ceph-mon[213841]: pgmap v392: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 3400/87573 objects degraded (3.882%); 4431/87573 objects misplaced (5.060%); 10 MiB/s, 159 objects/s recovering 2023-10-09T04:08:22.503 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1926184140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:22.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:22 smithi163 ceph-mon[221067]: pgmap v392: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 3400/87573 objects degraded (3.882%); 4431/87573 objects misplaced (5.060%); 10 MiB/s, 159 objects/s recovering 2023-10-09T04:08:22.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1926184140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:22.525 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:08:22.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:22.584 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:08:22.585 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:22.624 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:08:22.667 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:22.672 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:22 smithi181 ceph-mon[196979]: pgmap v392: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 3400/87573 objects degraded (3.882%); 4431/87573 objects misplaced (5.060%); 10 MiB/s, 159 objects/s recovering 2023-10-09T04:08:22.672 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1926184140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:22.672 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:08:22.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:22.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:22.717 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:08:22.761 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:08:22.775 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:22.780 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.781 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:08:22.827 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:22.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:22.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:23.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:23.049 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:23.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:23.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:23.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:23.333 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:23.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:23.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:23.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:23.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:23.596 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:23.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:08:24.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:24.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:24.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:24.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:24.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:24 smithi163 ceph-mon[221067]: pgmap v393: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 3400/86265 objects degraded (3.941%); 4431/86265 objects misplaced (5.136%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:08:24.559 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:24.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:24.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:24.702 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:24.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:24.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:24 smithi163 ceph-mon[213841]: pgmap v393: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 3400/86265 objects degraded (3.941%); 4431/86265 objects misplaced (5.136%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:08:24.843 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:24 smithi181 ceph-mon[196979]: pgmap v393: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 3400/86265 objects degraded (3.941%); 4431/86265 objects misplaced (5.136%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:08:25.067 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:25.187 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:08:25.187 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:08:25.195 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:25.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:25.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:25.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:25.378 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:25.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:25 smithi163 ceph-mon[213841]: pgmap v394: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 3400/81522 objects degraded (4.171%); 3472/81522 objects misplaced (4.259%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T04:08:25.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:25 smithi163 ceph-mon[221067]: pgmap v394: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 3400/81522 objects degraded (4.171%); 3472/81522 objects misplaced (4.259%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T04:08:26.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:25 smithi181 ceph-mon[196979]: pgmap v394: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 3400/81522 objects degraded (4.171%); 3472/81522 objects misplaced (4.259%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T04:08:26.282 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:26.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:26.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:26.420 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:26.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:08:27.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/81522 objects degraded (4.171%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:27.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/81522 objects degraded (4.171%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:27.530 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/81522 objects degraded (4.171%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:27.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:08:27.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:27.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:27.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:27.881 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:27.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:27.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:27.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:28.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:28.147 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:28.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:28 smithi163 ceph-mon[221067]: pgmap v395: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 3400/76032 objects degraded (4.472%); 3472/76032 objects misplaced (4.566%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T04:08:28.266 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:28 smithi181 ceph-mon[196979]: pgmap v395: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 3400/76032 objects degraded (4.472%); 3472/76032 objects misplaced (4.566%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T04:08:28.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:28.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:28.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:28.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:28 smithi163 ceph-mon[213841]: pgmap v395: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 3400/76032 objects degraded (4.472%); 3472/76032 objects misplaced (4.566%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T04:08:28.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:28.686 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:28.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:28.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:28.848 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:28.949 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:29.031 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:29.159 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:29.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:29.266 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:29.300 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:29.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:29.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:29.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:29.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:29.507 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:29.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:29.561 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:29.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:29.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:29.706 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:29.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:29.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:29.876 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:29.957 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:30.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:30.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:30.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:30.355 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:30.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:30.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:30.483 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:08:30.485 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:08:30.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:30.565 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:30.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:30.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:30 smithi181 ceph-mon[196979]: pgmap v396: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 3400/74820 objects degraded (4.544%); 3385/74820 objects misplaced (4.524%); 6.3 MiB/s, 121 objects/s recovering 2023-10-09T04:08:30.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2117615855' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:08:30.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:08:30.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/359266035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:30.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:30.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:30.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:30.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[213841]: pgmap v396: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 3400/74820 objects degraded (4.544%); 3385/74820 objects misplaced (4.524%); 6.3 MiB/s, 121 objects/s recovering 2023-10-09T04:08:30.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2117615855' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:08:30.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:08:30.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/359266035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:30.776 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:30.777 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:08:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:08:30] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:08:30.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[221067]: pgmap v396: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 3400/74820 objects degraded (4.544%); 3385/74820 objects misplaced (4.524%); 6.3 MiB/s, 121 objects/s recovering 2023-10-09T04:08:30.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2117615855' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:08:30.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:08:30.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/359266035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:30.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:30.778 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:08:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:08:31.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:31 smithi181 ceph-mon[196979]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[221067]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: failed to encode map e154 with expected crc 2023-10-09T04:08:31.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:31 smithi163 ceph-mon[213841]: failed to encode map e154 with expected crc 2023-10-09T04:08:32.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:32.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:32.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:32.542 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:32.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:32.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:32.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: pgmap v398: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 3400/66699 objects degraded (5.098%); 3385/66699 objects misplaced (5.075%); 3.5 MiB/s, 123 objects/s recovering 2023-10-09T04:08:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:08:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/66699 objects degraded (5.098%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: pgmap v398: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 3400/66699 objects degraded (5.098%); 3385/66699 objects misplaced (5.075%); 3.5 MiB/s, 123 objects/s recovering 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/66699 objects degraded (5.098%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:32.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: pgmap v398: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 3400/66699 objects degraded (5.098%); 3385/66699 objects misplaced (5.075%); 3.5 MiB/s, 123 objects/s recovering 2023-10-09T04:08:32.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:08:32.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: failed to encode map e155 with expected crc 2023-10-09T04:08:32.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/66699 objects degraded (5.098%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:08:32.823 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:33.528 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:08:33.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:33.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:33.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:33.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:33 smithi181 ceph-mon[196979]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:33 smithi181 ceph-mon[196979]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:08:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:33 smithi181 ceph-mon[196979]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:33 smithi181 ceph-mon[196979]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[213841]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[213841]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:08:33.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[213841]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.781 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[213841]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[221067]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[221067]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:08:33.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[221067]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:33 smithi163 ceph-mon[221067]: failed to encode map e156 with expected crc 2023-10-09T04:08:33.797 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:34.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:34.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:34.120 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:34.202 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:34.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:34.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:34.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:34.432 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:08:34.481 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:08:34.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:34.534 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:08:34.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:08:34.589 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:34.627 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:08:34.669 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:08:34.710 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:08:34.724 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:08:34.724 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:08:34.724 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:08:34.724 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:08:34.724 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:08:34.733 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:34.746 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:34.749 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:34 smithi163 ceph-mon[213841]: pgmap v401: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 3400/65172 objects degraded (5.217%); 3385/65172 objects misplaced (5.194%); 309 KiB/s, 132 objects/s recovering 2023-10-09T04:08:34.750 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:34.750 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2026215083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:34.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:34 smithi163 ceph-mon[221067]: pgmap v401: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 3400/65172 objects degraded (5.217%); 3385/65172 objects misplaced (5.194%); 309 KiB/s, 132 objects/s recovering 2023-10-09T04:08:34.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:34.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2026215083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:34.751 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:08:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:34.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:08:34.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:34.919 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:35.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:34 smithi181 ceph-mon[196979]: pgmap v401: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 3400/65172 objects degraded (5.217%); 3385/65172 objects misplaced (5.194%); 309 KiB/s, 132 objects/s recovering 2023-10-09T04:08:35.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:35.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2026215083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:35.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:35 smithi163 ceph-mon[213841]: pgmap v402: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 3400/60648 objects degraded (5.606%); 3385/60648 objects misplaced (5.581%); 0 B/s, 189 objects/s recovering 2023-10-09T04:08:35.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:35 smithi163 ceph-mon[221067]: pgmap v402: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 3400/60648 objects degraded (5.606%); 3385/60648 objects misplaced (5.581%); 0 B/s, 189 objects/s recovering 2023-10-09T04:08:35.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:08:36.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:35 smithi181 ceph-mon[196979]: pgmap v402: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 3400/60648 objects degraded (5.606%); 3385/60648 objects misplaced (5.581%); 0 B/s, 189 objects/s recovering 2023-10-09T04:08:36.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:36.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:36.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:36.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:36.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:08:36.699 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:36.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:36.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:37.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:37.129 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:37.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:37.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:37.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:37.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:37.340 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:37.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:37.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:37.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:37.435 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/60648 objects degraded (5.606%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:37.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:37.580 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:37.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/60648 objects degraded (5.606%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:37.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/60648 objects degraded (5.606%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:37.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:37.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:37.766 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:37.826 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:37.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:37.905 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:37.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:38.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:38.103 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:38.116 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:38.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:38.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:38.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:38.359 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:38 smithi163 ceph-mon[213841]: pgmap v403: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 3400/54870 objects degraded (6.196%); 3385/54870 objects misplaced (6.169%); 0 B/s, 101 objects/s recovering 2023-10-09T04:08:38.359 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:38 smithi163 ceph-mon[221067]: pgmap v403: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 3400/54870 objects degraded (6.196%); 3385/54870 objects misplaced (6.169%); 0 B/s, 101 objects/s recovering 2023-10-09T04:08:38.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:38.434 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:38.513 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:38.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:38.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:38.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:08:38.699 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:38 smithi181 ceph-mon[196979]: pgmap v403: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 3400/54870 objects degraded (6.196%); 3385/54870 objects misplaced (6.169%); 0 B/s, 101 objects/s recovering 2023-10-09T04:08:39.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:39.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:39.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:39.190 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:39.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:39.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:39.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:39.373 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:08:39.401 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:39.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:39.448 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4285748313' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:08:39.448 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:08:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:39.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:39.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:39.594 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:39.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:39.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:39.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:39.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4285748313' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:08:39.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:08:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4285748313' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:08:39.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:08:39.865 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:40.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:40.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:40.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:40.400 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:40.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:40.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[213841]: pgmap v405: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3400/53805 objects degraded (6.319%); 3385/53805 objects misplaced (6.291%); 0 B/s, 89 objects/s recovering 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:40 smithi163 ceph-mon[221067]: pgmap v405: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3400/53805 objects degraded (6.319%); 3385/53805 objects misplaced (6.291%); 0 B/s, 89 objects/s recovering 2023-10-09T04:08:40.584 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:40.678 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:40.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:08:40.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:08:40.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: failed to encode map e157 with expected crc 2023-10-09T04:08:40.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:40 smithi181 ceph-mon[196979]: pgmap v405: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3400/53805 objects degraded (6.319%); 3385/53805 objects misplaced (6.291%); 0 B/s, 89 objects/s recovering 2023-10-09T04:08:41.027 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:08:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:08:40] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:08:41.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:41.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:41.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:41.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:41.226 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:41.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:41.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:41.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:41.414 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:41.517 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:41.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:41.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:41.680 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:41.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:08:41.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:41.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:41.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:41.964 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:42.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:42.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:42.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:42.332 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:42.384 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:42.409 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:08:42.409 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:08:42.409 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:08:42.409 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:08:42.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49817 6.c 6:3ffecb2a:::benchmark_data_smithi163_287034_object49816:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 195.70842208299999, 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21672543999999999, 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49817 2023-10-09T04:08:42.410 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687659+0000", 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687682+0000", 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3472000000000002e-05 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687694+0000", 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1080999999999999e-05 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687701+0000", 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9000000000000006e-06 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687750+0000", 2023-10-09T04:08:42.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8877e-05 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687874+0000", 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123812 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.688008+0000", 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134256 2023-10-09T04:08:42.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.690036+0000", 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020273660000000001 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692667+0000", 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026314289999999998 2023-10-09T04:08:42.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904254+0000", 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21158665099999999 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904341+0000", 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7026000000000003e-05 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.416 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904386+0000", 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4781999999999999e-05 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49825 6.c 6:38f6223e:::benchmark_data_smithi163_287034_object49824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 195.703509717, 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.214667942, 2023-10-09T04:08:42.417 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49825 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692571+0000", 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692600+0000", 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9017999999999999e-05 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692612+0000", 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.171e-05 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692620+0000", 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1240000000000005e-06 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692694+0000", 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4127000000000004e-05 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.421 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692812+0000", 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011761599999999999 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:08:42.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692939+0000", 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012688700000000001 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904425+0000", 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.211486216 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904454+0000", 2023-10-09T04:08:42.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9196999999999998e-05 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907121+0000", 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002666982 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907194+0000", 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3281999999999994e-05 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907241+0000", 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6106999999999997e-05 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:49821 6.7 e111/102 6:e6e39e35:::benchmark_data_smithi163_287034_object49820:head v 111'3139, mlcod=111'3137)", 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.693011+0000", 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 195.703071255, 2023-10-09T04:08:42.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21149889599999999, 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693011+0000", 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693011+0000", 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693010+0000", 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.693046+0000", 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5963000000000001e-05 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902261+0000", 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20921489300000001 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902282+0000", 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0928999999999999e-05 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902343+0000", 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0392e-05 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902372+0000", 2023-10-09T04:08:42.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9485999999999999e-05 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904484+0000", 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021115830000000002 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904510+0000", 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6239999999999999e-05 2023-10-09T04:08:42.430 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:49817 6.c e111/102 ondisk, result = 0)", 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.694745+0000", 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 195.70133722899999, 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20965001799999999, 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.431 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694745+0000", 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694745+0000", 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.432 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694744+0000", 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.694751+0000", 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.657e-06 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.433 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902268+0000", 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20751671599999999 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902279+0000", 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1395e-05 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.434 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904236+0000", 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019574560000000002 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904245+0000", 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.435 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904395+0000", 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015013 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49829 6.6 6:64b4de1c:::benchmark_data_smithi163_287034_object49828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.695061+0000", 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 195.70102105999999, 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21301373700000001, 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.436 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49829 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695061+0000", 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.437 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695061+0000", 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695059+0000", 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.438 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.695085+0000", 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5497e-05 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902270+0000", 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20718461099999999 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902285+0000", 2023-10-09T04:08:42.439 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5475e-05 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902344+0000", 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9011000000000001e-05 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902545+0000", 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020052399999999999 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.440 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.902710+0000", 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165833 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.906460+0000", 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037491299999999998 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.441 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.906756+0000", 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029698300000000001 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907983+0000", 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012264559999999999 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.908037+0000", 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3680000000000001e-05 2023-10-09T04:08:42.442 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.908075+0000", 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8368999999999998e-05 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:72550 6.b 6:d0472b57:::benchmark_data_smithi163_287034_object72549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:08:42.443 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:47.323803+0000", 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 175.072279772, 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19669533, 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 72550 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.444 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323803+0000", 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323803+0000", 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.445 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323802+0000", 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323823+0000", 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0995000000000001e-05 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323832+0000", 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2e-06 2023-10-09T04:08:42.446 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323839+0000", 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5989999999999998e-06 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.323927+0000", 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8055999999999999e-05 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.447 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.324061+0000", 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013399899999999999 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.324213+0000", 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000152514 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.448 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.331884+0000", 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076709819999999998 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.331895+0000", 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1195e-05 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520404+0000", 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18850883199999999 2023-10-09T04:08:42.449 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520458+0000", 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3542999999999999e-05 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:47.520498+0000", 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0225999999999997e-05 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.450 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74814 6.8 e115/102 6:16d1b9ba:::benchmark_data_smithi163_287034_object74813:head v 115'4588, mlcod=115'4586)", 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.629639+0000", 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.76644367899999, 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20429694800000001, 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.451 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629639+0000", 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629639+0000", 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629638+0000", 2023-10-09T04:08:42.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629678+0000", 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0129999999999997e-05 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629686+0000", 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3080000000000002e-06 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.453 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629691+0000", 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.254e-06 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629806+0000", 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115343 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.629827+0000", 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1121999999999999e-05 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637214+0000", 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0073863389999999996 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833936+0000", 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19672177099999999 2023-10-09T04:08:42.455 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74819 6.8 e115/102 6:15765e6f:::benchmark_data_smithi163_287034_object74818:head v 115'4589, mlcod=115'4586)", 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.632540+0000", 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.76354243599999, 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20142286600000001, 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.456 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632540+0000", 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632540+0000", 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632539+0000", 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632581+0000", 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1063999999999999e-05 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.458 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632588+0000", 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8990000000000001e-06 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632594+0000", 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1619999999999999e-06 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632633+0000", 2023-10-09T04:08:42.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9753999999999999e-05 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632652+0000", 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9153000000000001e-05 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833941+0000", 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.201288997 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.460 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833963+0000", 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1254e-05 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74820 6.8 e115/102 6:1dfd3969:::benchmark_data_smithi163_287034_object74819:head v 115'4590, mlcod=115'4586)", 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.76347420499999, 2023-10-09T04:08:42.461 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.201374003, 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:08:42.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632608+0000", 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632638+0000", 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9832000000000002e-05 2023-10-09T04:08:42.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632643+0000", 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3720000000000004e-06 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.632648+0000", 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6940000000000003e-06 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635127+0000", 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024792460000000001 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635147+0000", 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9981999999999999e-05 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833965+0000", 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19881852699999999 2023-10-09T04:08:42.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833982+0000", 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6657999999999999e-05 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:74826 6.c 6:3c5736f5:::benchmark_data_smithi163_287034_object74825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:08:42.466 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.635420+0000", 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.76066235299999, 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20139859900000001, 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 74826 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635420+0000", 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635420+0000", 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635419+0000", 2023-10-09T04:08:42.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635443+0000", 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4479999999999999e-05 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635455+0000", 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2052000000000001e-05 2023-10-09T04:08:42.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635462+0000", 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3490000000000003e-06 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635506+0000", 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3711999999999998e-05 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635652+0000", 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014605400000000001 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.635788+0000", 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013596999999999999 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833768+0000", 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.197980351 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833867+0000", 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8288000000000005e-05 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836718+0000", 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028510089999999998 2023-10-09T04:08:42.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836774+0000", 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6316000000000001e-05 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836819+0000", 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4450999999999998e-05 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.473 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:74834 6.6 6:6df70a8f:::benchmark_data_smithi163_287034_object74833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.636840+0000", 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.75924253700001, 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20286137700000001, 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.474 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 74834 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636840+0000", 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636840+0000", 2023-10-09T04:08:42.475 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636839+0000", 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636867+0000", 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8098000000000001e-05 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636883+0000", 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5999999999999999e-05 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636900+0000", 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6571000000000001e-05 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.636945+0000", 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5806999999999997e-05 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637072+0000", 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127007 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637161+0000", 2023-10-09T04:08:42.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8671000000000005e-05 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836353+0000", 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.199191808 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.836502+0000", 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000148782 2023-10-09T04:08:42.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839613+0000", 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031116669999999998 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839658+0000", 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4487999999999998e-05 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839701+0000", 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3353999999999999e-05 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74824 6.f e115/102 6:fbaaa0a0:::benchmark_data_smithi163_287034_object74823:head v 115'4572, mlcod=115'4570)", 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.637749+0000", 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.758333204, 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.201700351, 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.481 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637749+0000", 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637749+0000", 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637748+0000", 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.637771+0000", 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2456999999999999e-05 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833661+0000", 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19588987499999999 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833675+0000", 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.483e-05 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833727+0000", 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1326000000000003e-05 2023-10-09T04:08:42.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833756+0000", 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9680999999999999e-05 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839417+0000", 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005661068 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839449+0000", 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2044e-05 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.44666.0:74826 6.c e115/102 ondisk, result = 0)", 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.638598+0000", 2023-10-09T04:08:42.486 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.75748421, 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19517346999999999, 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638598+0000", 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638598+0000", 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638597+0000", 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.638603+0000", 2023-10-09T04:08:42.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2780000000000001e-06 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833666+0000", 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.195063559 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833679+0000", 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2798e-05 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833754+0000", 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4839999999999995e-05 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833762+0000", 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9500000000000001e-06 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.490 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833772+0000", 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6870000000000006e-06 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.44666.0:74831 6.5 e115/102 6:ae4cc717:::benchmark_data_smithi163_287034_object74830:head v 115'4683, mlcod=115'4681)", 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:49.639852+0000", 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.756230362, 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19929846000000001, 2023-10-09T04:08:42.491 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.492 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.492 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.492 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.492 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639852+0000", 2023-10-09T04:08:42.492 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639852+0000", 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639851+0000", 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:08:42.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.639901+0000", 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9347e-05 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833669+0000", 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.193768836 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833681+0000", 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1488000000000001e-05 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833804+0000", 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123533 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.833826+0000", 2023-10-09T04:08:42.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1231999999999999e-05 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839114+0000", 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052885750000000002 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:49.839150+0000", 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6173000000000001e-05 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.496 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117673 6.b 6:d92697e4:::benchmark_data_smithi163_287034_object9410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.556461+0000", 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 120.839621509, 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.212801672, 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.497 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117673 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556461+0000", 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556461+0000", 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556459+0000", 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556467+0000", 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8019999999999992e-06 2023-10-09T04:08:42.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556484+0000", 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7116e-05 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556492+0000", 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1219999999999995e-06 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556536+0000", 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3903999999999998e-05 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556667+0000", 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000131233 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.556805+0000", 2023-10-09T04:08:42.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013789600000000001 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.560507+0000", 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003701321 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565033+0000", 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045267170000000004 2023-10-09T04:08:42.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.769160+0000", 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20412709800000001 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.769220+0000", 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0000000000000002e-05 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.769262+0000", 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2030000000000002e-05 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117683 6.b 6:df64dfc0:::benchmark_data_smithi163_287034_object9420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 120.834260918, 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21390703, 2023-10-09T04:08:42.504 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117683 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561824+0000", 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.525e-06 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561832+0000", 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6060000000000003e-06 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561837+0000", 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5090000000000004e-06 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561900+0000", 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2890000000000003e-05 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.562018+0000", 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011803699999999999 2023-10-09T04:08:42.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.562119+0000", 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101275 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565635+0000", 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035162940000000001 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.568886+0000", 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032510759999999999 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775646+0000", 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.206759888 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775692+0000", 2023-10-09T04:08:42.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5757999999999998e-05 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775728+0000", 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6587000000000001e-05 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117690 6.16 6:6acd74ed:::benchmark_data_smithi163_287034_object9427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:08:42.511 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.565518+0000", 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 120.83056449599999, 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.207014166, 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117690 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565518+0000", 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565518+0000", 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565516+0000", 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565523+0000", 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8889999999999996e-06 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565535+0000", 2023-10-09T04:08:42.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992e-05 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565542+0000", 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9480000000000002e-06 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565583+0000", 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0784000000000001e-05 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565714+0000", 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013164700000000001 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.566032+0000", 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031758099999999999 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.567619+0000", 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015872709999999999 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.572154+0000", 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045347080000000001 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.772448+0000", 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.200293998 2023-10-09T04:08:42.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.772496+0000", 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8421000000000003e-05 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.772532+0000", 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5617000000000001e-05 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.518 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117698 6.b 6:ddb7b18d:::benchmark_data_smithi163_287034_object9435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.573595+0000", 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 120.82248721000001, 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.205324229, 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117698 2023-10-09T04:08:42.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573595+0000", 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573595+0000", 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573594+0000", 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573600+0000", 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.494e-06 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573615+0000", 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4701999999999999e-05 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573623+0000", 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7970000000000001e-06 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573669+0000", 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6350000000000002e-05 2023-10-09T04:08:42.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573762+0000", 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2372000000000003e-05 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.573831+0000", 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9620000000000001e-05 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.584319+0000", 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010487575000000001 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.584338+0000", 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9004999999999998e-05 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.778835+0000", 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.1944969 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.778880+0000", 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5374000000000003e-05 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.778919+0000", 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9267000000000002e-05 2023-10-09T04:08:42.525 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117700 6.b 6:d4f1f752:::benchmark_data_smithi163_287034_object9437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.576448+0000", 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 120.819634275, 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20256489899999999, 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.526 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117700 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576448+0000", 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576448+0000", 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576447+0000", 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576453+0000", 2023-10-09T04:08:42.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8690000000000001e-06 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576468+0000", 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5077e-05 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576477+0000", 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4329999999999993e-06 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576586+0000", 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000108575 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576704+0000", 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118476 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:08:42.530 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576818+0000", 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000114023 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.584343+0000", 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075247430000000004 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.587405+0000", 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003061673 2023-10-09T04:08:42.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.778952+0000", 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19154690299999999 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.778984+0000", 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1943000000000002e-05 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.779013+0000", 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9388999999999999e-05 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117701 6.b 6:d1e1e429:::benchmark_data_smithi163_287034_object9438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.576489+0000", 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 120.819593346, 2023-10-09T04:08:42.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20317095499999999, 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117701 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576489+0000", 2023-10-09T04:08:42.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576489+0000", 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576488+0000", 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576491+0000", 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.027e-06 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576498+0000", 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3949999999999998e-06 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:08:42.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.576504+0000", 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7289999999999997e-06 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.580056+0000", 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035521229999999999 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.580176+0000", 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120023 2023-10-09T04:08:42.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.580291+0000", 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011506900000000001 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.584348+0000", 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0040574000000000001 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.587610+0000", 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032618970000000001 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.779587+0000", 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19197736700000001 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.779628+0000", 2023-10-09T04:08:42.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0312999999999998e-05 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.779660+0000", 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2178000000000003e-05 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:08:42.540 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:08:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:42.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:42.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:42.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:42.703 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:42.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:42 smithi181 ceph-mon[196979]: pgmap v406: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 947 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3400/45525 objects degraded (7.468%); 3385/45525 objects misplaced (7.435%); 0 B/s, 130 objects/s recovering 2023-10-09T04:08:42.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:42 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:42.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/45525 objects degraded (7.468%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:42.790 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:42.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:42 smithi163 ceph-mon[213841]: pgmap v406: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 947 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3400/45525 objects degraded (7.468%); 3385/45525 objects misplaced (7.435%); 0 B/s, 130 objects/s recovering 2023-10-09T04:08:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:42 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:42.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/45525 objects degraded (7.468%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:42.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:42 smithi163 ceph-mon[221067]: pgmap v406: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 947 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3400/45525 objects degraded (7.468%); 3385/45525 objects misplaced (7.435%); 0 B/s, 130 objects/s recovering 2023-10-09T04:08:42.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:42 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:42.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/45525 objects degraded (7.468%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:42.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:42.973 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:43.455 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:43.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:43.529 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:08:43.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:43.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:08:43.719 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:43.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/318420534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:08:43.722 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:43 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:08:43.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/318420534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:08:43.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:43 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:08:43.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/318420534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:08:43.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:43 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:08:44.264 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:44.359 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:08:44.586 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:44.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:44.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:44.723 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: pgmap v407: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3400/43833 objects degraded (7.757%); 3385/43833 objects misplaced (7.722%); 0 B/s, 116 objects/s recovering 2023-10-09T04:08:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: failed to encode map e158 with expected crc 2023-10-09T04:08:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:08:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:08:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: failed to encode map e158 with expected crc 2023-10-09T04:08:44.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:44 smithi181 ceph-mon[196979]: 6.10 scrub starts 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: pgmap v407: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3400/43833 objects degraded (7.757%); 3385/43833 objects misplaced (7.722%); 0 B/s, 116 objects/s recovering 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: failed to encode map e158 with expected crc 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: failed to encode map e158 with expected crc 2023-10-09T04:08:44.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[221067]: 6.10 scrub starts 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: pgmap v407: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 912 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3400/43833 objects degraded (7.757%); 3385/43833 objects misplaced (7.722%); 0 B/s, 116 objects/s recovering 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: failed to encode map e158 with expected crc 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: failed to encode map e158 with expected crc 2023-10-09T04:08:44.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:44 smithi163 ceph-mon[213841]: 6.10 scrub starts 2023-10-09T04:08:45.608 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:45.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:45.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:45 smithi163 ceph-mon[213841]: 6.10 scrub ok 2023-10-09T04:08:45.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:45 smithi163 ceph-mon[213841]: 6.11 scrub starts 2023-10-09T04:08:45.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:45 smithi163 ceph-mon[221067]: 6.10 scrub ok 2023-10-09T04:08:45.685 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:45 smithi163 ceph-mon[221067]: 6.11 scrub starts 2023-10-09T04:08:45.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:45 smithi181 ceph-mon[196979]: 6.10 scrub ok 2023-10-09T04:08:45.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:45 smithi181 ceph-mon[196979]: 6.11 scrub starts 2023-10-09T04:08:45.788 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:45.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:45.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:45.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:08:45.998 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:46.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:08:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:46.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:08:46.415 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:08:46.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:46.464 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:46.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:46.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:46 smithi163 ceph-mon[221067]: 6.11 scrub ok 2023-10-09T04:08:46.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:46 smithi163 ceph-mon[221067]: pgmap v409: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 818 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 3400/39318 objects degraded (8.647%); 3385/39318 objects misplaced (8.609%); 0 B/s, 136 objects/s recovering 2023-10-09T04:08:46.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:46 smithi163 ceph-mon[213841]: 6.11 scrub ok 2023-10-09T04:08:46.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:46 smithi163 ceph-mon[213841]: pgmap v409: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 818 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 3400/39318 objects degraded (8.647%); 3385/39318 objects misplaced (8.609%); 0 B/s, 136 objects/s recovering 2023-10-09T04:08:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:46.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:46.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:46 smithi181 ceph-mon[196979]: 6.11 scrub ok 2023-10-09T04:08:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:46 smithi181 ceph-mon[196979]: pgmap v409: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 818 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 3400/39318 objects degraded (8.647%); 3385/39318 objects misplaced (8.609%); 0 B/s, 136 objects/s recovering 2023-10-09T04:08:46.831 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:46.845 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:08:46.896 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:08:46.938 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:08:46.983 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:08:47.038 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:08:47.051 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:08:47.051 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:08:47.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:47.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:47.091 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:08:47.125 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:08:47.152 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:08:47.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:47.226 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:47.390 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:47.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:47.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:47.525 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2006762341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3400/39318 objects degraded (8.647%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:47.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:47 smithi181 ceph-mon[196979]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:08:47.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2006762341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3400/39318 objects degraded (8.647%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:47.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[221067]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:08:47.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:08:47.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: failed to encode map e159 with expected crc 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2006762341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3400/39318 objects degraded (8.647%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:08:47.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:47 smithi163 ceph-mon[213841]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:08:48.058 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:48.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:48.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:08:48.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:48.351 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:48.584 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: pgmap v411: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 703 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3400/33810 objects degraded (10.056%); 3385/33810 objects misplaced (10.012%) 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3411248878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: pgmap v411: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 703 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3400/33810 objects degraded (10.056%); 3385/33810 objects misplaced (10.012%) 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3411248878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:08:48.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: failed to encode map e160 with expected crc 2023-10-09T04:08:48.625 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:48 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:08:48.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:48.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:48.970 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:08:49.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: pgmap v411: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 703 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3400/33810 objects degraded (10.056%); 3385/33810 objects misplaced (10.012%) 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: failed to encode map e160 with expected crc 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: failed to encode map e160 with expected crc 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: failed to encode map e160 with expected crc 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3411248878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: failed to encode map e160 with expected crc 2023-10-09T04:08:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:48 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:08:49.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:49.108 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:49.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:49.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:49.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:08:49.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:08:49.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[221067]: pgmap v414: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 687 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3400/33003 objects degraded (10.302%); 3385/33003 objects misplaced (10.257%) 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:08:49.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:49.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:49 smithi163 ceph-mon[213841]: pgmap v414: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 687 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3400/33003 objects degraded (10.302%); 3385/33003 objects misplaced (10.257%) 2023-10-09T04:08:49.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:08:49.921 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: failed to encode map e161 with expected crc 2023-10-09T04:08:49.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:49.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:49.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:49.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:49 smithi181 ceph-mon[196979]: pgmap v414: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 687 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3400/33003 objects degraded (10.302%); 3385/33003 objects misplaced (10.257%) 2023-10-09T04:08:49.950 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:49.950 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:50.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:50.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:50.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:50.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:50.221 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:50.381 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:50.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:50.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:50.556 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:50.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:50.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:50.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:50.824 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:50 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:08:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:50 smithi163 ceph-mon[221067]: 6.15 scrub starts 2023-10-09T04:08:50.834 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:08:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:08:50] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.43.0" 2023-10-09T04:08:50.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:50 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:08:50.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:50 smithi163 ceph-mon[213841]: 6.15 scrub starts 2023-10-09T04:08:50.867 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:08:50.867 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:08:50.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:50 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:08:50.890 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:50 smithi181 ceph-mon[196979]: 6.15 scrub starts 2023-10-09T04:08:51.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:51.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:51.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:51.220 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:51.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:51.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:51.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:51.449 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:51.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:08:51.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:51.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:51.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:51 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:08:51.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:51 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:08:51.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:51 smithi163 ceph-mon[213841]: pgmap v415: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 547 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 743/26280 objects degraded (2.827%); 1147/26280 objects misplaced (4.365%) 2023-10-09T04:08:51.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:51 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:08:51.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:51 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:08:51.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:51 smithi163 ceph-mon[221067]: pgmap v415: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 547 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 743/26280 objects degraded (2.827%); 1147/26280 objects misplaced (4.365%) 2023-10-09T04:08:51.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:08:51.783 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:51.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:51.826 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:51.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:51.917 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:51 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:08:51.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:51 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:08:51.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:51 smithi181 ceph-mon[196979]: pgmap v415: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 547 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 743/26280 objects degraded (2.827%); 1147/26280 objects misplaced (4.365%) 2023-10-09T04:08:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:51.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:52.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:52.060 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:52.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:52.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:52.200 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:08:52.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:52.282 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:52.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:52.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:52.442 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:52.487 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:52.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:52.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:52.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:52.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:52 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:08:52.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:52 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:08:52.792 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 743/26280 objects degraded (2.827%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:08:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:52 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:08:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:52 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:08:52.793 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 743/26280 objects degraded (2.827%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:08:52.809 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:52.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:52.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:53.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:52 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:08:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:52 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:08:53.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 743/26280 objects degraded (2.827%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:08:53.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:53.072 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:53.145 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:53.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:53.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:53.288 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:53.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:53.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:53.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:08:53.550 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:53.626 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:53 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:08:53.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1112828904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:53.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:53 smithi181 ceph-mon[196979]: pgmap v416: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 519 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 743/24942 objects degraded (2.979%); 1147/24942 objects misplaced (4.599%) 2023-10-09T04:08:53.656 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:08:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:53 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:08:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1112828904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:53.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:53 smithi163 ceph-mon[213841]: pgmap v416: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 519 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 743/24942 objects degraded (2.979%); 1147/24942 objects misplaced (4.599%) 2023-10-09T04:08:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:53 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:08:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1112828904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:08:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:53 smithi163 ceph-mon[221067]: pgmap v416: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 519 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 743/24942 objects degraded (2.979%); 1147/24942 objects misplaced (4.599%) 2023-10-09T04:08:53.881 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:08:53.881 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:08:54.307 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:08:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:54.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:08:54.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:54.464 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:54.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:54.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:54.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1112828904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:54 smithi181 ceph-mon[196979]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:54.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:08:54.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1112828904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:54.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:08:54.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[221067]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1112828904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:54 smithi163 ceph-mon[213841]: failed to encode map e162 with expected crc 2023-10-09T04:08:54.890 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:54.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:54.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:54.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:55.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:55.167 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:55.194 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:55.269 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:08:55.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:55.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:55.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:08:55.746 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:55.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:55 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:08:55.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:55 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:08:55.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:55 smithi163 ceph-mon[221067]: pgmap v418: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 433 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 743/20811 objects degraded (3.570%); 797/20811 objects misplaced (3.830%); 2.6 MiB/s, 61 objects/s recovering 2023-10-09T04:08:55.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:55 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:08:55.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:55 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:08:55.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:55 smithi163 ceph-mon[213841]: pgmap v418: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 433 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 743/20811 objects degraded (3.570%); 797/20811 objects misplaced (3.830%); 2.6 MiB/s, 61 objects/s recovering 2023-10-09T04:08:55.840 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:55 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:08:55.840 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:55 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:08:55.840 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:55 smithi181 ceph-mon[196979]: pgmap v418: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 433 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 743/20811 objects degraded (3.570%); 797/20811 objects misplaced (3.830%); 2.6 MiB/s, 61 objects/s recovering 2023-10-09T04:08:55.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:08:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:55.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:08:56.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:56.040 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:56.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:56.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:56.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:56.256 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:56.727 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:08:56.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2437111573' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:08:56.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2437111573' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:08:56.862 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2437111573' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:08:57.132 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:08:57.132 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:08:57.252 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:08:57.744 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:57.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:57.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:57.850 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:08:57.897 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:57.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:08:57.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: 6.6 scrub starts 2023-10-09T04:08:57.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: failed to encode map e163 with expected crc 2023-10-09T04:08:57.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2437111573' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:08:57.953 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:08:57.954 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: failed to encode map e163 with expected crc 2023-10-09T04:08:57.954 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 743/20811 objects degraded (3.570%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:08:57.954 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:57 smithi181 ceph-mon[196979]: pgmap v420: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 305 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 743/14697 objects degraded (5.055%); 797/14697 objects misplaced (5.423%); 2.6 MiB/s, 60 objects/s recovering 2023-10-09T04:08:57.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:08:57.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:57.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:08:57.992 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:08:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:08:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: 6.6 scrub starts 2023-10-09T04:08:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: failed to encode map e163 with expected crc 2023-10-09T04:08:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2437111573' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:08:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:08:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: failed to encode map e163 with expected crc 2023-10-09T04:08:58.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 743/20811 objects degraded (3.570%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:08:58.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[213841]: pgmap v420: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 305 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 743/14697 objects degraded (5.055%); 797/14697 objects misplaced (5.423%); 2.6 MiB/s, 60 objects/s recovering 2023-10-09T04:08:58.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: 6.6 scrub starts 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: failed to encode map e163 with expected crc 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2437111573' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: failed to encode map e163 with expected crc 2023-10-09T04:08:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 743/20811 objects degraded (3.570%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:08:58.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:57 smithi163 ceph-mon[221067]: pgmap v420: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 305 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 743/14697 objects degraded (5.055%); 797/14697 objects misplaced (5.423%); 2.6 MiB/s, 60 objects/s recovering 2023-10-09T04:08:58.183 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:58.528 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:08:58.546 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:58.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:58.658 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:58.711 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:58.724 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:08:58.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:08:58.773 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:58.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:58.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:58.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:58.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:58 smithi163 ceph-mon[213841]: 6.15 scrub starts 2023-10-09T04:08:58.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:58.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:58.820 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:58 smithi163 ceph-mon[221067]: 6.15 scrub starts 2023-10-09T04:08:58.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:08:58.996 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:58 smithi181 ceph-mon[196979]: 6.15 scrub starts 2023-10-09T04:08:59.083 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:59.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:08:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:59.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:08:59.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:08:59.258 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:08:59.287 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:59.307 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:08:59.341 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:08:59.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:08:59.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:59.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:08:59.392 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:08:59.444 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:08:59.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:08:59.506 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:08:59.507 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:59.565 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:08:59.612 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:08:59.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:08:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:59.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:08:59.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:59.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:59.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:59.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3891911827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:59.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[213841]: pgmap v421: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 286 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 743/13782 objects degraded (5.391%); 797/13782 objects misplaced (5.783%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:08:59.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:08:59.887 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:08:59.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:08:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:08:59.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:08:59.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:59.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:59.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:08:59.995 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3891911827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:08:59.995 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:08:59 smithi181 ceph-mon[196979]: pgmap v421: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 286 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 743/13782 objects degraded (5.391%); 797/13782 objects misplaced (5.783%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:09:00.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:00.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3891911827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:00.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:08:59 smithi163 ceph-mon[221067]: pgmap v421: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 286 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 743/13782 objects degraded (5.391%); 797/13782 objects misplaced (5.783%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T04:09:00.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:00.155 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:00.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:00.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:00.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:00.400 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:09:00.400 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:09:00.400 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:09:00.400 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:09:00.400 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:09:00.459 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:00.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:09:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:09:00] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.43.0" 2023-10-09T04:09:01.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:09:01.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:01 smithi163 ceph-mon[221067]: 6.5 scrub starts 2023-10-09T04:09:01.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:01 smithi163 ceph-mon[213841]: 6.5 scrub starts 2023-10-09T04:09:01.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:01 smithi181 ceph-mon[196979]: 6.5 scrub starts 2023-10-09T04:09:01.533 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:01.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:01.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:01.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:01.683 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:01.689 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:01.779 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:01.780 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:01.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:01.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:01.953 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:02.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:02.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:02.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:02.183 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:02.256 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:02 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:09:02.256 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:02 smithi181 ceph-mon[196979]: pgmap v422: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 132 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 743/6384 objects degraded (11.638%); 743/6384 objects misplaced (11.638%); 1.6 MiB/s, 43 objects/s recovering 2023-10-09T04:09:02.256 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:02 smithi181 ceph-mon[196979]: 6.16 scrub starts 2023-10-09T04:09:02.256 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:02 smithi181 ceph-mon[196979]: 6.18 scrub starts 2023-10-09T04:09:02.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:02.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:02.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[221067]: pgmap v422: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 132 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 743/6384 objects degraded (11.638%); 743/6384 objects misplaced (11.638%); 1.6 MiB/s, 43 objects/s recovering 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[221067]: 6.16 scrub starts 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[221067]: 6.18 scrub starts 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[213841]: pgmap v422: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 132 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 743/6384 objects degraded (11.638%); 743/6384 objects misplaced (11.638%); 1.6 MiB/s, 43 objects/s recovering 2023-10-09T04:09:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[213841]: 6.16 scrub starts 2023-10-09T04:09:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:02 smithi163 ceph-mon[213841]: 6.18 scrub starts 2023-10-09T04:09:02.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:02.413 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:02.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:02.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:02.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:02.608 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:02.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:02.667 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 108260 objects 2023-10-09T04:09:02.667 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :152.636 2023-10-09T04:09:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:02.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:02.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:02.804 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:02.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:02.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:02.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:02.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:02.989 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:03.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:03.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:03.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:03.171 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:03.232 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:09:03.257 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:03 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 743/6384 objects degraded (11.638%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:03.257 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:03 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1864908982' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:03.257 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:03 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:03.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:03 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 743/6384 objects degraded (11.638%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:03.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:03 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1864908982' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:03.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:03 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:03.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:03.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:03.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:03.343 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:09:03.343 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:49821 6.7 6:e6e39e35:::benchmark_data_smithi163_287034_object49820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:26.692589+0000", 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 216.64843843700001, 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.212208916, 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.346 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 49821 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692589+0000", 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692589+0000", 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692588+0000", 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692640+0000", 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2082e-05 2023-10-09T04:09:03.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692654+0000", 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3631999999999999e-05 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692660+0000", 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8429999999999998e-06 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692711+0000", 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1452000000000001e-05 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692817+0000", 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010610799999999999 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692954+0000", 2023-10-09T04:09:03.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013706999999999999 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.695117+0000", 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021632380000000001 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696456+0000", 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013387200000000001 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.904705+0000", 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20824909599999999 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.904758+0000", 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2609999999999999e-05 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.904798+0000", 2023-10-09T04:09:03.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0308000000000001e-05 2023-10-09T04:09:03.353 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.353 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.353 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.353 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:72545 6.7 6:e4211a21:::benchmark_data_smithi163_287034_object72544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:47.323767+0000", 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 196.01726090099999, 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.196504911, 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72545 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323767+0000", 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323767+0000", 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323766+0000", 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323882+0000", 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116344 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323905+0000", 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3388999999999999e-05 2023-10-09T04:09:03.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323912+0000", 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.72e-06 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.323960+0000", 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7527000000000001e-05 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.324074+0000", 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114881 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.324182+0000", 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107188 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.326717+0000", 2023-10-09T04:09:03.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025357539999999999 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.326882+0000", 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000164714 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520170+0000", 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.19328774700000001 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520229+0000", 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9098999999999998e-05 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:47.520272+0000", 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2688000000000002e-05 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:74830 6.7 6:eb8bab59:::benchmark_data_smithi163_287034_object74829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:49.642634+0000", 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 193.698393229, 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.200693344, 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 74830 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.361 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642634+0000", 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642634+0000", 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642633+0000", 2023-10-09T04:09:03.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:03.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642679+0000", 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5886000000000001e-05 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642692+0000", 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3062e-05 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642698+0000", 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7080000000000002e-06 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642744+0000", 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6353000000000002e-05 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642856+0000", 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111678 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.643006+0000", 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015066400000000001 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.647826+0000", 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048200309999999998 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.647839+0000", 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2446000000000001e-05 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.843241+0000", 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.19540242499999999 2023-10-09T04:09:03.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.843292+0000", 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1063999999999998e-05 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.843328+0000", 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5605000000000002e-05 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.368 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:74832 6.7 6:e9d54e03:::benchmark_data_smithi163_287034_object74831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:49.642707+0000", 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 193.69832025900001, 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20554681799999999, 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.369 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 74832 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642707+0000", 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642707+0000", 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642707+0000", 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642745+0000", 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8334000000000003e-05 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642750+0000", 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.476e-06 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.642753+0000", 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8399999999999999e-06 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.645534+0000", 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027817480000000001 2023-10-09T04:09:03.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.645654+0000", 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119777 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.645778+0000", 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012349999999999999 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.647922+0000", 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021441659999999999 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.648564+0000", 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064189200000000003 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.848167+0000", 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.19960303900000001 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.848216+0000", 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9113999999999998e-05 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:49.848254+0000", 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8408999999999998e-05 2023-10-09T04:09:03.375 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118877 6.a 6:5646758f:::benchmark_data_smithi163_287034_object10614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 140.330231997, 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24332414399999999, 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.376 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118877 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:09:03.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010794+0000", 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010802+0000", 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3140000000000004e-06 2023-10-09T04:09:03.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010818+0000", 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5945000000000001e-05 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010825+0000", 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1659999999999997e-06 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010868+0000", 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2886999999999998e-05 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148662+0000", 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.137794378 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148810+0000", 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147527 2023-10-09T04:09:03.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.152530+0000", 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003719703 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.153038+0000", 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050862599999999998 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254050+0000", 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.101012064 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254090+0000", 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9533999999999998e-05 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254120+0000", 2023-10-09T04:09:03.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9903000000000002e-05 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:140890 6.d 6:bf7f2424:::benchmark_data_smithi163_287034_object32627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:13.205656+0000", 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 110.13537144199999, 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.197470546, 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.383 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 140890 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205656+0000", 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205656+0000", 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205655+0000", 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205660+0000", 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6749999999999999e-06 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205673+0000", 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2649999999999999e-05 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205678+0000", 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6489999999999999e-06 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205725+0000", 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7351000000000001e-05 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205861+0000", 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000136188 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205966+0000", 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010428 2023-10-09T04:09:03.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.217925+0000", 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011959245 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.286788+0000", 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.068862758999999996 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.403023+0000", 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.116235914 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.403081+0000", 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7315999999999998e-05 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.403127+0000", 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6013000000000001e-05 2023-10-09T04:09:03.389 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:140893 6.d 6:b7d2fbf6:::benchmark_data_smithi163_287034_object32630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:13.205688+0000", 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 110.13533945099999, 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.203262255, 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.390 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 140893 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205688+0000", 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205688+0000", 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205688+0000", 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205690+0000", 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2189999999999998e-06 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205694+0000", 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6509999999999999e-06 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.205696+0000", 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3640000000000002e-06 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.210156+0000", 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044595939999999999 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.210257+0000", 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101019 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.210352+0000", 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4785999999999997e-05 2023-10-09T04:09:03.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.217930+0000", 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0075785089999999998 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.293400+0000", 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.075470010000000004 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.408887+0000", 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11548702800000001 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.408924+0000", 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7282e-05 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.408950+0000", 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6089e-05 2023-10-09T04:09:03.396 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:140899 6.d 6:b2fffee7:::benchmark_data_smithi163_287034_object32636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:13.208868+0000", 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 110.132159745, 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20024135500000001, 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.397 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 140899 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.208868+0000", 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.208868+0000", 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.208867+0000", 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.208872+0000", 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8119999999999998e-06 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.208881+0000", 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2240000000000001e-06 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.208886+0000", 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2220000000000001e-06 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.214765+0000", 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0058787370000000002 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.214891+0000", 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012569400000000001 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.215011+0000", 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120541 2023-10-09T04:09:03.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.218048+0000", 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030370829999999999 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.293444+0000", 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.075395424000000003 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.409013+0000", 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115569331 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.409060+0000", 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6891000000000002e-05 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:13.409109+0000", 2023-10-09T04:09:03.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9379000000000003e-05 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151856 6.d 6:ba33a9cc:::benchmark_data_smithi163_287034_object43593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 95.714063295000003, 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23517902700000001, 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.404 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151856 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626966+0000", 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.877e-06 2023-10-09T04:09:03.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626972+0000", 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7089999999999999e-06 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626975+0000", 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1319999999999998e-06 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627008+0000", 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3343999999999998e-05 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627097+0000", 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8706000000000001e-05 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627179+0000", 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2310000000000003e-05 2023-10-09T04:09:03.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.631260+0000", 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040813139999999999 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794861+0000", 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16360079 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862048+0000", 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067187075999999998 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862105+0000", 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6997000000000003e-05 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862143+0000", 2023-10-09T04:09:03.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8050999999999999e-05 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151866 6.d 6:b4107b04:::benchmark_data_smithi163_287034_object43603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 95.704106503999995, 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22535977800000001, 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.411 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151866 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636920+0000", 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636926+0000", 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:09:03.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636939+0000", 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2904e-05 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636944+0000", 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8879999999999996e-06 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636983+0000", 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8402000000000003e-05 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637122+0000", 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000139307 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637246+0000", 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012406599999999999 2023-10-09T04:09:03.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.641603+0000", 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043568410000000002 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794991+0000", 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15338833599999999 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862183+0000", 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067192247999999996 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862248+0000", 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4317000000000001e-05 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862281+0000", 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3144e-05 2023-10-09T04:09:03.417 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169261 6.d 6:b3c8e3b4:::benchmark_data_smithi163_287034_object60998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.330185763000003, 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22831853499999999, 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.418 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169261 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010841+0000", 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010846+0000", 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1510000000000003e-06 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010858+0000", 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1446e-05 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010864+0000", 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3620000000000002e-06 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010914+0000", 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9552000000000003e-05 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011016+0000", 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102262 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011120+0000", 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104622 2023-10-09T04:09:03.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020057+0000", 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089365069999999998 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179106+0000", 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15904936 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239056+0000", 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.059950160000000002 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239115+0000", 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8446000000000003e-05 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239160+0000", 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5503999999999997e-05 2023-10-09T04:09:03.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169262 6.d 6:b9ed22f3:::benchmark_data_smithi163_287034_object60999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 72.325168864999995, 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23726620200000001, 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169262 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015858+0000", 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015863+0000", 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.87e-06 2023-10-09T04:09:03.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015871+0000", 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7810000000000005e-06 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015874+0000", 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6440000000000002e-06 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015992+0000", 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117414 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016078+0000", 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6527999999999996e-05 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016159+0000", 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0648000000000004e-05 2023-10-09T04:09:03.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020242+0000", 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040829209999999998 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179222+0000", 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15898009699999999 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253062+0000", 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.073840684000000004 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253100+0000", 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7886000000000003e-05 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253125+0000", 2023-10-09T04:09:03.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4675999999999999e-05 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205752 6.a 6:5755c8f4:::benchmark_data_smithi163_287034_object97489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.468797002999999, 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22375711200000001, 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.432 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205752 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872230+0000", 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872233+0000", 2023-10-09T04:09:03.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8949999999999998e-06 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872238+0000", 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2449999999999998e-06 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872241+0000", 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8600000000000001e-06 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872286+0000", 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5157999999999997e-05 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872387+0000", 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010013699999999999 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:09:03.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872470+0000", 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3599999999999999e-05 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881422+0000", 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089518930000000007 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881460+0000", 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7812000000000003e-05 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095901+0000", 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.214440778 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095945+0000", 2023-10-09T04:09:03.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4332000000000002e-05 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095988+0000", 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2778000000000003e-05 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205754 6.d 6:b347cae0:::benchmark_data_smithi163_287034_object97491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.468775252, 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22360279999999999, 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205754 2023-10-09T04:09:03.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872255+0000", 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.796e-06 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872261+0000", 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.117e-06 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872265+0000", 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6839999999999998e-06 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877378+0000", 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051136020000000001 2023-10-09T04:09:03.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877474+0000", 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5499000000000002e-05 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877596+0000", 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012222700000000001 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.882034+0000", 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044384189999999999 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.890599+0000", 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085642140000000005 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891540+0000", 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00094105299999999996 2023-10-09T04:09:03.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891569+0000", 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9382000000000001e-05 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095855+0000", 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.204286197 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.446 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205761 6.a 6:5be756cf:::benchmark_data_smithi163_287034_object97498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.463614329999999, 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.218706593, 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.447 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205761 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877412+0000", 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877421+0000", 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4700000000000002e-06 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877430+0000", 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8929999999999994e-06 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877434+0000", 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9559999999999996e-06 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877558+0000", 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012344299999999999 2023-10-09T04:09:03.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877639+0000", 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1212999999999996e-05 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877720+0000", 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0533999999999998e-05 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881593+0000", 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003873479 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881717+0000", 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123604 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096041+0000", 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21432412100000001 2023-10-09T04:09:03.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096082+0000", 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1437999999999999e-05 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096120+0000", 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7599999999999999e-05 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205765 6.a 6:57307337:::benchmark_data_smithi163_287034_object97502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.883543+0000", 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.457484487999999, 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.212665823, 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205765 2023-10-09T04:09:03.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883543+0000", 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883543+0000", 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883542+0000", 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883546+0000", 2023-10-09T04:09:03.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0899999999999998e-06 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883555+0000", 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.405e-06 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883559+0000", 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.403e-06 2023-10-09T04:09:03.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883593+0000", 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3691e-05 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883668+0000", 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4684000000000005e-05 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883738+0000", 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0525000000000007e-05 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.887458+0000", 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037203119999999999 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.887470+0000", 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2109e-05 2023-10-09T04:09:03.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096151+0000", 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20868013199999999 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096177+0000", 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6112999999999999e-05 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096209+0000", 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2277999999999998e-05 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:205759 6.4 e158/124 6:2231df6a:::benchmark_data_smithi163_287034_object97496:head v 158'10014, mlcod=158'10012)", 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.889431+0000", 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.451596374000001, 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.206444344, 2023-10-09T04:09:03.462 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889431+0000", 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889431+0000", 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889431+0000", 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889439+0000", 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.473e-06 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889447+0000", 2023-10-09T04:09:03.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4090000000000003e-06 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889452+0000", 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1080000000000003e-06 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889494+0000", 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2141999999999997e-05 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889513+0000", 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9009e-05 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095857+0000", 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.206343835 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095876+0000", 2023-10-09T04:09:03.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9043e-05 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.44666.0:205763 6.14 e158/124 6:2c565c3b:::benchmark_data_smithi163_287034_object97500:head v 158'9972, mlcod=158'9970)", 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.889461+0000", 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.451566207999999, 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20639120799999999, 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:03.467 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889461+0000", 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889461+0000", 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889461+0000", 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889465+0000", 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2480000000000001e-06 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889468+0000", 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3000000000000002e-06 2023-10-09T04:09:03.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889470+0000", 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1500000000000002e-06 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889505+0000", 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4586999999999999e-05 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.889520+0000", 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5617999999999999e-05 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095817+0000", 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20629660799999999 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095853+0000", 2023-10-09T04:09:03.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5882e-05 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.44666.0:205754 6.d e158/102 ondisk, result = 0)", 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.891461+0000", 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.449567093999999, 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.204401055, 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:09:03.472 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891461+0000", 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891461+0000", 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.473 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891460+0000", 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891462+0000", 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.889e-06 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891466+0000", 2023-10-09T04:09:03.474 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.873e-06 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891468+0000", 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2400000000000002e-06 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891533+0000", 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4298e-05 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.475 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891537+0000", 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.929e-06 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095862+0000", 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.204325017 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.476 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.44666.0:205752 6.a e158/102 ondisk, result = 0)", 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.893676+0000", 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.44735154, 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20231986699999999, 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.893676+0000", 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.477 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.893676+0000", 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.893676+0000", 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.478 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.893680+0000", 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2660000000000003e-06 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095828+0000", 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.202148514 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095835+0000", 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7780000000000002e-06 2023-10-09T04:09:03.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095886+0000", 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0796e-05 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095894+0000", 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2539999999999992e-06 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:03.480 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095996+0000", 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010185500000000001 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:03.481 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:09:03.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:03.516 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:03 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 743/6384 objects degraded (11.638%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:03 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1864908982' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:03 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:03.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:09:03.986 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T04:09:03.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T04:09:04.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:09:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: 6.1 scrub starts 2023-10-09T04:09:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:09:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:09:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: pgmap v424: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 98 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 743/4755 objects degraded (15.626%); 743/4755 objects misplaced (15.626%); 216 KiB/s, 21 objects/s recovering 2023-10-09T04:09:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: 6.6 deep-scrub starts 2023-10-09T04:09:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[221067]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: 6.1 scrub starts 2023-10-09T04:09:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: pgmap v424: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 98 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 743/4755 objects degraded (15.626%); 743/4755 objects misplaced (15.626%); 216 KiB/s, 21 objects/s recovering 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: 6.6 deep-scrub starts 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:04 smithi163 ceph-mon[213841]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.414 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: 6.1 scrub starts 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: pgmap v424: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 98 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 743/4755 objects degraded (15.626%); 743/4755 objects misplaced (15.626%); 216 KiB/s, 21 objects/s recovering 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: 6.6 deep-scrub starts 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: failed to encode map e164 with expected crc 2023-10-09T04:09:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:04 smithi181 ceph-mon[196979]: failed to encode map e164 with expected crc 2023-10-09T04:09:05.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:05.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:05.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:05 smithi163 ceph-mon[221067]: failed to encode map e164 with expected crc 2023-10-09T04:09:05.501 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:05.501 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:05 smithi163 ceph-mon[213841]: failed to encode map e164 with expected crc 2023-10-09T04:09:05.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:05.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:05.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:05 smithi181 ceph-mon[196979]: failed to encode map e164 with expected crc 2023-10-09T04:09:05.524 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:05.553 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:05.598 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:05.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:05.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:05.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:05.750 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:05.773 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:09:05.773 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:09:05.773 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:09:05.773 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:09:05.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:05.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:05.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:05.826 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:05.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:05.937 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:06.058 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:06.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:06.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:06.191 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:06.264 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:09:06.266 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:09:06.294 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:06 smithi163 ceph-mon[221067]: pgmap v425: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 52 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 743/2565 objects degraded (28.967%); 743/2565 objects misplaced (28.967%); 198 KiB/s, 64 objects/s recovering 2023-10-09T04:09:06.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:06 smithi163 ceph-mon[213841]: pgmap v425: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 52 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 743/2565 objects degraded (28.967%); 743/2565 objects misplaced (28.967%); 198 KiB/s, 64 objects/s recovering 2023-10-09T04:09:06.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:06.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:06.448 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:09:06.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:06.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:06 smithi181 ceph-mon[196979]: pgmap v425: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 52 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 743/2565 objects degraded (28.967%); 743/2565 objects misplaced (28.967%); 198 KiB/s, 64 objects/s recovering 2023-10-09T04:09:06.528 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:06.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:06.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:06.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:06.712 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:06.801 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:07 smithi181 ceph-mon[196979]: from='client.54787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 743/2565 objects degraded (28.967%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:07.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:09:07.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:07 smithi163 ceph-mon[221067]: from='client.54787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:07.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 743/2565 objects degraded (28.967%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:07.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:07 smithi163 ceph-mon[213841]: from='client.54787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:07.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 743/2565 objects degraded (28.967%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:08.240 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:09:08.240 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:09:08.240 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:09:08.240 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:09:08.241 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:09:08.426 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:08 smithi181 ceph-mon[196979]: pgmap v426: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 173 KiB/s, 56 objects/s recovering 2023-10-09T04:09:08.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:08 smithi163 ceph-mon[221067]: pgmap v426: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 173 KiB/s, 56 objects/s recovering 2023-10-09T04:09:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:08 smithi163 ceph-mon[213841]: pgmap v426: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 173 KiB/s, 56 objects/s recovering 2023-10-09T04:09:08.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:09:09.235 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:09.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:09.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:09.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:09.826 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:09.869 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:09.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:09.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:09.999 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:10.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:10.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:10.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:10.233 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:10.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:10.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:10.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:10.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:10.418 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:10.602 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118953 6.c 6:3024d817:::benchmark_data_smithi163_287034_object10690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 146.064475853, 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96349797500000001, 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.603 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118953 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.604 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535426+0000", 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.605 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535432+0000", 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0039999999999996e-06 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535441+0000", 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0080000000000008e-06 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.606 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535448+0000", 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.737e-06 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535513+0000", 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5147000000000005e-05 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.607 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535523+0000", 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0067e-05 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.481396+0000", 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94587314700000003 2023-10-09T04:09:10.608 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.483600+0000", 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022032530000000001 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.483670+0000", 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0501999999999997e-05 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.497950+0000", 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014279490000000001 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498056+0000", 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106544 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498844+0000", 2023-10-09T04:09:10.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00078815099999999996 2023-10-09T04:09:10.611 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498889+0000", 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4614999999999999e-05 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.613 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498925+0000", 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6409000000000003e-05 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118965 6.c 6:37b70bb5:::benchmark_data_smithi163_287034_object10702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 146.01426726899999, 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91336464399999995, 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.614 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118965 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585634+0000", 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585641+0000", 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0720000000000002e-06 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585655+0000", 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3332e-05 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585661+0000", 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4740000000000002e-06 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585709+0000", 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7892999999999997e-05 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:09:10.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585715+0000", 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4180000000000002e-06 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486249+0000", 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90053330399999998 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486335+0000", 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6322000000000005e-05 2023-10-09T04:09:10.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486411+0000", 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5829999999999997e-05 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498155+0000", 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011743709999999999 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498809+0000", 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00065421399999999997 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498938+0000", 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012885899999999999 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498971+0000", 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3025999999999999e-05 2023-10-09T04:09:10.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499001+0000", 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9971000000000002e-05 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.622 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118966 6.c 6:3491ce90:::benchmark_data_smithi163_287034_object10703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 146.01422855199999, 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91791875999999994, 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118966 2023-10-09T04:09:10.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585674+0000", 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585677+0000", 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6410000000000002e-06 2023-10-09T04:09:10.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585683+0000", 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1410000000000002e-06 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585687+0000", 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9369999999999997e-06 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585735+0000", 2023-10-09T04:09:10.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7969000000000001e-05 2023-10-09T04:09:10.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:09:10.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585741+0000", 2023-10-09T04:09:10.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7339999999999996e-06 2023-10-09T04:09:10.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489228+0000", 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90348698199999999 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489334+0000", 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106357 2023-10-09T04:09:10.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489437+0000", 2023-10-09T04:09:10.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103119 2023-10-09T04:09:10.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498838+0000", 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00940123 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499006+0000", 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016752499999999999 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503505+0000", 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044992570000000004 2023-10-09T04:09:10.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503553+0000", 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8050000000000002e-05 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.634 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.635 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503594+0000", 2023-10-09T04:09:10.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0227e-05 2023-10-09T04:09:10.635 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.635 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.635 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.635 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118982 6.c 6:3e1a0c35:::benchmark_data_smithi163_287034_object10719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.994388163, 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90241945000000001, 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.636 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118982 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:09:10.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605514+0000", 2023-10-09T04:09:10.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605521+0000", 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8539999999999999e-06 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605532+0000", 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.114e-05 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.641 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605537+0000", 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.118e-06 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605601+0000", 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4351999999999996e-05 2023-10-09T04:09:10.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605612+0000", 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0536e-05 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492104+0000", 2023-10-09T04:09:10.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88649252599999995 2023-10-09T04:09:10.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492184+0000", 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9084000000000004e-05 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492265+0000", 2023-10-09T04:09:10.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1630000000000003e-05 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499011+0000", 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067454790000000004 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503639+0000", 2023-10-09T04:09:10.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046281070000000002 2023-10-09T04:09:10.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507860+0000", 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042216689999999999 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507899+0000", 2023-10-09T04:09:10.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.841e-05 2023-10-09T04:09:10.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507935+0000", 2023-10-09T04:09:10.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5747000000000002e-05 2023-10-09T04:09:10.649 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:119013 6.c 6:3a6a5865:::benchmark_data_smithi163_287034_object10750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:09:10.650 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.851340483, 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75951261400000003, 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.651 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 119013 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748561+0000", 2023-10-09T04:09:10.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748576+0000", 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4914e-05 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748593+0000", 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6529999999999999e-05 2023-10-09T04:09:10.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748600+0000", 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6789999999999999e-06 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748669+0000", 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9541000000000004e-05 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748682+0000", 2023-10-09T04:09:10.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3251000000000001e-05 2023-10-09T04:09:10.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494591+0000", 2023-10-09T04:09:10.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.74590919700000002 2023-10-09T04:09:10.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494688+0000", 2023-10-09T04:09:10.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6761000000000004e-05 2023-10-09T04:09:10.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494788+0000", 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9431000000000004e-05 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499015+0000", 2023-10-09T04:09:10.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042273709999999997 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503667+0000", 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046520449999999996 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507982+0000", 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043144380000000003 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.508033+0000", 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1585000000000002e-05 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.508075+0000", 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2299999999999998e-05 2023-10-09T04:09:10.663 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.664 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.664 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.664 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:132609 6.c 6:35106937:::benchmark_data_smithi163_287034_object24346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T04:09:10.664 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:02.933476+0000", 2023-10-09T04:09:10.665 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 127.66642735799999, 2023-10-09T04:09:10.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11452306700000001, 2023-10-09T04:09:10.665 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.665 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.666 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.666 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 132609 2023-10-09T04:09:10.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.666 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933476+0000", 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933476+0000", 2023-10-09T04:09:10.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933475+0000", 2023-10-09T04:09:10.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933491+0000", 2023-10-09T04:09:10.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6252e-05 2023-10-09T04:09:10.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933507+0000", 2023-10-09T04:09:10.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6286999999999998e-05 2023-10-09T04:09:10.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933513+0000", 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2670000000000003e-06 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933574+0000", 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0189000000000003e-05 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933700+0000", 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000126358 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.933808+0000", 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107763 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.936878+0000", 2023-10-09T04:09:10.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030704730000000002 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:02.937533+0000", 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00065436300000000004 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:03.047939+0000", 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11040628700000001 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:03.047974+0000", 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4808999999999998e-05 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:03.047999+0000", 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5400000000000001e-05 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.674 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:152285 6.c 6:385adff8:::benchmark_data_smithi163_287034_object44022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:28.306131+0000", 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 102.293772464, 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12219577, 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.675 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 152285 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306131+0000", 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306131+0000", 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306129+0000", 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306138+0000", 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7940000000000008e-06 2023-10-09T04:09:10.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306152+0000", 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4076e-05 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306159+0000", 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9460000000000001e-06 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306216+0000", 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7255e-05 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306346+0000", 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012931999999999999 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306454+0000", 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108323 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.322099+0000", 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015645429999999998 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.322243+0000", 2023-10-09T04:09:10.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014337700000000001 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428246+0000", 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.106003352 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428288+0000", 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1718999999999998e-05 2023-10-09T04:09:10.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428327+0000", 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8897999999999999e-05 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:152287 6.c 6:374e51ad:::benchmark_data_smithi163_287034_object44024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:09:10.682 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:28.306178+0000", 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 102.293725018, 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.122274232, 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 152287 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306178+0000", 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306178+0000", 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306178+0000", 2023-10-09T04:09:10.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306181+0000", 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8600000000000001e-06 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306187+0000", 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4160000000000001e-06 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.306191+0000", 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9140000000000001e-06 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.314282+0000", 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080910259999999994 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.314407+0000", 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012499399999999999 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.314526+0000", 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118719 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.322251+0000", 2023-10-09T04:09:10.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077255479999999996 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.328142+0000", 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0058904229999999997 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428367+0000", 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10022486 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428414+0000", 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6910000000000003e-05 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428453+0000", 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8939000000000001e-05 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.689 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:152293 6.c 6:3a7f7788:::benchmark_data_smithi163_287034_object44030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:28.318102+0000", 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 102.28180087, 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11058264199999999, 2023-10-09T04:09:10.690 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 152293 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318102+0000", 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318102+0000", 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318101+0000", 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318107+0000", 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1929999999999998e-06 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318131+0000", 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3312999999999999e-05 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318138+0000", 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2239999999999998e-06 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318621+0000", 2023-10-09T04:09:10.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00048274599999999999 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318710+0000", 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9588999999999999e-05 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.318789+0000", 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8687000000000001e-05 2023-10-09T04:09:10.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.322303+0000", 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035137760000000001 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.328275+0000", 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005972687 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428591+0000", 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.100315477 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428643+0000", 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2623000000000001e-05 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:28.428685+0000", 2023-10-09T04:09:10.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1700999999999999e-05 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:186822 6.c 6:363d9db9:::benchmark_data_smithi163_287034_object78559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:18.841517+0000", 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 51.758386039999998, 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14537045900000001, 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.698 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 186822 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841517+0000", 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841517+0000", 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841516+0000", 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841523+0000", 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7900000000000002e-06 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841540+0000", 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7207e-05 2023-10-09T04:09:10.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841546+0000", 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6420000000000001e-06 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.845953+0000", 2023-10-09T04:09:10.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044065739999999999 2023-10-09T04:09:10.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.846093+0000", 2023-10-09T04:09:10.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000140035 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.846259+0000", 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000165754 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851091+0000", 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048319640000000002 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851120+0000", 2023-10-09T04:09:10.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9507999999999999e-05 2023-10-09T04:09:10.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.986799+0000", 2023-10-09T04:09:10.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13567926299999999 2023-10-09T04:09:10.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.986847+0000", 2023-10-09T04:09:10.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7463999999999998e-05 2023-10-09T04:09:10.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.986888+0000", 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0781000000000001e-05 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.709 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:186830 6.c 6:3c7a66bf:::benchmark_data_smithi163_287034_object78567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:09:10.709 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:18.851800+0000", 2023-10-09T04:09:10.709 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 51.748103147999998, 2023-10-09T04:09:10.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.143798601, 2023-10-09T04:09:10.709 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.710 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.710 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.710 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.710 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.710 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 186830 2023-10-09T04:09:10.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.711 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851800+0000", 2023-10-09T04:09:10.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851800+0000", 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851799+0000", 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851806+0000", 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6780000000000003e-06 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851821+0000", 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5142e-05 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851827+0000", 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7479999999999996e-06 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.855956+0000", 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041287590000000001 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.856117+0000", 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016046300000000001 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.856266+0000", 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014936300000000001 2023-10-09T04:09:10.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.859956+0000", 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036902979999999998 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.860463+0000", 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050628999999999995 2023-10-09T04:09:10.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.995496+0000", 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13503367099999999 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.995549+0000", 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3134999999999999e-05 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.995599+0000", 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9357000000000002e-05 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:198770 6.c 6:3b926135:::benchmark_data_smithi163_287034_object90507:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:35.953524+0000", 2023-10-09T04:09:10.718 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.646379183999997, 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.105275047, 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 198770 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953524+0000", 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953524+0000", 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953523+0000", 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953529+0000", 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4910000000000002e-06 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953545+0000", 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5948e-05 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953551+0000", 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4489999999999996e-06 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953605+0000", 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3684000000000003e-05 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953746+0000", 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014094300000000001 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.953878+0000", 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013215699999999999 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.965774+0000", 2023-10-09T04:09:10.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011895786 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.966157+0000", 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00038288500000000002 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.058731+0000", 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.092574133000000003 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.058768+0000", 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7035e-05 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.058799+0000", 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1089e-05 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:198771 6.c 6:3336f068:::benchmark_data_smithi163_287034_object90508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:35.956792+0000", 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.643110978999999, 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10829069700000001, 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 198771 2023-10-09T04:09:10.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.956792+0000", 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.956792+0000", 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.956791+0000", 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.956798+0000", 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8260000000000004e-06 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.956813+0000", 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5452999999999999e-05 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.956820+0000", 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9589999999999998e-06 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957804+0000", 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00098342099999999995 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957923+0000", 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118746 2023-10-09T04:09:10.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.958061+0000", 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013884800000000001 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.965778+0000", 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077165430000000002 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.966192+0000", 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000414208 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.065015+0000", 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.098822569999999998 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.065053+0000", 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8386999999999997e-05 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.065083+0000", 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9912000000000001e-05 2023-10-09T04:09:10.734 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.735 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.735 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.735 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:198773 6.c 6:3e64ba5c:::benchmark_data_smithi163_287034_object90510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T04:09:10.735 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:35.957599+0000", 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 34.642304258000003, 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.107587004, 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.736 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 198773 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957599+0000", 2023-10-09T04:09:10.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957599+0000", 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957598+0000", 2023-10-09T04:09:10.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957603+0000", 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9259999999999999e-06 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957618+0000", 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5125e-05 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.957624+0000", 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7679999999999997e-06 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.962321+0000", 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046965899999999996 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.962473+0000", 2023-10-09T04:09:10.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015256300000000001 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.962642+0000", 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000168288 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.965862+0000", 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032203319999999998 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:35.966244+0000", 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00038209399999999998 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.065123+0000", 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.098879238999999994 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.065159+0000", 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6242999999999998e-05 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:36.065186+0000", 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6562000000000001e-05 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:201176 6.c 6:3908047e:::benchmark_data_smithi163_287034_object92913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:39.354240+0000", 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 31.245663499999999, 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14421740799999999, 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.744 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 201176 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354240+0000", 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354240+0000", 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354239+0000", 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354244+0000", 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1449999999999999e-06 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354263+0000", 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9398e-05 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354271+0000", 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5669999999999997e-06 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354326+0000", 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4938000000000002e-05 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354459+0000", 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013323600000000001 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354600+0000", 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014062600000000001 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.357949+0000", 2023-10-09T04:09:10.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033488530000000002 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.358248+0000", 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029936399999999999 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.498386+0000", 2023-10-09T04:09:10.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14013837900000001 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.498423+0000", 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6891999999999998e-05 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.498457+0000", 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3994e-05 2023-10-09T04:09:10.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:201192 6.c 6:3722fa0d:::benchmark_data_smithi163_287034_object92929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:39.401884+0000", 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 31.198019638000002, 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10562745799999999, 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 201192 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401884+0000", 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401884+0000", 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401882+0000", 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401888+0000", 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7919999999999998e-06 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401901+0000", 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2566e-05 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401906+0000", 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.401963+0000", 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6873000000000002e-05 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.402105+0000", 2023-10-09T04:09:10.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014165400000000001 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.402240+0000", 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013579999999999999 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.406746+0000", 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004505144 2023-10-09T04:09:10.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.406816+0000", 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0178999999999993e-05 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.507429+0000", 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.100613493 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.507468+0000", 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8988e-05 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.507511+0000", 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2910000000000001e-05 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.759 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:215151 6.c 6:394855ec:::benchmark_data_smithi163_287034_object106888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:00.480488+0000", 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 10.11941566, 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.108162893, 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.760 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 215151 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480488+0000", 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480488+0000", 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480486+0000", 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480494+0000", 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0730000000000005e-06 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480512+0000", 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8274000000000002e-05 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480520+0000", 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7419999999999997e-06 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480579+0000", 2023-10-09T04:09:10.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8909999999999997e-05 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480712+0000", 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000133039 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480826+0000", 2023-10-09T04:09:10.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113758 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.489574+0000", 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0087484610000000008 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.489723+0000", 2023-10-09T04:09:10.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014861799999999999 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.588563+0000", 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.098840237999999997 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.588616+0000", 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2562999999999998e-05 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.588651+0000", 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4838000000000001e-05 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:215152 6.c 6:306fe5e8:::benchmark_data_smithi163_287034_object106889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:00.480536+0000", 2023-10-09T04:09:10.768 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 10.119366848, 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.108221752, 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 215152 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480536+0000", 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480536+0000", 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480536+0000", 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480539+0000", 2023-10-09T04:09:10.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.393e-06 2023-10-09T04:09:10.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480557+0000", 2023-10-09T04:09:10.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7759999999999999e-05 2023-10-09T04:09:10.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.480562+0000", 2023-10-09T04:09:10.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3689999999999998e-06 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.485170+0000", 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046072409999999998 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.485277+0000", 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107651 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.485388+0000", 2023-10-09T04:09:10.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011045299999999999 2023-10-09T04:09:10.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.776 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.776 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.489756+0000", 2023-10-09T04:09:10.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043679260000000003 2023-10-09T04:09:10.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.490029+0000", 2023-10-09T04:09:10.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00027334000000000001 2023-10-09T04:09:10.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.588699+0000", 2023-10-09T04:09:10.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.098669571999999997 2023-10-09T04:09:10.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.779 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.779 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.588733+0000", 2023-10-09T04:09:10.779 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4703000000000003e-05 2023-10-09T04:09:10.779 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:00.588758+0000", 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4859999999999999e-05 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:216453 6.c 6:331e527e:::benchmark_data_smithi163_287034_object108190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:02.434388+0000", 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 8.1655151089999993, 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13862418500000001, 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.781 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 216453 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434388+0000", 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434388+0000", 2023-10-09T04:09:10.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.784 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.784 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434386+0000", 2023-10-09T04:09:10.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434395+0000", 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3599999999999996e-06 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.786 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434408+0000", 2023-10-09T04:09:10.786 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3511e-05 2023-10-09T04:09:10.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434415+0000", 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1920000000000002e-06 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434470+0000", 2023-10-09T04:09:10.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.537e-05 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434594+0000", 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012374599999999999 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434704+0000", 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011008799999999999 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.789 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.438729+0000", 2023-10-09T04:09:10.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004025392 2023-10-09T04:09:10.790 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.790 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.790 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.447073+0000", 2023-10-09T04:09:10.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0083440919999999991 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.572925+0000", 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12585126299999999 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.572969+0000", 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4978999999999997e-05 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573012+0000", 2023-10-09T04:09:10.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2885e-05 2023-10-09T04:09:10.793 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.793 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.793 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.794 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:216459 6.c 6:30c38a9c:::benchmark_data_smithi163_287034_object108196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T04:09:10.794 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:02.440534+0000", 2023-10-09T04:09:10.794 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 8.1593687999999993, 2023-10-09T04:09:10.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.132590931, 2023-10-09T04:09:10.794 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:09:10.794 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 216459 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:09:10.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440534+0000", 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:09:10.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440534+0000", 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440533+0000", 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:09:10.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440542+0000", 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9080000000000001e-06 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440555+0000", 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2544999999999999e-05 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440561+0000", 2023-10-09T04:09:10.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8140000000000003e-06 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440609+0000", 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8059000000000002e-05 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440732+0000", 2023-10-09T04:09:10.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012246199999999999 2023-10-09T04:09:10.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:09:10.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440850+0000", 2023-10-09T04:09:10.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011856799999999999 2023-10-09T04:09:10.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.447574+0000", 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067238269999999999 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.451382+0000", 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038073410000000001 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573053+0000", 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.121671258 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573092+0000", 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8645e-05 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:09:10.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573125+0000", 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3896999999999997e-05 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:09:10.804 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:09:10.805 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:10 smithi163 ceph-mon[221067]: pgmap v427: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 173 KiB/s, 56 objects/s recovering 2023-10-09T04:09:10.805 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:10 smithi181 ceph-mon[196979]: pgmap v427: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 173 KiB/s, 56 objects/s recovering 2023-10-09T04:09:10.805 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:10 smithi163 ceph-mon[213841]: pgmap v427: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 173 KiB/s, 56 objects/s recovering 2023-10-09T04:09:11.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:09:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:09:10] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2023-10-09T04:09:11.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:09:11.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:11.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:11.427 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":425,"stamp":"2023-10-09T04:09:05.401465+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":54990776,"num_objects":855,"num_object_clones":0,"num_object_copies":2565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":743,"num_objects_misplaced":743,"num_objects_unfound":0,"num_objects_dirty":855,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":158562,"num_write_kb":3897742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4309,"num_bytes_recovered":97257056,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":257371,"ondisk_log_size":257371,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2613248,"kb_used_data":779904,"kb_used_omap":126,"kb_used_meta":1833089,"kb_avail":747347968,"statfs":{"total":767960285184,"available":765284319232,"internally_reserved":0,"allocated":798621696,"data_stored":794184583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1877083771},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,0,4,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-299958272,"num_objects":-4577,"num_object_clones":0,"num_object_copies":-13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-54,"num_objects_unfound":0,"num_objects_dirty":-4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4672,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":566,"num_bytes_recovered":1769472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.719461"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":141,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689995+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:08:56.689995+0000","last_peered":"2023-10-09T04:08:56.689995+0000","last_clean":"2023-10-09T04:08:56.689995+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:08:56.689995+0000","last_undegraded":"2023-10-09T04:08:56.689995+0000","last_fullsized":"2023-10-09T04:08:56.689995+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":97,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.638844+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:08:53.638844+0000","last_peered":"2023-10-09T04:08:53.638844+0000","last_clean":"2023-10-09T04:08:53.638844+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:08:53.638844+0000","last_undegraded":"2023-10-09T04:08:53.638844+0000","last_fullsized":"2023-10-09T04:08:53.638844+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.636337+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:08:53.636337+0000","last_peered":"2023-10-09T04:08:53.636337+0000","last_clean":"2023-10-09T04:08:53.636337+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:08:53.636337+0000","last_undegraded":"2023-10-09T04:08:53.636337+0000","last_fullsized":"2023-10-09T04:08:53.636337+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.632536+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:08:53.632536+0000","last_peered":"2023-10-09T04:08:53.632536+0000","last_clean":"2023-10-09T04:08:53.632536+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:08:53.632536+0000","last_undegraded":"2023-10-09T04:08:53.632536+0000","last_fullsized":"2023-10-09T04:08:53.632536+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":115,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.633715+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:08:53.633715+0000","last_peered":"2023-10-09T04:08:53.633715+0000","last_clean":"2023-10-09T04:08:53.633715+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:08:53.633715+0000","last_undegraded":"2023-10-09T04:08:53.633715+0000","last_fullsized":"2023-10-09T04:08:53.633715+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.651139+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:08:53.651139+0000","last_peered":"2023-10-09T04:08:53.651139+0000","last_clean":"2023-10-09T04:08:53.651139+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:08:53.651139+0000","last_undegraded":"2023-10-09T04:08:53.651139+0000","last_fullsized":"2023-10-09T04:08:53.651139+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"163'10256","reported_seq":17301,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.434843+0000","last_change":"2023-10-09T04:08:47.760125+0000","last_active":"2023-10-09T04:09:00.434843+0000","last_peered":"2023-10-09T04:09:00.434843+0000","last_clean":"2023-10-09T04:09:00.434843+0000","last_became_active":"2023-10-09T04:08:47.559033+0000","last_became_peered":"2023-10-09T04:08:47.559033+0000","last_unstale":"2023-10-09T04:09:00.434843+0000","last_undegraded":"2023-10-09T04:09:00.434843+0000","last_fullsized":"2023-10-09T04:09:00.434843+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":5,"last_scrub":"129'7222","last_scrub_stamp":"2023-10-09T04:06:45.048975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:45.048975+0000","objects_scrubbed":3069,"log_size":10256,"log_dups_size":0,"ondisk_log_size":10256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:27.554921+0000","scrub_duration":0.31357458100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293760,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3034,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3408,"num_bytes_recovered":72220672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.632919+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:08:53.632919+0000","last_peered":"2023-10-09T04:08:53.632919+0000","last_clean":"2023-10-09T04:08:53.632919+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:08:53.632919+0000","last_undegraded":"2023-10-09T04:08:53.632919+0000","last_fullsized":"2023-10-09T04:08:53.632919+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.645062+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:08:53.645062+0000","last_peered":"2023-10-09T04:08:53.645062+0000","last_clean":"2023-10-09T04:08:53.645062+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:08:53.645062+0000","last_undegraded":"2023-10-09T04:08:53.645062+0000","last_fullsized":"2023-10-09T04:08:53.645062+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"163'10356","reported_seq":13215,"reported_epoch":164,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:09:04.826231+0000","last_change":"2023-10-09T04:08:46.064687+0000","last_active":"2023-10-09T04:09:04.826231+0000","last_peered":"2023-10-09T04:09:04.826231+0000","last_clean":"2023-10-09T04:08:08.889730+0000","last_became_active":"2023-10-09T04:08:12.519179+0000","last_became_peered":"2023-10-09T04:08:12.519179+0000","last_unstale":"2023-10-09T04:09:04.826231+0000","last_undegraded":"2023-10-09T04:08:12.334260+0000","last_fullsized":"2023-10-09T04:08:12.225637+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"138'8328","last_scrub_stamp":"2023-10-09T04:07:27.518301+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:07:27.518301+0000","objects_scrubbed":2035,"log_size":10356,"log_dups_size":0,"ondisk_log_size":10356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:16.747824+0000","scrub_duration":0.61337365300000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":743,"num_objects_misplaced":743,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2035,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":894,"num_bytes_recovered":23265280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6","objects":-743}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.648121+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:08:53.648121+0000","last_peered":"2023-10-09T04:08:53.648121+0000","last_clean":"2023-10-09T04:08:53.648121+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:08:53.648121+0000","last_undegraded":"2023-10-09T04:08:53.648121+0000","last_fullsized":"2023-10-09T04:08:53.648121+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":59,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.658446+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:08:53.658446+0000","last_peered":"2023-10-09T04:08:53.658446+0000","last_clean":"2023-10-09T04:08:53.658446+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:08:53.658446+0000","last_undegraded":"2023-10-09T04:08:53.658446+0000","last_fullsized":"2023-10-09T04:08:53.658446+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.10","version":"163'10221","reported_seq":20665,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:01.977005+0000","last_change":"2023-10-09T04:08:43.815420+0000","last_active":"2023-10-09T04:09:01.977005+0000","last_peered":"2023-10-09T04:09:01.977005+0000","last_clean":"2023-10-09T04:09:01.977005+0000","last_became_active":"2023-10-09T04:08:02.626026+0000","last_became_peered":"2023-10-09T04:08:02.626026+0000","last_unstale":"2023-10-09T04:09:01.977005+0000","last_undegraded":"2023-10-09T04:09:01.977005+0000","last_fullsized":"2023-10-09T04:09:01.977005+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":5,"last_scrub":"158'9850","last_scrub_stamp":"2023-10-09T04:08:43.815181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:08:43.815181+0000","objects_scrubbed":387,"log_size":10221,"log_dups_size":0,"ondisk_log_size":10221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:29:05.133993+0000","scrub_duration":0.24388148000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":851968,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":374,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657851+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:08:53.657851+0000","last_peered":"2023-10-09T04:08:53.657851+0000","last_clean":"2023-10-09T04:08:53.657851+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:08:53.657851+0000","last_undegraded":"2023-10-09T04:08:53.657851+0000","last_fullsized":"2023-10-09T04:08:53.657851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.636203+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:08:53.636203+0000","last_peered":"2023-10-09T04:08:53.636203+0000","last_clean":"2023-10-09T04:08:53.636203+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:08:53.636203+0000","last_undegraded":"2023-10-09T04:08:53.636203+0000","last_fullsized":"2023-10-09T04:08:53.636203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"163'10175","reported_seq":23737,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.008988+0000","last_change":"2023-10-09T04:08:44.863845+0000","last_active":"2023-10-09T04:09:02.008988+0000","last_peered":"2023-10-09T04:09:02.008988+0000","last_clean":"2023-10-09T04:09:02.008988+0000","last_became_active":"2023-10-09T04:08:32.498505+0000","last_became_peered":"2023-10-09T04:08:32.498505+0000","last_unstale":"2023-10-09T04:09:02.008988+0000","last_undegraded":"2023-10-09T04:09:02.008988+0000","last_fullsized":"2023-10-09T04:09:02.008988+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"158'9824","last_scrub_stamp":"2023-10-09T04:08:44.863686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:08:44.863686+0000","objects_scrubbed":362,"log_size":10175,"log_dups_size":0,"ondisk_log_size":10175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:19.148312+0000","scrub_duration":0.25116014199999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":524288,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":354,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":128,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689325+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:08:56.689325+0000","last_peered":"2023-10-09T04:08:56.689325+0000","last_clean":"2023-10-09T04:08:56.689325+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:08:56.689325+0000","last_undegraded":"2023-10-09T04:08:56.689325+0000","last_fullsized":"2023-10-09T04:08:56.689325+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.688880+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:08:56.688880+0000","last_peered":"2023-10-09T04:08:56.688880+0000","last_clean":"2023-10-09T04:08:56.688880+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:08:56.688880+0000","last_undegraded":"2023-10-09T04:08:56.688880+0000","last_fullsized":"2023-10-09T04:08:56.688880+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"163'10462","reported_seq":10583,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.583146+0000","last_change":"2023-10-09T04:09:01.448964+0000","last_active":"2023-10-09T04:09:02.583146+0000","last_peered":"2023-10-09T04:09:02.583146+0000","last_clean":"2023-10-09T04:09:02.583146+0000","last_became_active":"2023-10-09T04:06:45.471218+0000","last_became_peered":"2023-10-09T04:06:45.471218+0000","last_unstale":"2023-10-09T04:09:02.583146+0000","last_undegraded":"2023-10-09T04:09:02.583146+0000","last_fullsized":"2023-10-09T04:09:02.583146+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10462,"log_dups_size":0,"ondisk_log_size":10462,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6936,"num_write_kb":218912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657788+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:08:53.657788+0000","last_peered":"2023-10-09T04:08:53.657788+0000","last_clean":"2023-10-09T04:08:53.657788+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:08:53.657788+0000","last_undegraded":"2023-10-09T04:08:53.657788+0000","last_fullsized":"2023-10-09T04:08:53.657788+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":97,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.638637+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:08:53.638637+0000","last_peered":"2023-10-09T04:08:53.638637+0000","last_clean":"2023-10-09T04:08:53.638637+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:08:53.638637+0000","last_undegraded":"2023-10-09T04:08:53.638637+0000","last_fullsized":"2023-10-09T04:08:53.638637+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.17","version":"163'10110","reported_seq":10329,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.347071+0000","last_change":"2023-10-09T04:06:38.312336+0000","last_active":"2023-10-09T04:09:00.347071+0000","last_peered":"2023-10-09T04:09:00.347071+0000","last_clean":"2023-10-09T04:09:00.347071+0000","last_became_active":"2023-10-09T04:06:35.331160+0000","last_became_peered":"2023-10-09T04:06:35.331160+0000","last_unstale":"2023-10-09T04:09:00.347071+0000","last_undegraded":"2023-10-09T04:09:00.347071+0000","last_fullsized":"2023-10-09T04:09:00.347071+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"126'6981","last_scrub_stamp":"2023-10-09T04:06:38.312243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:38.312243+0000","objects_scrubbed":3182,"log_size":10110,"log_dups_size":0,"ondisk_log_size":10110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:27.105520+0000","scrub_duration":0.56485964200000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3342336,"num_objects":51,"num_object_clones":0,"num_object_copies":153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3131,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":117,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.710266+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:08:56.710266+0000","last_peered":"2023-10-09T04:08:56.710266+0000","last_clean":"2023-10-09T04:08:56.710266+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:08:56.710266+0000","last_undegraded":"2023-10-09T04:08:56.710266+0000","last_fullsized":"2023-10-09T04:08:56.710266+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.648332+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:08:53.648332+0000","last_peered":"2023-10-09T04:08:53.648332+0000","last_clean":"2023-10-09T04:08:53.648332+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:08:53.648332+0000","last_undegraded":"2023-10-09T04:08:53.648332+0000","last_fullsized":"2023-10-09T04:08:53.648332+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"163'10326","reported_seq":10480,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.607369+0000","last_change":"2023-10-09T04:06:41.794687+0000","last_active":"2023-10-09T04:09:02.607369+0000","last_peered":"2023-10-09T04:09:02.607369+0000","last_clean":"2023-10-09T04:09:02.607369+0000","last_became_active":"2023-10-09T04:06:35.296891+0000","last_became_peered":"2023-10-09T04:06:35.296891+0000","last_unstale":"2023-10-09T04:09:02.607369+0000","last_undegraded":"2023-10-09T04:09:02.607369+0000","last_fullsized":"2023-10-09T04:09:02.607369+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"126'7207","last_scrub_stamp":"2023-10-09T04:06:41.794514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:41.794514+0000","objects_scrubbed":3122,"log_size":10326,"log_dups_size":0,"ondisk_log_size":10326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:31.793050+0000","scrub_duration":0.64420732400000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3122,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.658524+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:08:53.658524+0000","last_peered":"2023-10-09T04:08:53.658524+0000","last_clean":"2023-10-09T04:08:53.658524+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:08:53.658524+0000","last_undegraded":"2023-10-09T04:08:53.658524+0000","last_fullsized":"2023-10-09T04:08:53.658524+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.632465+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:08:53.632465+0000","last_peered":"2023-10-09T04:08:53.632465+0000","last_clean":"2023-10-09T04:08:53.632465+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:08:53.632465+0000","last_undegraded":"2023-10-09T04:08:53.632465+0000","last_fullsized":"2023-10-09T04:08:53.632465+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"163'10092","reported_seq":10189,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.439476+0000","last_change":"2023-10-09T04:08:58.051656+0000","last_active":"2023-10-09T04:09:00.439476+0000","last_peered":"2023-10-09T04:09:00.439476+0000","last_clean":"2023-10-09T04:09:00.439476+0000","last_became_active":"2023-10-09T04:06:35.304634+0000","last_became_peered":"2023-10-09T04:06:35.304634+0000","last_unstale":"2023-10-09T04:09:00.439476+0000","last_undegraded":"2023-10-09T04:09:00.439476+0000","last_fullsized":"2023-10-09T04:09:00.439476+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4620288,"num_objects":71,"num_object_clones":0,"num_object_copies":213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6649,"num_write_kb":215008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":52,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.633600+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:08:53.633600+0000","last_peered":"2023-10-09T04:08:53.633600+0000","last_clean":"2023-10-09T04:08:53.633600+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:08:53.633600+0000","last_undegraded":"2023-10-09T04:08:53.633600+0000","last_fullsized":"2023-10-09T04:08:53.633600+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.651486+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:08:53.651486+0000","last_peered":"2023-10-09T04:08:53.651486+0000","last_clean":"2023-10-09T04:08:53.651486+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:08:53.651486+0000","last_undegraded":"2023-10-09T04:08:53.651486+0000","last_fullsized":"2023-10-09T04:08:53.651486+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"163'13625","reported_seq":13694,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.427765+0000","last_change":"2023-10-09T04:04:51.281106+0000","last_active":"2023-10-09T04:09:00.427765+0000","last_peered":"2023-10-09T04:09:00.427765+0000","last_clean":"2023-10-09T04:09:00.427765+0000","last_became_active":"2023-10-09T04:04:51.280862+0000","last_became_peered":"2023-10-09T04:04:51.280862+0000","last_unstale":"2023-10-09T04:09:00.427765+0000","last_undegraded":"2023-10-09T04:09:00.427765+0000","last_fullsized":"2023-10-09T04:09:00.427765+0000","mapping_epoch":102,"log_start":"113'3600","ondisk_log_start":"113'3600","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:47.717190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5046272,"num_objects":77,"num_object_clones":0,"num_object_copies":231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13625,"num_write_kb":438464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657967+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:08:53.657967+0000","last_peered":"2023-10-09T04:08:53.657967+0000","last_clean":"2023-10-09T04:08:53.657967+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:08:53.657967+0000","last_undegraded":"2023-10-09T04:08:53.657967+0000","last_fullsized":"2023-10-09T04:08:53.657967+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.647771+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:08:53.647771+0000","last_peered":"2023-10-09T04:08:53.647771+0000","last_clean":"2023-10-09T04:08:53.647771+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:08:53.647771+0000","last_undegraded":"2023-10-09T04:08:53.647771+0000","last_fullsized":"2023-10-09T04:08:53.647771+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"163'13418","reported_seq":13497,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.455724+0000","last_change":"2023-10-09T04:06:45.485649+0000","last_active":"2023-10-09T04:09:00.455724+0000","last_peered":"2023-10-09T04:09:00.455724+0000","last_clean":"2023-10-09T04:09:00.455724+0000","last_became_active":"2023-10-09T04:06:45.471499+0000","last_became_peered":"2023-10-09T04:06:45.471499+0000","last_unstale":"2023-10-09T04:09:00.455724+0000","last_undegraded":"2023-10-09T04:09:00.455724+0000","last_fullsized":"2023-10-09T04:09:00.455724+0000","mapping_epoch":129,"log_start":"120'6218","ondisk_log_start":"120'6218","created":102,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.604485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6422528,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13418,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.658321+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:08:53.658321+0000","last_peered":"2023-10-09T04:08:53.658321+0000","last_clean":"2023-10-09T04:08:53.658321+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:08:53.658321+0000","last_undegraded":"2023-10-09T04:08:53.658321+0000","last_fullsized":"2023-10-09T04:08:53.658321+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.688595+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:08:56.688595+0000","last_peered":"2023-10-09T04:08:56.688595+0000","last_clean":"2023-10-09T04:08:56.688595+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:08:56.688595+0000","last_undegraded":"2023-10-09T04:08:56.688595+0000","last_fullsized":"2023-10-09T04:08:56.688595+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"163'10258","reported_seq":10476,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.391040+0000","last_change":"2023-10-09T04:06:40.080819+0000","last_active":"2023-10-09T04:09:00.391040+0000","last_peered":"2023-10-09T04:09:00.391040+0000","last_clean":"2023-10-09T04:09:00.391040+0000","last_became_active":"2023-10-09T04:06:35.281212+0000","last_became_peered":"2023-10-09T04:06:35.281212+0000","last_unstale":"2023-10-09T04:09:00.391040+0000","last_undegraded":"2023-10-09T04:09:00.391040+0000","last_fullsized":"2023-10-09T04:09:00.391040+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'7097","last_scrub_stamp":"2023-10-09T04:06:40.080754+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:40.080754+0000","objects_scrubbed":3223,"log_size":10258,"log_dups_size":0,"ondisk_log_size":10258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:12.223504+0000","scrub_duration":1.2295141649999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2883584,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3179,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657697+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:08:53.657697+0000","last_peered":"2023-10-09T04:08:53.657697+0000","last_clean":"2023-10-09T04:08:53.657697+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:08:53.657697+0000","last_undegraded":"2023-10-09T04:08:53.657697+0000","last_fullsized":"2023-10-09T04:08:53.657697+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.636492+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:08:53.636492+0000","last_peered":"2023-10-09T04:08:53.636492+0000","last_clean":"2023-10-09T04:08:53.636492+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:08:53.636492+0000","last_undegraded":"2023-10-09T04:08:53.636492+0000","last_fullsized":"2023-10-09T04:08:53.636492+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"163'10282","reported_seq":10440,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.039600+0000","last_change":"2023-10-09T04:06:36.749232+0000","last_active":"2023-10-09T04:09:02.039600+0000","last_peered":"2023-10-09T04:09:02.039600+0000","last_clean":"2023-10-09T04:09:02.039600+0000","last_became_active":"2023-10-09T04:06:35.284070+0000","last_became_peered":"2023-10-09T04:06:35.284070+0000","last_unstale":"2023-10-09T04:09:02.039600+0000","last_undegraded":"2023-10-09T04:09:02.039600+0000","last_fullsized":"2023-10-09T04:09:02.039600+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7102","last_scrub_stamp":"2023-10-09T04:06:36.748987+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:36.748987+0000","objects_scrubbed":3199,"log_size":10282,"log_dups_size":0,"ondisk_log_size":10282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:44.997815+0000","scrub_duration":0.68062300200000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3188,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.648040+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:08:53.648040+0000","last_peered":"2023-10-09T04:08:53.648040+0000","last_clean":"2023-10-09T04:08:53.648040+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:08:53.648040+0000","last_undegraded":"2023-10-09T04:08:53.648040+0000","last_fullsized":"2023-10-09T04:08:53.648040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.634401+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:08:53.634401+0000","last_peered":"2023-10-09T04:08:53.634401+0000","last_clean":"2023-10-09T04:08:53.634401+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:08:53.634401+0000","last_undegraded":"2023-10-09T04:08:53.634401+0000","last_fullsized":"2023-10-09T04:08:53.634401+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"163'13314","reported_seq":13384,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.655635+0000","last_change":"2023-10-09T04:04:51.279219+0000","last_active":"2023-10-09T04:09:02.655635+0000","last_peered":"2023-10-09T04:09:02.655635+0000","last_clean":"2023-10-09T04:09:02.655635+0000","last_became_active":"2023-10-09T04:04:51.278902+0000","last_became_peered":"2023-10-09T04:04:51.278902+0000","last_unstale":"2023-10-09T04:09:02.655635+0000","last_undegraded":"2023-10-09T04:09:02.655635+0000","last_fullsized":"2023-10-09T04:09:02.655635+0000","mapping_epoch":102,"log_start":"120'5900","ondisk_log_start":"120'5900","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:37.687116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13314,"num_write_kb":426048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":66,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709406+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:08:56.709406+0000","last_peered":"2023-10-09T04:08:56.709406+0000","last_clean":"2023-10-09T04:08:56.709406+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:08:56.709406+0000","last_undegraded":"2023-10-09T04:08:56.709406+0000","last_fullsized":"2023-10-09T04:08:56.709406+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689250+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:08:56.689250+0000","last_peered":"2023-10-09T04:08:56.689250+0000","last_clean":"2023-10-09T04:08:56.689250+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:08:56.689250+0000","last_undegraded":"2023-10-09T04:08:56.689250+0000","last_fullsized":"2023-10-09T04:08:56.689250+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"163'13406","reported_seq":13478,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.668250+0000","last_change":"2023-10-09T04:04:51.275697+0000","last_active":"2023-10-09T04:09:02.668250+0000","last_peered":"2023-10-09T04:09:02.668250+0000","last_clean":"2023-10-09T04:09:02.668250+0000","last_became_active":"2023-10-09T04:04:51.275402+0000","last_became_peered":"2023-10-09T04:04:51.275402+0000","last_unstale":"2023-10-09T04:09:02.668250+0000","last_undegraded":"2023-10-09T04:09:02.668250+0000","last_fullsized":"2023-10-09T04:09:02.668250+0000","mapping_epoch":102,"log_start":"114'4300","ondisk_log_start":"114'4300","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:26.490300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13406,"num_write_kb":428992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.690067+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:08:56.690067+0000","last_peered":"2023-10-09T04:08:56.690067+0000","last_clean":"2023-10-09T04:08:56.690067+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:08:56.690067+0000","last_undegraded":"2023-10-09T04:08:56.690067+0000","last_fullsized":"2023-10-09T04:08:56.690067+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689451+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:08:56.689451+0000","last_peered":"2023-10-09T04:08:56.689451+0000","last_clean":"2023-10-09T04:08:56.689451+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:08:56.689451+0000","last_undegraded":"2023-10-09T04:08:56.689451+0000","last_fullsized":"2023-10-09T04:08:56.689451+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"163'13576","reported_seq":13656,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.646674+0000","last_change":"2023-10-09T04:06:45.497813+0000","last_active":"2023-10-09T04:09:02.646674+0000","last_peered":"2023-10-09T04:09:02.646674+0000","last_clean":"2023-10-09T04:09:02.646674+0000","last_became_active":"2023-10-09T04:06:45.470723+0000","last_became_peered":"2023-10-09T04:06:45.470723+0000","last_unstale":"2023-10-09T04:09:02.646674+0000","last_undegraded":"2023-10-09T04:09:02.646674+0000","last_fullsized":"2023-10-09T04:09:02.646674+0000","mapping_epoch":129,"log_start":"116'4942","ondisk_log_start":"116'4942","created":102,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":8634,"log_dups_size":0,"ondisk_log_size":8634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:31.235972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13576,"num_write_kb":434432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.688383+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:08:56.688383+0000","last_peered":"2023-10-09T04:08:56.688383+0000","last_clean":"2023-10-09T04:08:56.688383+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:08:56.688383+0000","last_undegraded":"2023-10-09T04:08:56.688383+0000","last_fullsized":"2023-10-09T04:08:56.688383+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689868+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:08:56.689868+0000","last_peered":"2023-10-09T04:08:56.689868+0000","last_clean":"2023-10-09T04:08:56.689868+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:08:56.689868+0000","last_undegraded":"2023-10-09T04:08:56.689868+0000","last_fullsized":"2023-10-09T04:08:56.689868+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"163'13549","reported_seq":13619,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.308778+0000","last_change":"2023-10-09T04:04:51.279867+0000","last_active":"2023-10-09T04:09:00.308778+0000","last_peered":"2023-10-09T04:09:00.308778+0000","last_clean":"2023-10-09T04:09:00.308778+0000","last_became_active":"2023-10-09T04:04:51.279588+0000","last_became_peered":"2023-10-09T04:04:51.279588+0000","last_unstale":"2023-10-09T04:09:00.308778+0000","last_undegraded":"2023-10-09T04:09:00.308778+0000","last_fullsized":"2023-10-09T04:09:00.308778+0000","mapping_epoch":102,"log_start":"113'3500","ondisk_log_start":"113'3500","created":102,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10049,"log_dups_size":0,"ondisk_log_size":10049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:53.265205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5242904,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13549,"num_write_kb":436097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.644534+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:08:53.644534+0000","last_peered":"2023-10-09T04:08:53.644534+0000","last_clean":"2023-10-09T04:08:53.644534+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:08:53.644534+0000","last_undegraded":"2023-10-09T04:08:53.644534+0000","last_fullsized":"2023-10-09T04:08:53.644534+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.636424+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:08:53.636424+0000","last_peered":"2023-10-09T04:08:53.636424+0000","last_clean":"2023-10-09T04:08:53.636424+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:08:53.636424+0000","last_undegraded":"2023-10-09T04:08:53.636424+0000","last_fullsized":"2023-10-09T04:08:53.636424+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":137,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.638737+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:08:53.638737+0000","last_peered":"2023-10-09T04:08:53.638737+0000","last_clean":"2023-10-09T04:08:53.638737+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:08:53.638737+0000","last_undegraded":"2023-10-09T04:08:53.638737+0000","last_fullsized":"2023-10-09T04:08:53.638737+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.648398+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:08:53.648398+0000","last_peered":"2023-10-09T04:08:53.648398+0000","last_clean":"2023-10-09T04:08:53.648398+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:08:53.648398+0000","last_undegraded":"2023-10-09T04:08:53.648398+0000","last_fullsized":"2023-10-09T04:08:53.648398+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.636279+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:08:53.636279+0000","last_peered":"2023-10-09T04:08:53.636279+0000","last_clean":"2023-10-09T04:08:53.636279+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:08:53.636279+0000","last_undegraded":"2023-10-09T04:08:53.636279+0000","last_fullsized":"2023-10-09T04:08:53.636279+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"163'10040","reported_seq":10193,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:01.995305+0000","last_change":"2023-10-09T04:06:43.612195+0000","last_active":"2023-10-09T04:09:01.995305+0000","last_peered":"2023-10-09T04:09:01.995305+0000","last_clean":"2023-10-09T04:09:01.995305+0000","last_became_active":"2023-10-09T04:06:35.280772+0000","last_became_peered":"2023-10-09T04:06:35.280772+0000","last_unstale":"2023-10-09T04:09:01.995305+0000","last_undegraded":"2023-10-09T04:09:01.995305+0000","last_fullsized":"2023-10-09T04:09:01.995305+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"127'7121","last_scrub_stamp":"2023-10-09T04:06:43.612051+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:43.612051+0000","objects_scrubbed":2930,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:34.878053+0000","scrub_duration":0.51436748099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2919,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.6","version":"163'10272","reported_seq":10398,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.509215+0000","last_change":"2023-10-09T04:09:03.509196+0000","last_active":"2023-10-09T04:09:03.509215+0000","last_peered":"2023-10-09T04:09:03.509215+0000","last_clean":"2023-10-09T04:09:03.509215+0000","last_became_active":"2023-10-09T04:06:45.477025+0000","last_became_peered":"2023-10-09T04:06:45.477025+0000","last_unstale":"2023-10-09T04:09:03.509215+0000","last_undegraded":"2023-10-09T04:09:03.509215+0000","last_fullsized":"2023-10-09T04:09:03.509215+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10272,"log_dups_size":0,"ondisk_log_size":10272,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6225920,"num_objects":95,"num_object_clones":0,"num_object_copies":285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6746,"num_write_kb":218912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.650859+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:08:53.650859+0000","last_peered":"2023-10-09T04:08:53.650859+0000","last_clean":"2023-10-09T04:08:53.650859+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:08:53.650859+0000","last_undegraded":"2023-10-09T04:08:53.650859+0000","last_fullsized":"2023-10-09T04:08:53.650859+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.688949+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:08:56.688949+0000","last_peered":"2023-10-09T04:08:56.688949+0000","last_clean":"2023-10-09T04:08:56.688949+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:08:56.688949+0000","last_undegraded":"2023-10-09T04:08:56.688949+0000","last_fullsized":"2023-10-09T04:08:56.688949+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.638773+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:08:53.638773+0000","last_peered":"2023-10-09T04:08:53.638773+0000","last_clean":"2023-10-09T04:08:53.638773+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:08:53.638773+0000","last_undegraded":"2023-10-09T04:08:53.638773+0000","last_fullsized":"2023-10-09T04:08:53.638773+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"163'10142","reported_seq":10243,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.447012+0000","last_change":"2023-10-09T04:08:51.042057+0000","last_active":"2023-10-09T04:09:00.447012+0000","last_peered":"2023-10-09T04:09:00.447012+0000","last_clean":"2023-10-09T04:09:00.447012+0000","last_became_active":"2023-10-09T04:06:35.267902+0000","last_became_peered":"2023-10-09T04:06:35.267902+0000","last_unstale":"2023-10-09T04:09:00.447012+0000","last_undegraded":"2023-10-09T04:09:00.447012+0000","last_fullsized":"2023-10-09T04:09:00.447012+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10142,"log_dups_size":0,"ondisk_log_size":10142,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1343488,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6698,"num_write_kb":215008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":147,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696489+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:08:56.696489+0000","last_peered":"2023-10-09T04:08:56.696489+0000","last_clean":"2023-10-09T04:08:56.696489+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:08:56.696489+0000","last_undegraded":"2023-10-09T04:08:56.696489+0000","last_fullsized":"2023-10-09T04:08:56.696489+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.636439+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:08:53.636439+0000","last_peered":"2023-10-09T04:08:53.636439+0000","last_clean":"2023-10-09T04:08:53.636439+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:08:53.636439+0000","last_undegraded":"2023-10-09T04:08:53.636439+0000","last_fullsized":"2023-10-09T04:08:53.636439+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":132,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699024+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:08:56.699024+0000","last_peered":"2023-10-09T04:08:56.699024+0000","last_clean":"2023-10-09T04:08:56.699024+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:08:56.699024+0000","last_undegraded":"2023-10-09T04:08:56.699024+0000","last_fullsized":"2023-10-09T04:08:56.699024+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"163'10005","reported_seq":10220,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.417346+0000","last_change":"2023-10-09T04:06:36.855957+0000","last_active":"2023-10-09T04:09:00.417346+0000","last_peered":"2023-10-09T04:09:00.417346+0000","last_clean":"2023-10-09T04:09:00.417346+0000","last_became_active":"2023-10-09T04:06:35.290034+0000","last_became_peered":"2023-10-09T04:06:35.290034+0000","last_unstale":"2023-10-09T04:09:00.417346+0000","last_undegraded":"2023-10-09T04:09:00.417346+0000","last_fullsized":"2023-10-09T04:09:00.417346+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'6925","last_scrub_stamp":"2023-10-09T04:06:36.855876+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:36.855876+0000","objects_scrubbed":3147,"log_size":10005,"log_dups_size":0,"ondisk_log_size":10005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:15:54.835217+0000","scrub_duration":1.131125129,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2949120,"num_objects":45,"num_object_clones":0,"num_object_copies":135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3102,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.647870+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:08:53.647870+0000","last_peered":"2023-10-09T04:08:53.647870+0000","last_clean":"2023-10-09T04:08:53.647870+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:08:53.647870+0000","last_undegraded":"2023-10-09T04:08:53.647870+0000","last_fullsized":"2023-10-09T04:08:53.647870+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":503,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657488+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:08:53.657488+0000","last_peered":"2023-10-09T04:08:53.657488+0000","last_clean":"2023-10-09T04:08:53.657488+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:08:53.657488+0000","last_undegraded":"2023-10-09T04:08:53.657488+0000","last_fullsized":"2023-10-09T04:08:53.657488+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":59,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.650841+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:08:53.650841+0000","last_peered":"2023-10-09T04:08:53.650841+0000","last_clean":"2023-10-09T04:08:53.650841+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:08:53.650841+0000","last_undegraded":"2023-10-09T04:08:53.650841+0000","last_fullsized":"2023-10-09T04:08:53.650841+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":43,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689638+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:08:56.689638+0000","last_peered":"2023-10-09T04:08:56.689638+0000","last_clean":"2023-10-09T04:08:56.689638+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:08:56.689638+0000","last_undegraded":"2023-10-09T04:08:56.689638+0000","last_fullsized":"2023-10-09T04:08:56.689638+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":143,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657377+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:08:53.657377+0000","last_peered":"2023-10-09T04:08:53.657377+0000","last_clean":"2023-10-09T04:08:53.657377+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:08:53.657377+0000","last_undegraded":"2023-10-09T04:08:53.657377+0000","last_fullsized":"2023-10-09T04:08:53.657377+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.634157+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:08:53.634157+0000","last_peered":"2023-10-09T04:08:53.634157+0000","last_clean":"2023-10-09T04:08:53.634157+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:08:53.634157+0000","last_undegraded":"2023-10-09T04:08:53.634157+0000","last_fullsized":"2023-10-09T04:08:53.634157+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.632691+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:08:53.632691+0000","last_peered":"2023-10-09T04:08:53.632691+0000","last_clean":"2023-10-09T04:08:53.632691+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:08:53.632691+0000","last_undegraded":"2023-10-09T04:08:53.632691+0000","last_fullsized":"2023-10-09T04:08:53.632691+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.4","version":"163'10327","reported_seq":10482,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.573008+0000","last_change":"2023-10-09T04:06:40.914492+0000","last_active":"2023-10-09T04:09:02.573008+0000","last_peered":"2023-10-09T04:09:02.573008+0000","last_clean":"2023-10-09T04:09:02.573008+0000","last_became_active":"2023-10-09T04:06:35.285472+0000","last_became_peered":"2023-10-09T04:06:35.285472+0000","last_unstale":"2023-10-09T04:09:02.573008+0000","last_undegraded":"2023-10-09T04:09:02.573008+0000","last_fullsized":"2023-10-09T04:09:02.573008+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"126'7221","last_scrub_stamp":"2023-10-09T04:06:40.914305+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:40.914305+0000","objects_scrubbed":3121,"log_size":10327,"log_dups_size":0,"ondisk_log_size":10327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:17.920132+0000","scrub_duration":0.80758971499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3121,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657772+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:08:53.657772+0000","last_peered":"2023-10-09T04:08:53.657772+0000","last_clean":"2023-10-09T04:08:53.657772+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:08:53.657772+0000","last_undegraded":"2023-10-09T04:08:53.657772+0000","last_fullsized":"2023-10-09T04:08:53.657772+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.638838+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:08:53.638838+0000","last_peered":"2023-10-09T04:08:53.638838+0000","last_clean":"2023-10-09T04:08:53.638838+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:08:53.638838+0000","last_undegraded":"2023-10-09T04:08:53.638838+0000","last_fullsized":"2023-10-09T04:08:53.638838+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.633982+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:08:53.633982+0000","last_peered":"2023-10-09T04:08:53.633982+0000","last_clean":"2023-10-09T04:08:53.633982+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:08:53.633982+0000","last_undegraded":"2023-10-09T04:08:53.633982+0000","last_fullsized":"2023-10-09T04:08:53.633982+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"163'10168","reported_seq":10256,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.055984+0000","last_change":"2023-10-09T04:09:03.593133+0000","last_active":"2023-10-09T04:09:03.055984+0000","last_peered":"2023-10-09T04:09:03.055984+0000","last_clean":"2023-10-09T04:09:03.055984+0000","last_became_active":"2023-10-09T04:06:35.290191+0000","last_became_peered":"2023-10-09T04:06:35.290191+0000","last_unstale":"2023-10-09T04:09:03.055984+0000","last_undegraded":"2023-10-09T04:09:03.055984+0000","last_fullsized":"2023-10-09T04:09:03.055984+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10168,"log_dups_size":0,"ondisk_log_size":10168,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491520,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6702,"num_write_kb":214720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":67,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657765+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:08:53.657765+0000","last_peered":"2023-10-09T04:08:53.657765+0000","last_clean":"2023-10-09T04:08:53.657765+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:08:53.657765+0000","last_undegraded":"2023-10-09T04:08:53.657765+0000","last_fullsized":"2023-10-09T04:08:53.657765+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.701062+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:08:56.701062+0000","last_peered":"2023-10-09T04:08:56.701062+0000","last_clean":"2023-10-09T04:08:56.701062+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:08:56.701062+0000","last_undegraded":"2023-10-09T04:08:56.701062+0000","last_fullsized":"2023-10-09T04:08:56.701062+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":93,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657607+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:08:53.657607+0000","last_peered":"2023-10-09T04:08:53.657607+0000","last_clean":"2023-10-09T04:08:53.657607+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:08:53.657607+0000","last_undegraded":"2023-10-09T04:08:53.657607+0000","last_fullsized":"2023-10-09T04:08:53.657607+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"163'10260","reported_seq":10472,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.655637+0000","last_change":"2023-10-09T04:06:43.998989+0000","last_active":"2023-10-09T04:09:02.655637+0000","last_peered":"2023-10-09T04:09:02.655637+0000","last_clean":"2023-10-09T04:09:02.655637+0000","last_became_active":"2023-10-09T04:06:35.309761+0000","last_became_peered":"2023-10-09T04:06:35.309761+0000","last_unstale":"2023-10-09T04:09:02.655637+0000","last_undegraded":"2023-10-09T04:09:02.655637+0000","last_fullsized":"2023-10-09T04:09:02.655637+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"128'7225","last_scrub_stamp":"2023-10-09T04:06:43.998914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:43.998914+0000","objects_scrubbed":3035,"log_size":10260,"log_dups_size":0,"ondisk_log_size":10260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:01.096310+0000","scrub_duration":0.32511040200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3035,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":91,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.657888+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:08:53.657888+0000","last_peered":"2023-10-09T04:08:53.657888+0000","last_clean":"2023-10-09T04:08:53.657888+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:08:53.657888+0000","last_undegraded":"2023-10-09T04:08:53.657888+0000","last_fullsized":"2023-10-09T04:08:53.657888+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.632621+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:08:53.632621+0000","last_peered":"2023-10-09T04:08:53.632621+0000","last_clean":"2023-10-09T04:08:53.632621+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:08:53.632621+0000","last_undegraded":"2023-10-09T04:08:53.632621+0000","last_fullsized":"2023-10-09T04:08:53.632621+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.688460+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:08:56.688460+0000","last_peered":"2023-10-09T04:08:56.688460+0000","last_clean":"2023-10-09T04:08:56.688460+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:08:56.688460+0000","last_undegraded":"2023-10-09T04:08:56.688460+0000","last_fullsized":"2023-10-09T04:08:56.688460+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.3","version":"163'10390","reported_seq":10611,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.430681+0000","last_change":"2023-10-09T04:06:37.747828+0000","last_active":"2023-10-09T04:09:00.430681+0000","last_peered":"2023-10-09T04:09:00.430681+0000","last_clean":"2023-10-09T04:09:00.430681+0000","last_became_active":"2023-10-09T04:06:35.269921+0000","last_became_peered":"2023-10-09T04:06:35.269921+0000","last_unstale":"2023-10-09T04:09:00.430681+0000","last_undegraded":"2023-10-09T04:09:00.430681+0000","last_fullsized":"2023-10-09T04:09:00.430681+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":102,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"125'7180","last_scrub_stamp":"2023-10-09T04:06:37.747725+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:37.747725+0000","objects_scrubbed":3259,"log_size":10390,"log_dups_size":0,"ondisk_log_size":10390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:22.963882+0000","scrub_duration":0.83890256799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3220,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.634496+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:08:53.634496+0000","last_peered":"2023-10-09T04:08:53.634496+0000","last_clean":"2023-10-09T04:08:53.634496+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:08:53.634496+0000","last_undegraded":"2023-10-09T04:08:53.634496+0000","last_fullsized":"2023-10-09T04:08:53.634496+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":146,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.695552+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:08:56.695552+0000","last_peered":"2023-10-09T04:08:56.695552+0000","last_clean":"2023-10-09T04:08:56.695552+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:08:56.695552+0000","last_undegraded":"2023-10-09T04:08:56.695552+0000","last_fullsized":"2023-10-09T04:08:56.695552+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"163'10466","reported_seq":10625,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:01.985505+0000","last_change":"2023-10-09T04:06:42.872402+0000","last_active":"2023-10-09T04:09:01.985505+0000","last_peered":"2023-10-09T04:09:01.985505+0000","last_clean":"2023-10-09T04:09:01.985505+0000","last_became_active":"2023-10-09T04:06:35.306098+0000","last_became_peered":"2023-10-09T04:06:35.306098+0000","last_unstale":"2023-10-09T04:09:01.985505+0000","last_undegraded":"2023-10-09T04:09:01.985505+0000","last_fullsized":"2023-10-09T04:09:01.985505+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"127'7283","last_scrub_stamp":"2023-10-09T04:06:42.872198+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:06:42.872198+0000","objects_scrubbed":3197,"log_size":10466,"log_dups_size":0,"ondisk_log_size":10466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:26.895471+0000","scrub_duration":0.738535999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3187,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.648129+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:08:53.648129+0000","last_peered":"2023-10-09T04:08:53.648129+0000","last_clean":"2023-10-09T04:08:53.648129+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:08:53.648129+0000","last_undegraded":"2023-10-09T04:08:53.648129+0000","last_fullsized":"2023-10-09T04:08:53.648129+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.689013+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:08:56.689013+0000","last_peered":"2023-10-09T04:08:56.689013+0000","last_clean":"2023-10-09T04:08:56.689013+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:08:56.689013+0000","last_undegraded":"2023-10-09T04:08:56.689013+0000","last_fullsized":"2023-10-09T04:08:56.689013+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"163'10119","reported_seq":10219,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:00.433594+0000","last_change":"2023-10-09T04:08:55.076743+0000","last_active":"2023-10-09T04:09:00.433594+0000","last_peered":"2023-10-09T04:09:00.433594+0000","last_clean":"2023-10-09T04:09:00.433594+0000","last_became_active":"2023-10-09T04:06:35.303590+0000","last_became_peered":"2023-10-09T04:06:35.303590+0000","last_unstale":"2023-10-09T04:09:00.433594+0000","last_undegraded":"2023-10-09T04:09:00.433594+0000","last_fullsized":"2023-10-09T04:09:00.433594+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":124,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:04:50.252100+0000","last_clean_scrub_stamp":"2023-10-09T04:04:50.252100+0000","objects_scrubbed":0,"log_size":10119,"log_dups_size":0,"ondisk_log_size":10119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7503872,"num_objects":115,"num_object_clones":0,"num_object_copies":345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6642,"num_write_kb":216192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.648470+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:08:53.648470+0000","last_peered":"2023-10-09T04:08:53.648470+0000","last_clean":"2023-10-09T04:08:53.648470+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:08:53.648470+0000","last_undegraded":"2023-10-09T04:08:53.648470+0000","last_fullsized":"2023-10-09T04:08:53.648470+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.690134+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:08:56.690134+0000","last_peered":"2023-10-09T04:08:56.690134+0000","last_clean":"2023-10-09T04:08:56.690134+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:08:56.690134+0000","last_undegraded":"2023-10-09T04:08:56.690134+0000","last_fullsized":"2023-10-09T04:08:56.690134+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":43,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:59.822467+0000","last_change":"2023-10-09T04:06:57.499785+0000","last_active":"2023-10-09T04:08:59.822467+0000","last_peered":"2023-10-09T04:08:59.822467+0000","last_clean":"2023-10-09T04:08:59.822467+0000","last_became_active":"2023-10-09T04:06:57.499419+0000","last_became_peered":"2023-10-09T04:06:57.499419+0000","last_unstale":"2023-10-09T04:08:59.822467+0000","last_undegraded":"2023-10-09T04:08:59.822467+0000","last_fullsized":"2023-10-09T04:08:59.822467+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:14.270664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696566+0000","last_change":"2023-10-09T04:06:45.454363+0000","last_active":"2023-10-09T04:08:56.696566+0000","last_peered":"2023-10-09T04:08:56.696566+0000","last_clean":"2023-10-09T04:08:56.696566+0000","last_became_active":"2023-10-09T04:06:45.454065+0000","last_became_peered":"2023-10-09T04:06:45.454065+0000","last_unstale":"2023-10-09T04:08:56.696566+0000","last_undegraded":"2023-10-09T04:08:56.696566+0000","last_fullsized":"2023-10-09T04:08:56.696566+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:21.580027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-10-09T04:08:53.632848+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:08:53.632848+0000","last_peered":"2023-10-09T04:08:53.632848+0000","last_clean":"2023-10-09T04:08:53.632848+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:08:53.632848+0000","last_undegraded":"2023-10-09T04:08:53.632848+0000","last_fullsized":"2023-10-09T04:08:53.632848+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.688231+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:08:56.688231+0000","last_peered":"2023-10-09T04:08:56.688231+0000","last_clean":"2023-10-09T04:08:56.688231+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:08:56.688231+0000","last_undegraded":"2023-10-09T04:08:56.688231+0000","last_fullsized":"2023-10-09T04:08:56.688231+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":54394904,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":743,"num_objects_misplaced":743,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158262,"num_write_kb":3895297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4303,"num_bytes_recovered":95485952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":754589696,"data_stored":754581552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":257155,"ondisk_log_size":257155,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201478,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":411520,"kb_used_data":114232,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93333632,"statfs":{"total":95995035648,"available":95573639168,"internally_reserved":0,"allocated":116973568,"data_stored":116467915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":1.5589999999999999,"15min":0.86299999999999999},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":17.398,"5min":17.398,"15min":17.398},"last":5.3520000000000003},{"interface":"front","average":{"1min":3.173,"5min":1.4510000000000001,"15min":0.82899999999999996},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.306999999999999,"5min":17.306999999999999,"15min":17.306999999999999},"last":1.9810000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":3.2320000000000002,"5min":1.752,"15min":1.0029999999999999},"min":{"1min":0.26200000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.051,"5min":10.051,"15min":10.051},"last":0.373},{"interface":"front","average":{"1min":2.2080000000000002,"5min":1.6220000000000001,"15min":1.0089999999999999},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.7680000000000007,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":2.0099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.5990000000000002,"5min":2.246,"15min":1.1140000000000001},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":13.887,"5min":58.615000000000002,"15min":58.615000000000002},"last":4.2249999999999996},{"interface":"front","average":{"1min":3.839,"5min":3.3820000000000001,"15min":1.5309999999999999},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.872,"5min":99.372,"15min":99.372},"last":4.2460000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":3.218,"15min":1.4590000000000001},"min":{"1min":0.33400000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.124000000000001,"5min":99.134,"15min":99.134},"last":3.9169999999999998},{"interface":"front","average":{"1min":2.6480000000000001,"5min":3.367,"15min":1.5329999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.140000000000001,"5min":99.117999999999995,"15min":99.117999999999995},"last":3.9350000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.80300000000000005,"15min":0.61499999999999999},"min":{"1min":0.27500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.0199999999999996,"5min":6.0199999999999996,"15min":6.3129999999999997},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.79900000000000004,"15min":0.621},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":5.3630000000000004,"5min":6.5590000000000002,"15min":6.5590000000000002},"last":1.9470000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.73999999999999999,"15min":0.58799999999999997},"min":{"1min":0.27700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.867,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.236,"5min":0.875,"15min":0.66100000000000003},"min":{"1min":0.30399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.6779999999999999,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.312}]},{"osd":6,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":0.877,"15min":0.65300000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.3559999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.48,"5min":0.94899999999999995,"15min":0.70699999999999996},"min":{"1min":0.35099999999999998,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":5.6420000000000003,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":1.161}]}]},{"osd":6,"up_from":47,"seq":201863463116,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":430008,"kb_used_data":132720,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93315144,"statfs":{"total":95995035648,"available":95554707456,"internally_reserved":0,"allocated":135905280,"data_stored":135415509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:14 2023","interfaces":[{"interface":"back","average":{"1min":3.3279999999999998,"5min":2.2810000000000001,"15min":1.117},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":35.393000000000001,"5min":78.983999999999995,"15min":78.983999999999995},"last":2.9780000000000002},{"interface":"front","average":{"1min":4.8010000000000002,"5min":2.9089999999999998,"15min":1.327},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":35.240000000000002,"5min":78.998000000000005,"15min":78.998000000000005},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":5.399,"5min":3.0640000000000001,"15min":1.369},"min":{"1min":0.30599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":57.094000000000001,"5min":57.094000000000001,"15min":57.094000000000001},"last":3.2210000000000001},{"interface":"front","average":{"1min":5.8369999999999997,"5min":3.1829999999999998,"15min":1.399},"min":{"1min":0.27000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":57.003,"5min":57.003,"15min":57.003},"last":9.484}]},{"osd":2,"last update":"Mon Oct 9 04:08:14 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":2.2200000000000002,"15min":1.113},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":12.795999999999999,"5min":42.195999999999998,"15min":42.195999999999998},"last":3.137},{"interface":"front","average":{"1min":2.528,"5min":2.2650000000000001,"15min":1.163},"min":{"1min":0.439,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.827,"5min":42.165999999999997,"15min":42.165999999999997},"last":3.1699999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":8.8030000000000008,"5min":3.7559999999999998,"15min":1.6619999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":57.509999999999998,"5min":57.509999999999998,"15min":57.509999999999998},"last":9.4489999999999998},{"interface":"front","average":{"1min":9.3079999999999998,"5min":4.0510000000000002,"15min":1.7869999999999999},"min":{"1min":0.41899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":57.453000000000003,"5min":57.453000000000003,"15min":57.453000000000003},"last":3.3740000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.52500000000000002,"15min":0.48399999999999999},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.72999999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.85599999999999998,"15min":0.62},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":1.7609999999999999,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.226,"15min":0.752},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.734,"5min":54.773000000000003,"15min":54.773000000000003},"last":1.679},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.3999999999999999,"15min":0.84899999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8220000000000001,"5min":54.853999999999999,"15min":54.853999999999999},"last":1.694}]},{"osd":7,"last update":"Mon Oct 9 04:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.64000000000000001,"15min":0.59099999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.752,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.77500000000000002,"15min":0.66500000000000004},"min":{"1min":0.34100000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.7679999999999998,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.69199999999999995}]}]},{"osd":1,"up_from":89,"seq":382252089424,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":119364,"kb_used_data":1660,"kb_used_omap":9,"kb_used_meta":117686,"kb_avail":93625788,"statfs":{"total":95995035648,"available":95872806912,"internally_reserved":0,"allocated":1699840,"data_stored":1074248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":120511465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,0,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":2.258,"15min":1},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0389999999999997,"5min":45.811999999999998,"15min":45.811999999999998},"last":0.17199999999999999},{"interface":"front","average":{"1min":2.7490000000000001,"5min":3.5089999999999999,"15min":1.431},"min":{"1min":0.24099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.4820000000000002,"5min":99.831999999999994,"15min":99.831999999999994},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.4670000000000001,"15min":1.0509999999999999},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":18.143000000000001,"5min":121.54600000000001,"15min":121.54600000000001},"last":0.22},{"interface":"front","average":{"1min":3.0030000000000001,"5min":2.7410000000000001,"15min":1.1579999999999999},"min":{"1min":0.26600000000000001,"5min":0.186,"15min":0.186},"max":{"1min":8.6669999999999998,"5min":99.899000000000001,"15min":99.899000000000001},"last":0.503}]},{"osd":3,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":2.8119999999999998,"15min":1.1879999999999999},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.56,"5min":42.588999999999999,"15min":42.588999999999999},"last":6.9809999999999999},{"interface":"front","average":{"1min":3.8330000000000002,"5min":4.0110000000000001,"15min":1.5920000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.163,"5min":100.02500000000001,"15min":100.02500000000001},"last":6.7460000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.581,"15min":1.3220000000000001},"min":{"1min":0.33600000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.5529999999999999,"5min":99.989999999999995,"15min":99.989999999999995},"last":0.439},{"interface":"front","average":{"1min":1.768,"5min":2.5099999999999998,"15min":1.304},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8239999999999998,"5min":100.249,"15min":100.249},"last":1.6970000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":2.492,"15min":1.3129999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.5780000000000003,"5min":99.933999999999997,"15min":99.933999999999997},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.6960000000000002,"15min":1.3819999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.4779999999999998,"5min":100.34099999999999,"15min":100.34099999999999},"last":6.0069999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":3.0209999999999999,"15min":1.4830000000000001},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.968,"5min":100.309,"15min":100.309},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.044,"5min":2.8319999999999999,"15min":1.4379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.9089999999999998,"5min":100.143,"15min":100.143},"last":0.29799999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.8599999999999999,"15min":1.45},"min":{"1min":0.373,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.8490000000000002,"5min":100.214,"15min":100.214},"last":6.8659999999999997},{"interface":"front","average":{"1min":2.7719999999999998,"5min":2.7280000000000002,"15min":1.4219999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":7.8869999999999996,"5min":100.113,"15min":100.113},"last":7.016}]}]},{"osd":0,"up_from":84,"seq":360777252950,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112468,"kb_used_data":12748,"kb_used_omap":28,"kb_used_meta":99683,"kb_avail":93632684,"statfs":{"total":95995035648,"available":95879868416,"internally_reserved":0,"allocated":13053952,"data_stored":12410448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":102076240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:08:44 2023","interfaces":[{"interface":"back","average":{"1min":4.157,"5min":4.2679999999999998,"15min":1.6719999999999999},"min":{"1min":0.19800000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":44.287999999999997,"5min":118.48,"15min":118.48},"last":3.7120000000000002},{"interface":"front","average":{"1min":4.4889999999999999,"5min":4.5720000000000001,"15min":1.766},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":43.906999999999996,"5min":118.384,"15min":118.384},"last":2.641}]},{"osd":2,"last update":"Mon Oct 9 04:08:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.5680000000000001,"15min":1.468},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":6.1280000000000001,"5min":118.54000000000001,"15min":118.54000000000001},"last":2.343},{"interface":"front","average":{"1min":5.3570000000000002,"5min":4.3819999999999997,"15min":1.74},"min":{"1min":0.29299999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":44.436999999999998,"5min":118.727,"15min":118.727},"last":2.21}]},{"osd":3,"last update":"Mon Oct 9 04:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.6959999999999997,"5min":7.4530000000000003,"15min":2.7450000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":44.460000000000001,"5min":118.73999999999999,"15min":118.73999999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":6.0570000000000004,"5min":7.4199999999999999,"15min":2.7679999999999998},"min":{"1min":0.34899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":43.972000000000001,"5min":118.584,"15min":118.584},"last":0.45500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":3.141,"15min":1.4390000000000001},"min":{"1min":0.29699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.6589999999999998,"5min":118.633,"15min":118.633},"last":0.505},{"interface":"front","average":{"1min":5.1150000000000002,"5min":3.9289999999999998,"15min":1.7150000000000001},"min":{"1min":0.47599999999999998,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":44.475000000000001,"5min":118.913,"15min":118.913},"last":2.2519999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":5.556,"5min":5.383,"15min":2.21},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":44.494,"5min":182.68899999999999,"15min":182.68899999999999},"last":2.282},{"interface":"front","average":{"1min":4.3479999999999999,"5min":5.0449999999999999,"15min":2.1070000000000002},"min":{"1min":0.41999999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":44.328000000000003,"5min":182.66300000000001,"15min":182.66300000000001},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":4.3680000000000003,"5min":3.0670000000000002,"15min":1.4630000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":44.384999999999998,"5min":118.864,"15min":118.864},"last":3.7610000000000001},{"interface":"front","average":{"1min":2.5249999999999999,"5min":2.6099999999999999,"15min":1.3009999999999999},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.317},"max":{"1min":6.6859999999999999,"5min":118.70399999999999,"15min":118.70399999999999},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":4.2629999999999999,"5min":5.4820000000000002,"15min":2.262},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":44.420000000000002,"5min":185.024,"15min":185.024},"last":3.8260000000000001},{"interface":"front","average":{"1min":1.996,"5min":4.798,"15min":2.044},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6200000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.52000000000000002}]}]},{"osd":2,"up_from":94,"seq":403726925898,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":434680,"kb_used_data":276656,"kb_used_omap":81,"kb_used_meta":157934,"kb_avail":93310472,"statfs":{"total":95995035648,"available":95549923328,"internally_reserved":0,"allocated":283295744,"data_stored":282687619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":161724634},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":3.7869999999999999,"5min":1.8240000000000001,"15min":0.83199999999999996},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":9.8719999999999999,"5min":9.8719999999999999,"15min":9.8719999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":5.2300000000000004,"5min":2.2090000000000001,"15min":0.997},"min":{"1min":0.33800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":25.131,"5min":25.131,"15min":25.131},"last":0.44800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":6.9770000000000003,"5min":3.274,"15min":1.341},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.221},"max":{"1min":53.695,"5min":68.009,"15min":68.009},"last":0.35299999999999998},{"interface":"front","average":{"1min":6.4340000000000002,"5min":3.5649999999999999,"15min":1.4330000000000001},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":53.439,"5min":66.975999999999999,"15min":66.975999999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.359,"15min":1.0409999999999999},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.77,"5min":27.562000000000001,"15min":27.562000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":5.4850000000000003,"5min":2.9830000000000001,"15min":1.2470000000000001},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":46.286999999999999,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.318}]},{"osd":4,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.7949999999999999,"15min":0.99399999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.9249999999999998,"5min":11.038,"15min":11.038},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.5,"15min":0.89700000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.1420000000000003,"5min":14.305,"15min":14.305},"last":0.70199999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":1.5569999999999999,"15min":0.95499999999999996},"min":{"1min":0.316,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":14.132999999999999,"5min":14.132999999999999,"15min":14.132999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.5099999999999998,"5min":1.8999999999999999,"15min":1.0629999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.883,"5min":11.087999999999999,"15min":11.087999999999999},"last":0.61599999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":1.425,"15min":0.89400000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.0979999999999999,"5min":5.0979999999999999,"15min":5.0979999999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":3.073,"5min":1.595,"15min":0.98499999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":14.103999999999999,"5min":14.103999999999999,"15min":14.103999999999999},"last":0.73299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.9849999999999999,"5min":1.829,"15min":1.056},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.8309999999999995,"5min":14.364000000000001,"15min":14.364000000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":3.2730000000000001,"5min":1.8260000000000001,"15min":1.0820000000000001},"min":{"1min":0.39000000000000001,"5min":0.37,"15min":0.37},"max":{"1min":14.157,"5min":14.157,"15min":14.157},"last":0.80200000000000005}]}]},{"osd":3,"up_from":99,"seq":425201762371,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":440148,"kb_used_data":195980,"kb_used_omap":7,"kb_used_meta":244088,"kb_avail":93305004,"statfs":{"total":95995035648,"available":95544324096,"internally_reserved":0,"allocated":200683520,"data_stored":200099575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":249946728},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":4.181,"5min":5.6589999999999998,"15min":2.496},"min":{"1min":0.23200000000000001,"5min":0.185,"15min":0.185},"max":{"1min":13.166,"5min":106.22,"15min":106.22},"last":0.19400000000000001},{"interface":"front","average":{"1min":4.2809999999999997,"5min":5.6660000000000004,"15min":2.4969999999999999},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.722,"5min":106.986,"15min":106.986},"last":3.226}]},{"osd":1,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9990000000000001,"5min":3.8370000000000002,"15min":1.498},"min":{"1min":0.22,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.108000000000001,"5min":106.121,"15min":106.121},"last":4.516},{"interface":"front","average":{"1min":4.4619999999999997,"5min":3.319,"15min":1.3400000000000001},"min":{"1min":0.30499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":16.977,"5min":65.015000000000001,"15min":65.015000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":3.726,"5min":4.4749999999999996,"15min":1.7370000000000001},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.205,"5min":106.105,"15min":106.105},"last":2.7799999999999998},{"interface":"front","average":{"1min":2.3460000000000001,"5min":4.6449999999999996,"15min":1.7889999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2170000000000005,"5min":106.70399999999999,"15min":106.70399999999999},"last":3.004}]},{"osd":4,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0750000000000002,"5min":3.839,"15min":1.675},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":37.652000000000001,"5min":106.85899999999999,"15min":106.85899999999999},"last":3.1640000000000001},{"interface":"front","average":{"1min":4.9370000000000003,"5min":2.2290000000000001,"15min":1.1399999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":37.442,"5min":37.442,"15min":37.442},"last":8.2710000000000008}]},{"osd":5,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":1.964,"15min":1.05},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.8940000000000001,"5min":12.819000000000001,"15min":12.819000000000001},"last":5.2249999999999996},{"interface":"front","average":{"1min":2.625,"5min":2.0139999999999998,"15min":1.4239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.228999999999999,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.42699999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6099999999999999,"5min":3.2210000000000001,"15min":1.8149999999999999},"min":{"1min":0.35499999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.255000000000001,"5min":106.036,"15min":106.036},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.7410000000000001,"5min":3.319,"15min":1.516},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":15.622,"5min":106.741,"15min":106.741},"last":3.1190000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":3.492,"15min":1.583},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":15.685,"5min":106.82599999999999,"15min":106.82599999999999},"last":3.1000000000000001},{"interface":"front","average":{"1min":2.96,"5min":3.2330000000000001,"15min":1.478},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.8159999999999998,"5min":106.084,"15min":106.084},"last":8.3109999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986388,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307004,"kb_used_data":1524,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438148,"statfs":{"total":95995035648,"available":95680663552,"internally_reserved":0,"allocated":1560576,"data_stored":1066299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":1.365,"15min":0.80600000000000005},"min":{"1min":0.22600000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.083,"5min":14.523,"15min":14.523},"last":0.249},{"interface":"front","average":{"1min":1.282,"5min":1.2989999999999999,"15min":0.81499999999999995},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.2549999999999999,"5min":8.3409999999999993,"15min":8.3409999999999993},"last":0.30099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":5.3109999999999999,"5min":2.3439999999999999,"15min":1.1479999999999999},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":79.075999999999993,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.36099999999999999},{"interface":"front","average":{"1min":5.7249999999999996,"5min":2.427,"15min":1.165},"min":{"1min":0.27000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":75.314999999999998,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.85699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:08:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.706,"15min":0.95599999999999996},"min":{"1min":0.316,"5min":0.245,"15min":0.245},"max":{"1min":7.4420000000000002,"5min":19.356000000000002,"15min":19.356000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.5459999999999998,"5min":1.595,"15min":0.90600000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.46,"5min":19.324000000000002,"15min":19.324000000000002},"last":0.52100000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.7110000000000001,"15min":0.93100000000000005},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.492,"5min":14.694000000000001,"15min":14.694000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.379,"5min":1.716,"15min":0.94399999999999995},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.8840000000000003,"5min":9.0449999999999999,"15min":9.0449999999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.75800000000000001,"15min":0.57299999999999995},"min":{"1min":0.27000000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.0870000000000002,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.048,"15min":0.69399999999999995},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9569999999999999,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 04:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0589999999999999,"15min":0.72099999999999997},"min":{"1min":0.308,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.6849999999999996,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.83199999999999996,"15min":0.64300000000000002},"min":{"1min":0.38200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:08:11 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.92300000000000004,"15min":0.67600000000000005},"min":{"1min":0.34699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.1219999999999999,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.377},{"interface":"front","average":{"1min":1.212,"5min":1.046,"15min":0.748},"min":{"1min":0.34399999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.9939999999999998,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.45800000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724751,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358056,"kb_used_data":44384,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93387096,"statfs":{"total":95995035648,"available":95628386304,"internally_reserved":0,"allocated":45449216,"data_stored":44962970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:21 2023","interfaces":[{"interface":"back","average":{"1min":4.8920000000000003,"5min":2.2629999999999999,"15min":1.153},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":47.954000000000001,"5min":63.037999999999997,"15min":63.037999999999997},"last":0.27700000000000002},{"interface":"front","average":{"1min":4.8339999999999996,"5min":2.3999999999999999,"15min":1.179},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":52.009,"5min":62.927999999999997,"15min":62.927999999999997},"last":0.32300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:08:38 2023","interfaces":[{"interface":"back","average":{"1min":4.593,"5min":3.1120000000000001,"15min":1.399},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":47.926000000000002,"5min":91.212999999999994,"15min":91.212999999999994},"last":0.307},{"interface":"front","average":{"1min":4.8319999999999999,"5min":3.5539999999999998,"15min":1.5680000000000001},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":47.972000000000001,"5min":91.254000000000005,"15min":91.254000000000005},"last":0.35099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:08:21 2023","interfaces":[{"interface":"back","average":{"1min":8.907,"5min":3.0259999999999998,"15min":1.3939999999999999},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":61.310000000000002,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.29699999999999999},{"interface":"front","average":{"1min":8.7479999999999993,"5min":2.7949999999999999,"15min":1.2909999999999999},"min":{"1min":0.33100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":61.154000000000003,"5min":61.154000000000003,"15min":61.154000000000003},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":3.4319999999999999,"15min":1.534},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.8650000000000002,"5min":91.834999999999994,"15min":91.834999999999994},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.5569999999999999,"5min":3.4289999999999998,"15min":1.5469999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.3979999999999997,"5min":91.802000000000007,"15min":91.802000000000007},"last":4.9809999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":0.88600000000000001,"15min":0.61099999999999999},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.253},"max":{"1min":4.0069999999999997,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":4.2030000000000003},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.0149999999999999,"15min":0.67800000000000005},"min":{"1min":0.30299999999999999,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":5.29,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":4.2869999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.89100000000000001,"15min":0.63900000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2910000000000004,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88300000000000001,"15min":0.65600000000000003},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":3.052,"5min":4.3099999999999996,"15min":4.3099999999999996},"last":0.81399999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.109,"15min":0.754},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.4930000000000003,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":5.0179999999999998},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.1859999999999999,"15min":0.79800000000000004},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.34599999999999997},"max":{"1min":8.3740000000000006,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":0.78400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":6098944,"data_stored":6094872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":281608192,"data_stored":281608192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":193462272,"data_stored":193462272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":34148352,"data_stored":34144280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":133758976,"data_stored":133758976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":105512960,"data_stored":105512960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:09:11.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:09:11.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:11.468 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:11.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:11.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:11.533 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:11.621 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:11.653 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:11.653 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:11.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:11.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:11.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:11.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:11.887 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:11.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:11.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:12.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:12.061 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:09:12.074 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:12.087 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:09:12.112 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:12.122 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:09:12.150 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:09:12.182 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:09:12.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:12.206 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:09:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:12.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:12.234 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:09:12.256 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.257 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:09:12.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:12.348 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:12 smithi181 ceph-mon[196979]: pgmap v428: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 0 B/s, 119 objects/s recovering 2023-10-09T04:09:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3111817117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 743/639 objects degraded (116.275%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3056560620' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:09:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:12 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:09:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[221067]: pgmap v428: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 0 B/s, 119 objects/s recovering 2023-10-09T04:09:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3111817117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 743/639 objects degraded (116.275%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3056560620' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:09:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:09:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[213841]: pgmap v428: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 12 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 743/639 objects degraded (116.275%); 743/639 objects misplaced (116.275%); 0 B/s, 119 objects/s recovering 2023-10-09T04:09:12.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3111817117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:12.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 743/639 objects degraded (116.275%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:09:12.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3056560620' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:09:12.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:12 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:09:13.061 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:09:13.061 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:09:13.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:13.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:13.224 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_0' removed 2023-10-09T04:09:13.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:13.345 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:13.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:13.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:13.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:13.525 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:13.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:09:14.091 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:14 smithi181 ceph-mon[196979]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:14 smithi181 ceph-mon[196979]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:09:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:14 smithi181 ceph-mon[196979]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:09:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:14 smithi181 ceph-mon[196979]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:14 smithi181 ceph-mon[196979]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[221067]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[221067]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:09:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[221067]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:09:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[221067]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[221067]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[213841]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[213841]: failed to encode map e165 with expected crc 2023-10-09T04:09:14.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:09:14.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[213841]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:09:14.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[213841]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:14.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:14 smithi163 ceph-mon[213841]: failed to encode map e165 with expected crc 2023-10-09T04:09:15.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:15.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:15.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:15.338 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:15.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:15 smithi181 ceph-mon[196979]: failed to encode map e165 with expected crc 2023-10-09T04:09:15.342 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:15 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 743/639 objects degraded (116.275%), 1 pg degraded) 2023-10-09T04:09:15.536 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:09:15.536 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:09:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:15 smithi163 ceph-mon[221067]: failed to encode map e165 with expected crc 2023-10-09T04:09:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:15 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 743/639 objects degraded (116.275%), 1 pg degraded) 2023-10-09T04:09:15.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:15 smithi163 ceph-mon[213841]: failed to encode map e165 with expected crc 2023-10-09T04:09:15.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:15 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 743/639 objects degraded (116.275%), 1 pg degraded) 2023-10-09T04:09:15.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:15.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:15.832 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:09:15.833 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:09:15.833 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T04:09:15.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T04:09:15.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:15.942 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:16.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:16.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:16.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:16.172 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:16.240 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:09:16.266 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:16.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:16 smithi163 ceph-mon[221067]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:16.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:16 smithi163 ceph-mon[213841]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:16.353 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:09:16.355 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:16.355 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:09:16.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:16.397 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:16.446 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:16.450 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:16 smithi181 ceph-mon[196979]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:16.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:16.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:16.580 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:16.621 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:09:16.624 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:16.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:16.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:16.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:16.754 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:16.850 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:16.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:16.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:16.982 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:17.030 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:17.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:17.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:17.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:17.160 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:17.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:17.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:17.222 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":431,"stamp":"2023-10-09T04:09:15.404021+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":296,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380180,"kb_used_data":550996,"kb_used_omap":126,"kb_used_meta":1828993,"kb_avail":747581036,"statfs":{"total":767960285184,"available":765522980864,"internally_reserved":0,"allocated":564219904,"data_stored":559763265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1872889467},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":40,"apply_latency_ms":40,"commit_latency_ns":40000000,"apply_latency_ns":40000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-54394904,"num_objects":-831,"num_object_clones":0,"num_object_copies":-2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-743,"num_objects_misplaced":-743,"num_objects_unfound":0,"num_objects_dirty":-831,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-158262,"num_write_kb":-3895297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4303,"num_bytes_recovered":-95485952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002555"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":142,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511456+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:09:03.511456+0000","last_peered":"2023-10-09T04:09:03.511456+0000","last_clean":"2023-10-09T04:09:03.511456+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:09:03.511456+0000","last_undegraded":"2023-10-09T04:09:03.511456+0000","last_fullsized":"2023-10-09T04:09:03.511456+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":98,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699828+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:08:56.699828+0000","last_peered":"2023-10-09T04:08:56.699828+0000","last_clean":"2023-10-09T04:08:56.699828+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:08:56.699828+0000","last_undegraded":"2023-10-09T04:08:56.699828+0000","last_fullsized":"2023-10-09T04:08:56.699828+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":113,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.698903+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:08:56.698903+0000","last_peered":"2023-10-09T04:08:56.698903+0000","last_clean":"2023-10-09T04:08:56.698903+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:08:56.698903+0000","last_undegraded":"2023-10-09T04:08:56.698903+0000","last_fullsized":"2023-10-09T04:08:56.698903+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":118,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709471+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:08:56.709471+0000","last_peered":"2023-10-09T04:08:56.709471+0000","last_clean":"2023-10-09T04:08:56.709471+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:08:56.709471+0000","last_undegraded":"2023-10-09T04:08:56.709471+0000","last_fullsized":"2023-10-09T04:08:56.709471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":117,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.236496+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:09:03.236496+0000","last_peered":"2023-10-09T04:09:03.236496+0000","last_clean":"2023-10-09T04:09:03.236496+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:09:03.236496+0000","last_undegraded":"2023-10-09T04:09:03.236496+0000","last_fullsized":"2023-10-09T04:09:03.236496+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699824+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:08:56.699824+0000","last_peered":"2023-10-09T04:08:56.699824+0000","last_clean":"2023-10-09T04:08:56.699824+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:08:56.699824+0000","last_undegraded":"2023-10-09T04:08:56.699824+0000","last_fullsized":"2023-10-09T04:08:56.699824+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":117,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709838+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:08:56.709838+0000","last_peered":"2023-10-09T04:08:56.709838+0000","last_clean":"2023-10-09T04:08:56.709838+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:08:56.709838+0000","last_undegraded":"2023-10-09T04:08:56.709838+0000","last_fullsized":"2023-10-09T04:08:56.709838+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":141,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.703181+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:08:56.703181+0000","last_peered":"2023-10-09T04:08:56.703181+0000","last_clean":"2023-10-09T04:08:56.703181+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:08:56.703181+0000","last_undegraded":"2023-10-09T04:08:56.703181+0000","last_fullsized":"2023-10-09T04:08:56.703181+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699923+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:08:56.699923+0000","last_peered":"2023-10-09T04:08:56.699923+0000","last_clean":"2023-10-09T04:08:56.699923+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:08:56.699923+0000","last_undegraded":"2023-10-09T04:08:56.699923+0000","last_fullsized":"2023-10-09T04:08:56.699923+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696257+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:08:56.696257+0000","last_peered":"2023-10-09T04:08:56.696257+0000","last_clean":"2023-10-09T04:08:56.696257+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:08:56.696257+0000","last_undegraded":"2023-10-09T04:08:56.696257+0000","last_fullsized":"2023-10-09T04:08:56.696257+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":126,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.697212+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:08:56.697212+0000","last_peered":"2023-10-09T04:08:56.697212+0000","last_clean":"2023-10-09T04:08:56.697212+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:08:56.697212+0000","last_undegraded":"2023-10-09T04:08:56.697212+0000","last_fullsized":"2023-10-09T04:08:56.697212+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.698992+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:08:56.698992+0000","last_peered":"2023-10-09T04:08:56.698992+0000","last_clean":"2023-10-09T04:08:56.698992+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:08:56.698992+0000","last_undegraded":"2023-10-09T04:08:56.698992+0000","last_fullsized":"2023-10-09T04:08:56.698992+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":129,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511701+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:09:03.511701+0000","last_peered":"2023-10-09T04:09:03.511701+0000","last_clean":"2023-10-09T04:09:03.511701+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:09:03.511701+0000","last_undegraded":"2023-10-09T04:09:03.511701+0000","last_fullsized":"2023-10-09T04:09:03.511701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511555+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:09:03.511555+0000","last_peered":"2023-10-09T04:09:03.511555+0000","last_clean":"2023-10-09T04:09:03.511555+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:09:03.511555+0000","last_undegraded":"2023-10-09T04:09:03.511555+0000","last_fullsized":"2023-10-09T04:09:03.511555+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696947+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:08:56.696947+0000","last_peered":"2023-10-09T04:08:56.696947+0000","last_clean":"2023-10-09T04:08:56.696947+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:08:56.696947+0000","last_undegraded":"2023-10-09T04:08:56.696947+0000","last_fullsized":"2023-10-09T04:08:56.696947+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":99,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.668089+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:09:03.668089+0000","last_peered":"2023-10-09T04:09:03.668089+0000","last_clean":"2023-10-09T04:09:03.668089+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:09:03.668089+0000","last_undegraded":"2023-10-09T04:09:03.668089+0000","last_fullsized":"2023-10-09T04:09:03.668089+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.042581+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:09:02.042581+0000","last_peered":"2023-10-09T04:09:02.042581+0000","last_clean":"2023-10-09T04:09:02.042581+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:09:02.042581+0000","last_undegraded":"2023-10-09T04:09:02.042581+0000","last_fullsized":"2023-10-09T04:09:02.042581+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700189+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:08:56.700189+0000","last_peered":"2023-10-09T04:08:56.700189+0000","last_clean":"2023-10-09T04:08:56.700189+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:08:56.700189+0000","last_undegraded":"2023-10-09T04:08:56.700189+0000","last_fullsized":"2023-10-09T04:08:56.700189+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696188+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:08:56.696188+0000","last_peered":"2023-10-09T04:08:56.696188+0000","last_clean":"2023-10-09T04:08:56.696188+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:08:56.696188+0000","last_undegraded":"2023-10-09T04:08:56.696188+0000","last_fullsized":"2023-10-09T04:08:56.696188+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":117,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709400+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:08:56.709400+0000","last_peered":"2023-10-09T04:08:56.709400+0000","last_clean":"2023-10-09T04:08:56.709400+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:08:56.709400+0000","last_undegraded":"2023-10-09T04:08:56.709400+0000","last_fullsized":"2023-10-09T04:08:56.709400+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":53,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.705340+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:08:56.705340+0000","last_peered":"2023-10-09T04:08:56.705340+0000","last_clean":"2023-10-09T04:08:56.705340+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:08:56.705340+0000","last_undegraded":"2023-10-09T04:08:56.705340+0000","last_fullsized":"2023-10-09T04:08:56.705340+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.702384+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:08:56.702384+0000","last_peered":"2023-10-09T04:08:56.702384+0000","last_clean":"2023-10-09T04:08:56.702384+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:08:56.702384+0000","last_undegraded":"2023-10-09T04:08:56.702384+0000","last_fullsized":"2023-10-09T04:08:56.702384+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":124,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.697151+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:08:56.697151+0000","last_peered":"2023-10-09T04:08:56.697151+0000","last_clean":"2023-10-09T04:08:56.697151+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:08:56.697151+0000","last_undegraded":"2023-10-09T04:08:56.697151+0000","last_fullsized":"2023-10-09T04:08:56.697151+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699450+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:08:56.699450+0000","last_peered":"2023-10-09T04:08:56.699450+0000","last_clean":"2023-10-09T04:08:56.699450+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:08:56.699450+0000","last_undegraded":"2023-10-09T04:08:56.699450+0000","last_fullsized":"2023-10-09T04:08:56.699450+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699607+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:08:56.699607+0000","last_peered":"2023-10-09T04:08:56.699607+0000","last_clean":"2023-10-09T04:08:56.699607+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:08:56.699607+0000","last_undegraded":"2023-10-09T04:08:56.699607+0000","last_fullsized":"2023-10-09T04:08:56.699607+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511430+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:09:03.511430+0000","last_peered":"2023-10-09T04:09:03.511430+0000","last_clean":"2023-10-09T04:09:03.511430+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:09:03.511430+0000","last_undegraded":"2023-10-09T04:09:03.511430+0000","last_fullsized":"2023-10-09T04:09:03.511430+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":107,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696969+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:08:56.696969+0000","last_peered":"2023-10-09T04:08:56.696969+0000","last_clean":"2023-10-09T04:08:56.696969+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:08:56.696969+0000","last_undegraded":"2023-10-09T04:08:56.696969+0000","last_fullsized":"2023-10-09T04:08:56.696969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":82,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.667874+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:09:03.667874+0000","last_peered":"2023-10-09T04:09:03.667874+0000","last_clean":"2023-10-09T04:09:03.667874+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:09:03.667874+0000","last_undegraded":"2023-10-09T04:09:03.667874+0000","last_fullsized":"2023-10-09T04:09:03.667874+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.701812+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:08:56.701812+0000","last_peered":"2023-10-09T04:08:56.701812+0000","last_clean":"2023-10-09T04:08:56.701812+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:08:56.701812+0000","last_undegraded":"2023-10-09T04:08:56.701812+0000","last_fullsized":"2023-10-09T04:08:56.701812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":118,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709476+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:08:56.709476+0000","last_peered":"2023-10-09T04:08:56.709476+0000","last_clean":"2023-10-09T04:08:56.709476+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:08:56.709476+0000","last_undegraded":"2023-10-09T04:08:56.709476+0000","last_fullsized":"2023-10-09T04:08:56.709476+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":67,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:02.042607+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:09:02.042607+0000","last_peered":"2023-10-09T04:09:02.042607+0000","last_clean":"2023-10-09T04:09:02.042607+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:09:02.042607+0000","last_undegraded":"2023-10-09T04:09:02.042607+0000","last_fullsized":"2023-10-09T04:09:02.042607+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511612+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:09:03.511612+0000","last_peered":"2023-10-09T04:09:03.511612+0000","last_clean":"2023-10-09T04:09:03.511612+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:09:03.511612+0000","last_undegraded":"2023-10-09T04:09:03.511612+0000","last_fullsized":"2023-10-09T04:09:03.511612+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511563+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:09:03.511563+0000","last_peered":"2023-10-09T04:09:03.511563+0000","last_clean":"2023-10-09T04:09:03.511563+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:09:03.511563+0000","last_undegraded":"2023-10-09T04:09:03.511563+0000","last_fullsized":"2023-10-09T04:09:03.511563+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511644+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:09:03.511644+0000","last_peered":"2023-10-09T04:09:03.511644+0000","last_clean":"2023-10-09T04:09:03.511644+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:09:03.511644+0000","last_undegraded":"2023-10-09T04:09:03.511644+0000","last_fullsized":"2023-10-09T04:09:03.511644+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511325+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:09:03.511325+0000","last_peered":"2023-10-09T04:09:03.511325+0000","last_clean":"2023-10-09T04:09:03.511325+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:09:03.511325+0000","last_undegraded":"2023-10-09T04:09:03.511325+0000","last_fullsized":"2023-10-09T04:09:03.511325+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511862+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:09:03.511862+0000","last_peered":"2023-10-09T04:09:03.511862+0000","last_clean":"2023-10-09T04:09:03.511862+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:09:03.511862+0000","last_undegraded":"2023-10-09T04:09:03.511862+0000","last_fullsized":"2023-10-09T04:09:03.511862+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.703021+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:08:56.703021+0000","last_peered":"2023-10-09T04:08:56.703021+0000","last_clean":"2023-10-09T04:08:56.703021+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:08:56.703021+0000","last_undegraded":"2023-10-09T04:08:56.703021+0000","last_fullsized":"2023-10-09T04:08:56.703021+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.698809+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:08:56.698809+0000","last_peered":"2023-10-09T04:08:56.698809+0000","last_clean":"2023-10-09T04:08:56.698809+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:08:56.698809+0000","last_undegraded":"2023-10-09T04:08:56.698809+0000","last_fullsized":"2023-10-09T04:08:56.698809+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":138,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700492+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:08:56.700492+0000","last_peered":"2023-10-09T04:08:56.700492+0000","last_clean":"2023-10-09T04:08:56.700492+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:08:56.700492+0000","last_undegraded":"2023-10-09T04:08:56.700492+0000","last_fullsized":"2023-10-09T04:08:56.700492+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700292+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:08:56.700292+0000","last_peered":"2023-10-09T04:08:56.700292+0000","last_clean":"2023-10-09T04:08:56.700292+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:08:56.700292+0000","last_undegraded":"2023-10-09T04:08:56.700292+0000","last_fullsized":"2023-10-09T04:08:56.700292+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":100,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699070+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:08:56.699070+0000","last_peered":"2023-10-09T04:08:56.699070+0000","last_clean":"2023-10-09T04:08:56.699070+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:08:56.699070+0000","last_undegraded":"2023-10-09T04:08:56.699070+0000","last_fullsized":"2023-10-09T04:08:56.699070+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700477+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:08:56.700477+0000","last_peered":"2023-10-09T04:08:56.700477+0000","last_clean":"2023-10-09T04:08:56.700477+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:08:56.700477+0000","last_undegraded":"2023-10-09T04:08:56.700477+0000","last_fullsized":"2023-10-09T04:08:56.700477+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511539+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:09:03.511539+0000","last_peered":"2023-10-09T04:09:03.511539+0000","last_clean":"2023-10-09T04:09:03.511539+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:09:03.511539+0000","last_undegraded":"2023-10-09T04:09:03.511539+0000","last_fullsized":"2023-10-09T04:09:03.511539+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":130,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700428+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:08:56.700428+0000","last_peered":"2023-10-09T04:08:56.700428+0000","last_clean":"2023-10-09T04:08:56.700428+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:08:56.700428+0000","last_undegraded":"2023-10-09T04:08:56.700428+0000","last_fullsized":"2023-10-09T04:08:56.700428+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":148,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.228518+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:09:03.228518+0000","last_peered":"2023-10-09T04:09:03.228518+0000","last_clean":"2023-10-09T04:09:03.228518+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:09:03.228518+0000","last_undegraded":"2023-10-09T04:09:03.228518+0000","last_fullsized":"2023-10-09T04:09:03.228518+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699644+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:08:56.699644+0000","last_peered":"2023-10-09T04:08:56.699644+0000","last_clean":"2023-10-09T04:08:56.699644+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:08:56.699644+0000","last_undegraded":"2023-10-09T04:08:56.699644+0000","last_fullsized":"2023-10-09T04:08:56.699644+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":133,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.593331+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:09:03.593331+0000","last_peered":"2023-10-09T04:09:03.593331+0000","last_clean":"2023-10-09T04:09:03.593331+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:09:03.593331+0000","last_undegraded":"2023-10-09T04:09:03.593331+0000","last_fullsized":"2023-10-09T04:09:03.593331+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.699726+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:08:56.699726+0000","last_peered":"2023-10-09T04:08:56.699726+0000","last_clean":"2023-10-09T04:08:56.699726+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:08:56.699726+0000","last_undegraded":"2023-10-09T04:08:56.699726+0000","last_fullsized":"2023-10-09T04:08:56.699726+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":504,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696189+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:08:56.696189+0000","last_peered":"2023-10-09T04:08:56.696189+0000","last_clean":"2023-10-09T04:08:56.696189+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:08:56.696189+0000","last_undegraded":"2023-10-09T04:08:56.696189+0000","last_fullsized":"2023-10-09T04:08:56.696189+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":60,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700438+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:08:56.700438+0000","last_peered":"2023-10-09T04:08:56.700438+0000","last_clean":"2023-10-09T04:08:56.700438+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:08:56.700438+0000","last_undegraded":"2023-10-09T04:08:56.700438+0000","last_fullsized":"2023-10-09T04:08:56.700438+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511735+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:09:03.511735+0000","last_peered":"2023-10-09T04:09:03.511735+0000","last_clean":"2023-10-09T04:09:03.511735+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:09:03.511735+0000","last_undegraded":"2023-10-09T04:09:03.511735+0000","last_fullsized":"2023-10-09T04:09:03.511735+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696091+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:08:56.696091+0000","last_peered":"2023-10-09T04:08:56.696091+0000","last_clean":"2023-10-09T04:08:56.696091+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:08:56.696091+0000","last_undegraded":"2023-10-09T04:08:56.696091+0000","last_fullsized":"2023-10-09T04:08:56.696091+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":118,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.710454+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:08:56.710454+0000","last_peered":"2023-10-09T04:08:56.710454+0000","last_clean":"2023-10-09T04:08:56.710454+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:08:56.710454+0000","last_undegraded":"2023-10-09T04:08:56.710454+0000","last_fullsized":"2023-10-09T04:08:56.710454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709623+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:08:56.709623+0000","last_peered":"2023-10-09T04:08:56.709623+0000","last_clean":"2023-10-09T04:08:56.709623+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:08:56.709623+0000","last_undegraded":"2023-10-09T04:08:56.709623+0000","last_fullsized":"2023-10-09T04:08:56.709623+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.697044+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:08:56.697044+0000","last_peered":"2023-10-09T04:08:56.697044+0000","last_clean":"2023-10-09T04:08:56.697044+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:08:56.697044+0000","last_undegraded":"2023-10-09T04:08:56.697044+0000","last_fullsized":"2023-10-09T04:08:56.697044+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700567+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:08:56.700567+0000","last_peered":"2023-10-09T04:08:56.700567+0000","last_clean":"2023-10-09T04:08:56.700567+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:08:56.700567+0000","last_undegraded":"2023-10-09T04:08:56.700567+0000","last_fullsized":"2023-10-09T04:08:56.700567+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":103,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709302+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:08:56.709302+0000","last_peered":"2023-10-09T04:08:56.709302+0000","last_clean":"2023-10-09T04:08:56.709302+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:08:56.709302+0000","last_undegraded":"2023-10-09T04:08:56.709302+0000","last_fullsized":"2023-10-09T04:08:56.709302+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":68,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696917+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:08:56.696917+0000","last_peered":"2023-10-09T04:08:56.696917+0000","last_clean":"2023-10-09T04:08:56.696917+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:08:56.696917+0000","last_undegraded":"2023-10-09T04:08:56.696917+0000","last_fullsized":"2023-10-09T04:08:56.696917+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":99,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.593302+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:09:03.593302+0000","last_peered":"2023-10-09T04:09:03.593302+0000","last_clean":"2023-10-09T04:09:03.593302+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:09:03.593302+0000","last_undegraded":"2023-10-09T04:09:03.593302+0000","last_fullsized":"2023-10-09T04:09:03.593302+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":94,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.696363+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:08:56.696363+0000","last_peered":"2023-10-09T04:08:56.696363+0000","last_clean":"2023-10-09T04:08:56.696363+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:08:56.696363+0000","last_undegraded":"2023-10-09T04:08:56.696363+0000","last_fullsized":"2023-10-09T04:08:56.696363+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":92,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.697048+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:08:56.697048+0000","last_peered":"2023-10-09T04:08:56.697048+0000","last_clean":"2023-10-09T04:08:56.697048+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:08:56.697048+0000","last_undegraded":"2023-10-09T04:08:56.697048+0000","last_fullsized":"2023-10-09T04:08:56.697048+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709552+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:08:56.709552+0000","last_peered":"2023-10-09T04:08:56.709552+0000","last_clean":"2023-10-09T04:08:56.709552+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:08:56.709552+0000","last_undegraded":"2023-10-09T04:08:56.709552+0000","last_fullsized":"2023-10-09T04:08:56.709552+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511328+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:09:03.511328+0000","last_peered":"2023-10-09T04:09:03.511328+0000","last_clean":"2023-10-09T04:09:03.511328+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:09:03.511328+0000","last_undegraded":"2023-10-09T04:09:03.511328+0000","last_fullsized":"2023-10-09T04:09:03.511328+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":106,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709553+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:08:56.709553+0000","last_peered":"2023-10-09T04:08:56.709553+0000","last_clean":"2023-10-09T04:08:56.709553+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:08:56.709553+0000","last_undegraded":"2023-10-09T04:08:56.709553+0000","last_fullsized":"2023-10-09T04:08:56.709553+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.227285+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:09:03.227285+0000","last_peered":"2023-10-09T04:09:03.227285+0000","last_clean":"2023-10-09T04:09:03.227285+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:09:03.227285+0000","last_undegraded":"2023-10-09T04:09:03.227285+0000","last_fullsized":"2023-10-09T04:09:03.227285+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":125,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.701940+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:08:56.701940+0000","last_peered":"2023-10-09T04:08:56.701940+0000","last_clean":"2023-10-09T04:08:56.701940+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:08:56.701940+0000","last_undegraded":"2023-10-09T04:08:56.701940+0000","last_fullsized":"2023-10-09T04:08:56.701940+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511625+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:09:03.511625+0000","last_peered":"2023-10-09T04:09:03.511625+0000","last_clean":"2023-10-09T04:09:03.511625+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:09:03.511625+0000","last_undegraded":"2023-10-09T04:09:03.511625+0000","last_fullsized":"2023-10-09T04:09:03.511625+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":108,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.700366+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:08:56.700366+0000","last_peered":"2023-10-09T04:08:56.700366+0000","last_clean":"2023-10-09T04:08:56.700366+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:08:56.700366+0000","last_undegraded":"2023-10-09T04:08:56.700366+0000","last_fullsized":"2023-10-09T04:08:56.700366+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511562+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:09:03.511562+0000","last_peered":"2023-10-09T04:09:03.511562+0000","last_clean":"2023-10-09T04:09:03.511562+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:09:03.511562+0000","last_undegraded":"2023-10-09T04:09:03.511562+0000","last_fullsized":"2023-10-09T04:09:03.511562+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":44,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.227614+0000","last_change":"2023-10-09T04:06:57.499785+0000","last_active":"2023-10-09T04:09:03.227614+0000","last_peered":"2023-10-09T04:09:03.227614+0000","last_clean":"2023-10-09T04:09:03.227614+0000","last_became_active":"2023-10-09T04:06:57.499419+0000","last_became_peered":"2023-10-09T04:06:57.499419+0000","last_unstale":"2023-10-09T04:09:03.227614+0000","last_undegraded":"2023-10-09T04:09:03.227614+0000","last_fullsized":"2023-10-09T04:09:03.227614+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:14.270664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":121,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:09.830208+0000","last_change":"2023-10-09T04:06:45.454363+0000","last_active":"2023-10-09T04:09:09.830208+0000","last_peered":"2023-10-09T04:09:09.830208+0000","last_clean":"2023-10-09T04:09:09.830208+0000","last_became_active":"2023-10-09T04:06:45.454065+0000","last_became_peered":"2023-10-09T04:06:45.454065+0000","last_unstale":"2023-10-09T04:09:09.830208+0000","last_undegraded":"2023-10-09T04:09:09.830208+0000","last_fullsized":"2023-10-09T04:09:09.830208+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:21.580027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-10-09T04:08:56.709768+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:08:56.709768+0000","last_peered":"2023-10-09T04:08:56.709768+0000","last_clean":"2023-10-09T04:08:56.709768+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:08:56.709768+0000","last_undegraded":"2023-10-09T04:08:56.709768+0000","last_fullsized":"2023-10-09T04:08:56.709768+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":123,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-10-09T04:09:03.511310+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:09:03.511310+0000","last_peered":"2023-10-09T04:09:03.511310+0000","last_clean":"2023-10-09T04:09:03.511310+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:09:03.511310+0000","last_undegraded":"2023-10-09T04:09:03.511310+0000","last_fullsized":"2023-10-09T04:09:03.511310+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201480,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":373768,"kb_used_data":72384,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93371384,"statfs":{"total":95995035648,"available":95612297216,"internally_reserved":0,"allocated":74121216,"data_stored":73615530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":1.5589999999999999,"15min":0.86299999999999999},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":17.398,"5min":17.398,"15min":17.398},"last":0.254},{"interface":"front","average":{"1min":3.173,"5min":1.4510000000000001,"15min":0.82899999999999996},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.306999999999999,"5min":17.306999999999999,"15min":17.306999999999999},"last":0.28699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":3.2320000000000002,"5min":1.752,"15min":1.0029999999999999},"min":{"1min":0.26200000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.051,"5min":10.051,"15min":10.051},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.2080000000000002,"5min":1.6220000000000001,"15min":1.0089999999999999},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.7680000000000007,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.5990000000000002,"5min":2.246,"15min":1.1140000000000001},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":13.887,"5min":58.615000000000002,"15min":58.615000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.839,"5min":3.3820000000000001,"15min":1.5309999999999999},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.872,"5min":99.372,"15min":99.372},"last":0.33600000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":3.218,"15min":1.4590000000000001},"min":{"1min":0.33400000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.124000000000001,"5min":99.134,"15min":99.134},"last":0.69699999999999995},{"interface":"front","average":{"1min":2.6480000000000001,"5min":3.367,"15min":1.5329999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.140000000000001,"5min":99.117999999999995,"15min":99.117999999999995},"last":0.71099999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.80300000000000005,"15min":0.61499999999999999},"min":{"1min":0.27500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.0199999999999996,"5min":6.0199999999999996,"15min":6.3129999999999997},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":0.79900000000000004,"15min":0.621},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":5.3630000000000004,"5min":6.5590000000000002,"15min":6.5590000000000002},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.73999999999999999,"15min":0.58799999999999997},"min":{"1min":0.27700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.867,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.236,"5min":0.875,"15min":0.66100000000000003},"min":{"1min":0.30399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.6779999999999999,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":0.877,"15min":0.65300000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.3559999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.48,"5min":0.94899999999999995,"15min":0.70699999999999996},"min":{"1min":0.35099999999999998,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":5.6420000000000003,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.46700000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463118,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":412804,"kb_used_data":115516,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93332348,"statfs":{"total":95995035648,"available":95572324352,"internally_reserved":0,"allocated":118288384,"data_stored":117794455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:14 2023","interfaces":[{"interface":"back","average":{"1min":3.3279999999999998,"5min":2.2810000000000001,"15min":1.117},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":35.393000000000001,"5min":78.983999999999995,"15min":78.983999999999995},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.8010000000000002,"5min":2.9089999999999998,"15min":1.327},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":35.240000000000002,"5min":78.998000000000005,"15min":78.998000000000005},"last":0.39200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":5.399,"5min":3.0640000000000001,"15min":1.369},"min":{"1min":0.30599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":57.094000000000001,"5min":57.094000000000001,"15min":57.094000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":5.8369999999999997,"5min":3.1829999999999998,"15min":1.399},"min":{"1min":0.27000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":57.003,"5min":57.003,"15min":57.003},"last":0.371}]},{"osd":2,"last update":"Mon Oct 9 04:08:14 2023","interfaces":[{"interface":"back","average":{"1min":2.492,"5min":2.2200000000000002,"15min":1.113},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":12.795999999999999,"5min":42.195999999999998,"15min":42.195999999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.528,"5min":2.2650000000000001,"15min":1.163},"min":{"1min":0.439,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":12.827,"5min":42.165999999999997,"15min":42.165999999999997},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":8.8030000000000008,"5min":3.7559999999999998,"15min":1.6619999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":57.509999999999998,"5min":57.509999999999998,"15min":57.509999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":9.3079999999999998,"5min":4.0510000000000002,"15min":1.7869999999999999},"min":{"1min":0.41899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":57.453000000000003,"5min":57.453000000000003,"15min":57.453000000000003},"last":1.0800000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.52500000000000002,"15min":0.48399999999999999},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.72999999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.85599999999999998,"15min":0.62},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":1.7609999999999999,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.28999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.226,"15min":0.752},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.734,"5min":54.773000000000003,"15min":54.773000000000003},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.3999999999999999,"15min":0.84899999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8220000000000001,"5min":54.853999999999999,"15min":54.853999999999999},"last":0.34699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:09:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.68100000000000005,"15min":0.60699999999999998},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":3.0630000000000002,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.68200000000000005},"min":{"1min":0.36699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.8730000000000002,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.443}]}]},{"osd":1,"up_from":89,"seq":382252089426,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":119376,"kb_used_data":1672,"kb_used_omap":9,"kb_used_meta":117686,"kb_avail":93625776,"statfs":{"total":95995035648,"available":95872794624,"internally_reserved":0,"allocated":1712128,"data_stored":1082738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":120511465},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":2.258,"15min":1},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0389999999999997,"5min":45.811999999999998,"15min":45.811999999999998},"last":0.20699999999999999},{"interface":"front","average":{"1min":2.7490000000000001,"5min":3.5089999999999999,"15min":1.431},"min":{"1min":0.24099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.4820000000000002,"5min":99.831999999999994,"15min":99.831999999999994},"last":0.27600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:08:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.4670000000000001,"15min":1.0509999999999999},"min":{"1min":0.20200000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":18.143000000000001,"5min":121.54600000000001,"15min":121.54600000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":3.0030000000000001,"5min":2.7410000000000001,"15min":1.1579999999999999},"min":{"1min":0.26600000000000001,"5min":0.186,"15min":0.186},"max":{"1min":8.6669999999999998,"5min":99.899000000000001,"15min":99.899000000000001},"last":0.30499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":2.8119999999999998,"15min":1.1879999999999999},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.56,"5min":42.588999999999999,"15min":42.588999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":3.8330000000000002,"5min":4.0110000000000001,"15min":1.5920000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.163,"5min":100.02500000000001,"15min":100.02500000000001},"last":0.32300000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.581,"15min":1.3220000000000001},"min":{"1min":0.33600000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.5529999999999999,"5min":99.989999999999995,"15min":99.989999999999995},"last":0.441},{"interface":"front","average":{"1min":1.768,"5min":2.5099999999999998,"15min":1.304},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8239999999999998,"5min":100.249,"15min":100.249},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":2.492,"15min":1.3129999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.5780000000000003,"5min":99.933999999999997,"15min":99.933999999999997},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.6960000000000002,"15min":1.3819999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.4779999999999998,"5min":100.34099999999999,"15min":100.34099999999999},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":3.0209999999999999,"15min":1.4830000000000001},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.968,"5min":100.309,"15min":100.309},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.044,"5min":2.8319999999999999,"15min":1.4379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.9089999999999998,"5min":100.143,"15min":100.143},"last":0.434}]},{"osd":7,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.8599999999999999,"15min":1.45},"min":{"1min":0.373,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.8490000000000002,"5min":100.214,"15min":100.214},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.7719999999999998,"5min":2.7280000000000002,"15min":1.4219999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":7.8869999999999996,"5min":100.113,"15min":100.113},"last":0.41599999999999998}]}]},{"osd":0,"up_from":84,"seq":360777252952,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":114268,"kb_used_data":2260,"kb_used_omap":28,"kb_used_meta":111971,"kb_avail":93630884,"statfs":{"total":95995035648,"available":95878025216,"internally_reserved":0,"allocated":2314240,"data_stored":1671010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":114659152},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:08:44 2023","interfaces":[{"interface":"back","average":{"1min":4.157,"5min":4.2679999999999998,"15min":1.6719999999999999},"min":{"1min":0.19800000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":44.287999999999997,"5min":118.48,"15min":118.48},"last":0.32400000000000001},{"interface":"front","average":{"1min":4.4889999999999999,"5min":4.5720000000000001,"15min":1.766},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":43.906999999999996,"5min":118.384,"15min":118.384},"last":0.27300000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:08:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.5680000000000001,"15min":1.468},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":6.1280000000000001,"5min":118.54000000000001,"15min":118.54000000000001},"last":0.311},{"interface":"front","average":{"1min":5.3570000000000002,"5min":4.3819999999999997,"15min":1.74},"min":{"1min":0.29299999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":44.436999999999998,"5min":118.727,"15min":118.727},"last":0.245}]},{"osd":3,"last update":"Mon Oct 9 04:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.6959999999999997,"5min":7.4530000000000003,"15min":2.7450000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":44.460000000000001,"5min":118.73999999999999,"15min":118.73999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":6.0570000000000004,"5min":7.4199999999999999,"15min":2.7679999999999998},"min":{"1min":0.34899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":43.972000000000001,"5min":118.584,"15min":118.584},"last":0.33800000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":3.141,"15min":1.4390000000000001},"min":{"1min":0.29699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.6589999999999998,"5min":118.633,"15min":118.633},"last":0.54200000000000004},{"interface":"front","average":{"1min":5.1150000000000002,"5min":3.9289999999999998,"15min":1.7150000000000001},"min":{"1min":0.47599999999999998,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":44.475000000000001,"5min":118.913,"15min":118.913},"last":0.72099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":5.556,"5min":5.383,"15min":2.21},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":44.494,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":4.3479999999999999,"5min":5.0449999999999999,"15min":2.1070000000000002},"min":{"1min":0.41999999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":44.328000000000003,"5min":182.66300000000001,"15min":182.66300000000001},"last":0.73399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":4.3680000000000003,"5min":3.0670000000000002,"15min":1.4630000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":44.384999999999998,"5min":118.864,"15min":118.864},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.5249999999999999,"5min":2.6099999999999999,"15min":1.3009999999999999},"min":{"1min":0.42399999999999999,"5min":0.317,"15min":0.317},"max":{"1min":6.6859999999999999,"5min":118.70399999999999,"15min":118.70399999999999},"last":0.70999999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":4.2629999999999999,"5min":5.4820000000000002,"15min":2.262},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":44.420000000000002,"5min":185.024,"15min":185.024},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.996,"5min":4.798,"15min":2.044},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6200000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.751}]}]},{"osd":2,"up_from":94,"seq":403726925900,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":415560,"kb_used_data":257536,"kb_used_omap":81,"kb_used_meta":157934,"kb_avail":93329592,"statfs":{"total":95995035648,"available":95569502208,"internally_reserved":0,"allocated":263716864,"data_stored":263100845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":161724634},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":21,"apply_latency_ms":21,"commit_latency_ns":21000000,"apply_latency_ns":21000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":3.7869999999999999,"5min":1.8240000000000001,"15min":0.83199999999999996},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":9.8719999999999999,"5min":9.8719999999999999,"15min":9.8719999999999999},"last":0.22},{"interface":"front","average":{"1min":5.2300000000000004,"5min":2.2090000000000001,"15min":0.997},"min":{"1min":0.33800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":25.131,"5min":25.131,"15min":25.131},"last":0.79900000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":6.9770000000000003,"5min":3.274,"15min":1.341},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.221},"max":{"1min":53.695,"5min":68.009,"15min":68.009},"last":0.83999999999999997},{"interface":"front","average":{"1min":6.4340000000000002,"5min":3.5649999999999999,"15min":1.4330000000000001},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":53.439,"5min":66.975999999999999,"15min":66.975999999999999},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.359,"15min":1.0409999999999999},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.77,"5min":27.562000000000001,"15min":27.562000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":5.4850000000000003,"5min":2.9830000000000001,"15min":1.2470000000000001},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":46.286999999999999,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.86699999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.7949999999999999,"15min":0.99399999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.9249999999999998,"5min":11.038,"15min":11.038},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.5,"15min":0.89700000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.1420000000000003,"5min":14.305,"15min":14.305},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":1.5569999999999999,"15min":0.95499999999999996},"min":{"1min":0.316,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":14.132999999999999,"5min":14.132999999999999,"15min":14.132999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":2.5099999999999998,"5min":1.8999999999999999,"15min":1.0629999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.883,"5min":11.087999999999999,"15min":11.087999999999999},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":1.425,"15min":0.89400000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.0979999999999999,"5min":5.0979999999999999,"15min":5.0979999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":3.073,"5min":1.595,"15min":0.98499999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":14.103999999999999,"5min":14.103999999999999,"15min":14.103999999999999},"last":0.95799999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:08:15 2023","interfaces":[{"interface":"back","average":{"1min":2.9849999999999999,"5min":1.829,"15min":1.056},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.8309999999999995,"5min":14.364000000000001,"15min":14.364000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.2730000000000001,"5min":1.8260000000000001,"15min":1.0820000000000001},"min":{"1min":0.39000000000000001,"5min":0.37,"15min":0.37},"max":{"1min":14.157,"5min":14.157,"15min":14.157},"last":0.92100000000000004}]}]},{"osd":3,"up_from":99,"seq":425201762373,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":338592,"kb_used_data":98584,"kb_used_omap":7,"kb_used_meta":239992,"kb_avail":93406560,"statfs":{"total":95995035648,"available":95648317440,"internally_reserved":0,"allocated":100950016,"data_stored":100362273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":245752424},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":4.181,"5min":5.6589999999999998,"15min":2.496},"min":{"1min":0.23200000000000001,"5min":0.185,"15min":0.185},"max":{"1min":13.166,"5min":106.22,"15min":106.22},"last":0.32300000000000001},{"interface":"front","average":{"1min":4.2809999999999997,"5min":5.6660000000000004,"15min":2.4969999999999999},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.722,"5min":106.986,"15min":106.986},"last":0.25900000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9990000000000001,"5min":3.8370000000000002,"15min":1.498},"min":{"1min":0.22,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.108000000000001,"5min":106.121,"15min":106.121},"last":0.31},{"interface":"front","average":{"1min":4.4619999999999997,"5min":3.319,"15min":1.3400000000000001},"min":{"1min":0.30499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":16.977,"5min":65.015000000000001,"15min":65.015000000000001},"last":0.38200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":3.726,"5min":4.4749999999999996,"15min":1.7370000000000001},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.205,"5min":106.105,"15min":106.105},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.3460000000000001,"5min":4.6449999999999996,"15min":1.7889999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2170000000000005,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.29999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0750000000000002,"5min":3.839,"15min":1.675},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":37.652000000000001,"5min":106.85899999999999,"15min":106.85899999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":4.9370000000000003,"5min":2.2290000000000001,"15min":1.1399999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":37.442,"5min":37.442,"15min":37.442},"last":0.36399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":1.964,"15min":1.05},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.8940000000000001,"5min":12.819000000000001,"15min":12.819000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.625,"5min":2.0139999999999998,"15min":1.4239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.228999999999999,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6099999999999999,"5min":3.2210000000000001,"15min":1.8149999999999999},"min":{"1min":0.35499999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.255000000000001,"5min":106.036,"15min":106.036},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.7410000000000001,"5min":3.319,"15min":1.516},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":15.622,"5min":106.741,"15min":106.741},"last":0.35599999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":3.492,"15min":1.583},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":15.685,"5min":106.82599999999999,"15min":106.82599999999999},"last":1.5760000000000001},{"interface":"front","average":{"1min":2.96,"5min":3.2330000000000001,"15min":1.478},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.8159999999999998,"5min":106.084,"15min":106.084},"last":0.41399999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986390,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307016,"kb_used_data":1536,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93438136,"statfs":{"total":95995035648,"available":95680651264,"internally_reserved":0,"allocated":1572864,"data_stored":1074429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:09:14 2023","interfaces":[{"interface":"back","average":{"1min":4.7850000000000001,"5min":2.2149999999999999,"15min":1.0900000000000001},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":63.128,"5min":63.128,"15min":63.128},"last":0.39800000000000002},{"interface":"front","average":{"1min":4.0149999999999997,"5min":1.9970000000000001,"15min":1.044},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":62.645000000000003,"5min":62.645000000000003,"15min":62.645000000000003},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":5.3109999999999999,"5min":2.3439999999999999,"15min":1.1479999999999999},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":79.075999999999993,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.58399999999999996},{"interface":"front","average":{"1min":5.7249999999999996,"5min":2.427,"15min":1.165},"min":{"1min":0.27000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":75.314999999999998,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.46300000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:08:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.706,"15min":0.95599999999999996},"min":{"1min":0.316,"5min":0.245,"15min":0.245},"max":{"1min":7.4420000000000002,"5min":19.356000000000002,"15min":19.356000000000002},"last":0.443},{"interface":"front","average":{"1min":2.5459999999999998,"5min":1.595,"15min":0.90600000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.46,"5min":19.324000000000002,"15min":19.324000000000002},"last":0.57299999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.7110000000000001,"15min":0.93100000000000005},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.492,"5min":14.694000000000001,"15min":14.694000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.379,"5min":1.716,"15min":0.94399999999999995},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.8840000000000003,"5min":9.0449999999999999,"15min":9.0449999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.75800000000000001,"15min":0.57299999999999995},"min":{"1min":0.27000000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.0870000000000002,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.048,"15min":0.69399999999999995},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9569999999999999,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.53200000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0589999999999999,"15min":0.72099999999999997},"min":{"1min":0.308,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.6849999999999996,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.623},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.83199999999999996,"15min":0.64300000000000002},"min":{"1min":0.38200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.60399999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:09:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.97599999999999998,"15min":0.69999999999999996},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5190000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.1000000000000001,"15min":0.76700000000000002},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.202,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.67800000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724753,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298796,"kb_used_data":1508,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446356,"statfs":{"total":95995035648,"available":95689068544,"internally_reserved":0,"allocated":1544192,"data_stored":1061985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:21 2023","interfaces":[{"interface":"back","average":{"1min":4.8920000000000003,"5min":2.2629999999999999,"15min":1.153},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":47.954000000000001,"5min":63.037999999999997,"15min":63.037999999999997},"last":0.68600000000000005},{"interface":"front","average":{"1min":4.8339999999999996,"5min":2.3999999999999999,"15min":1.179},"min":{"1min":0.23400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":52.009,"5min":62.927999999999997,"15min":62.927999999999997},"last":0.59199999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:08:38 2023","interfaces":[{"interface":"back","average":{"1min":4.593,"5min":3.1120000000000001,"15min":1.399},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":47.926000000000002,"5min":91.212999999999994,"15min":91.212999999999994},"last":0.623},{"interface":"front","average":{"1min":4.8319999999999999,"5min":3.5539999999999998,"15min":1.5680000000000001},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":47.972000000000001,"5min":91.254000000000005,"15min":91.254000000000005},"last":0.67200000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:08:21 2023","interfaces":[{"interface":"back","average":{"1min":8.907,"5min":3.0259999999999998,"15min":1.3939999999999999},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":61.310000000000002,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.70399999999999996},{"interface":"front","average":{"1min":8.7479999999999993,"5min":2.7949999999999999,"15min":1.2909999999999999},"min":{"1min":0.33100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":61.154000000000003,"5min":61.154000000000003,"15min":61.154000000000003},"last":0.60699999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":3.4319999999999999,"15min":1.534},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.8650000000000002,"5min":91.834999999999994,"15min":91.834999999999994},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.5569999999999999,"5min":3.4289999999999998,"15min":1.5469999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.3979999999999997,"5min":91.802000000000007,"15min":91.802000000000007},"last":0.65100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":0.88600000000000001,"15min":0.61099999999999999},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.253},"max":{"1min":4.0069999999999997,"5min":4.9539999999999997,"15min":4.9539999999999997},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.0149999999999999,"15min":0.67800000000000005},"min":{"1min":0.30299999999999999,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":5.29,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.64300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.89100000000000001,"15min":0.63900000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2910000000000004,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88300000000000001,"15min":0.65600000000000003},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":3.052,"5min":4.3099999999999996,"15min":4.3099999999999996},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.109,"15min":0.754},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.4930000000000003,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.1859999999999999,"15min":0.79800000000000004},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.34599999999999997},"max":{"1min":8.3740000000000006,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":0.746}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:09:17.228 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:09:17.228 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:09:17.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:17 smithi163 ceph-mon[213841]: from='client.44963 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:17.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2332452879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:17.240 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:09:17.305 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:17.332 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:17.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:17.486 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:17 smithi181 ceph-mon[196979]: from='client.44963 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:17.486 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2332452879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:17.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:17.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:17 smithi163 ceph-mon[221067]: from='client.44963 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:17.557 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2332452879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:17.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:17.616 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:17.800 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:09:18.019 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:09:18.019 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:09:18.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:18.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:18.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:18.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:18.496 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2332452879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:18 smithi181 ceph-mon[196979]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2332452879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[221067]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2332452879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:09:18.520 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:09:18.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:18.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:18 smithi163 ceph-mon[213841]: failed to encode map e166 with expected crc 2023-10-09T04:09:18.522 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:18.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:18.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:18.699 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:09:18.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:18.768 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:18.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:18.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:18.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:19.001 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:19.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:19.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:19.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:19.204 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:19.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:19.286 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:19.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:19.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:19.486 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:19.513 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:19.577 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_1' created 2023-10-09T04:09:19.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:19 smithi163 ceph-mon[213841]: failed to encode map e166 with expected crc 2023-10-09T04:09:19.650 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:19 smithi163 ceph-mon[221067]: failed to encode map e166 with expected crc 2023-10-09T04:09:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:19 smithi181 ceph-mon[196979]: failed to encode map e166 with expected crc 2023-10-09T04:09:20.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:20.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:20.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:09:20.403 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:20.439 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:20.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:20.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:20.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T04:09:20.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:20.612 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2985677999' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:09:20.612 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:09:20.613 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[221067]: failed to encode map e167 with expected crc 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2985677999' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:09:20.614 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:20 smithi163 ceph-mon[213841]: failed to encode map e167 with expected crc 2023-10-09T04:09:20.642 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:20.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:20.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2985677999' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:09:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:09:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:20.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:20.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:09:20.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:09:20.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:20 smithi181 ceph-mon[196979]: failed to encode map e167 with expected crc 2023-10-09T04:09:20.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:20.867 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:09:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:09:20] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.43.0" 2023-10-09T04:09:20.889 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:20.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:20.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:21.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:21.070 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:21.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:21.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:21.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:21.256 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:21.297 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:09:21.297 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:09:21.312 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:09:21.330 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:09:21.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:21.353 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:09:21.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:21.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:21.373 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:09:21.392 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:09:21.411 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.413 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:09:21.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:21.486 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:21.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:21.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[213841]: failed to encode map e167 with expected crc 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[213841]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[213841]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[213841]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[213841]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3269364379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[221067]: failed to encode map e167 with expected crc 2023-10-09T04:09:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[221067]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:09:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[221067]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[221067]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[221067]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3269364379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:21.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:21.664 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:21.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:21.711 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:21 smithi181 ceph-mon[196979]: failed to encode map e167 with expected crc 2023-10-09T04:09:21.711 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:21 smithi181 ceph-mon[196979]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:09:21.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:21 smithi181 ceph-mon[196979]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:21 smithi181 ceph-mon[196979]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:21 smithi181 ceph-mon[196979]: failed to encode map e168 with expected crc 2023-10-09T04:09:21.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3269364379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:21.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:21.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:21.842 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:22.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:22.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:22.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:22.134 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:22.148 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:22.176 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:22.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:22.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:22.310 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:22.324 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:22.363 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:22.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:22.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:22.424 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:09:22.424 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:09:22.424 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:09:22.424 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:09:22.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:22.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[213841]: failed to encode map e168 with expected crc 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[213841]: pgmap v437: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[213841]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[213841]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[213841]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[213841]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[221067]: failed to encode map e168 with expected crc 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[221067]: pgmap v437: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[221067]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:09:22.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[221067]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[221067]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.494 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:22 smithi163 ceph-mon[221067]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.496 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:22.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:22.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:22.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:22.674 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:22 smithi181 ceph-mon[196979]: failed to encode map e168 with expected crc 2023-10-09T04:09:22.674 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:22 smithi181 ceph-mon[196979]: pgmap v437: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:22.674 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:22 smithi181 ceph-mon[196979]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:09:22.674 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:22 smithi181 ceph-mon[196979]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:22 smithi181 ceph-mon[196979]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:22 smithi181 ceph-mon[196979]: failed to encode map e169 with expected crc 2023-10-09T04:09:22.676 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:22.719 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:22.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:22.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:22.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:22.855 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:23.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:23.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:23.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:09:23.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:23 smithi163 ceph-mon[221067]: failed to encode map e169 with expected crc 2023-10-09T04:09:23.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:23 smithi163 ceph-mon[213841]: failed to encode map e169 with expected crc 2023-10-09T04:09:23.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:23.491 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:23.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:23.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:23.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:23.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:23.669 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:23.711 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:23.717 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:23 smithi181 ceph-mon[196979]: failed to encode map e169 with expected crc 2023-10-09T04:09:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:23.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:23.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:23.846 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:24.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:24.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:24.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:24.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:09:24.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:24.169 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:24.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:24.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:24.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:24.341 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:24.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:24.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:24.490 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:24.516 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:24.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:24.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:24.662 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:24.689 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:24 smithi163 ceph-mon[221067]: pgmap v439: 89 pgs: 6 creating+peering, 3 unknown, 80 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:24.689 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:24 smithi163 ceph-mon[213841]: pgmap v439: 89 pgs: 6 creating+peering, 3 unknown, 80 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:24.693 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:24.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:24 smithi181 ceph-mon[196979]: pgmap v439: 89 pgs: 6 creating+peering, 3 unknown, 80 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:24.840 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:24.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:24.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:24.974 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:24.983 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:25.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:25.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:25.076 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:25.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:25.193 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:25.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:25.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:25.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:25.372 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:25.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:25.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:25.436 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:09:25.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/100737211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:09:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:25 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:09:25.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:25.549 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/100737211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:09:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:25 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:09:25.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/100737211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:09:25.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:25 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:09:25.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:25.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:25.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:25.724 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:25.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:25.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:25.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:25.903 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:26.437 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T04:09:26.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:26.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:26.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:26.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: pgmap v440: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:26.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:26.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: failed to encode map e170 with expected crc 2023-10-09T04:09:26.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: failed to encode map e170 with expected crc 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3024613745' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: pgmap v440: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: failed to encode map e170 with expected crc 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:09:26.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:09:26.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: failed to encode map e170 with expected crc 2023-10-09T04:09:26.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3024613745' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:26.683 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:26 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:26.685 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: pgmap v440: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: failed to encode map e170 with expected crc 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: failed to encode map e170 with expected crc 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3024613745' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:26 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:27.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:27.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:27.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:27.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:27.470 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:27.517 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:27.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[221067]: failed to encode map e171 with expected crc 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[221067]: failed to encode map e171 with expected crc 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[221067]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[213841]: failed to encode map e171 with expected crc 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[213841]: failed to encode map e171 with expected crc 2023-10-09T04:09:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:27 smithi163 ceph-mon[213841]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:09:27.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:27.652 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:27 smithi181 ceph-mon[196979]: failed to encode map e171 with expected crc 2023-10-09T04:09:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:27 smithi181 ceph-mon[196979]: failed to encode map e171 with expected crc 2023-10-09T04:09:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:27 smithi181 ceph-mon[196979]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:09:28.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:28.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:28.228 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:09:28.228 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:09:28.228 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:09:28.228 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:09:28.228 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:09:28.253 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:28.285 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:28.296 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T04:09:28.296 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-10-09T04:09:28.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:09:28.299 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:09:28.299 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:09:28.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:28.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:28.350 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T04:09:28.351 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:09:28.351 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_346082 2023-10-09T04:09:28.351 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:09:28.351 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:09:28.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:28.477 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:28.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:28.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:28 smithi163 ceph-mon[221067]: pgmap v443: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:28 smithi163 ceph-mon[213841]: pgmap v443: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:28.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:28.657 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:28 smithi181 ceph-mon[196979]: pgmap v443: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:28.659 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:28.700 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:28.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:28.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:28.842 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:28.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:28.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:29.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:29.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:09:29.072 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:29.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:29.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:29.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:29.232 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:09:29.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:29.337 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:09:29.353 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2033 2017 125.784 126.062 0.0124932 0.00791008 2023-10-09T04:09:29.381 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:09:29.417 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:29.427 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:09:29.470 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:09:29.502 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:09:29.514 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:29.535 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:29.538 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:09:29.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:29.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:29.576 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:09:29.622 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:09:29.627 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.627 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.627 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.627 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.627 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.628 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.628 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.628 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:09:29.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:29.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/95108028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:29.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/95108028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:29.684 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:29.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:29.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/95108028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:29.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:29.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:29.879 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:29.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:29.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:30.048 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:30.085 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:30.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:30.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:30.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:30.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:30.290 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:30.352 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:30.358 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 4719 4703 146.495 167.875 0.00364588 0.0068112 2023-10-09T04:09:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:30.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:30.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:30.497 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:30.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:30.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:30.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:30.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:30 smithi181 ceph-mon[196979]: pgmap v444: 89 pgs: 89 active+clean; 8.2 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 984 KiB/s wr, 15 op/s 2023-10-09T04:09:30.699 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:30.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:30.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:30.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:30 smithi163 ceph-mon[213841]: pgmap v444: 89 pgs: 89 active+clean; 8.2 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 984 KiB/s wr, 15 op/s 2023-10-09T04:09:30.834 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:09:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:09:30] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.43.0" 2023-10-09T04:09:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:30 smithi163 ceph-mon[221067]: pgmap v444: 89 pgs: 89 active+clean; 8.2 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 984 KiB/s wr, 15 op/s 2023-10-09T04:09:30.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:30.934 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:31.358 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 7836 7820 162.558 194.812 0.00386814 0.00614165 2023-10-09T04:09:31.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:31.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:31.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:31.855 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:31.906 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:31.907 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:32.067 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:32.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:32.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:32.235 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:32.358 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 10345 10329 161.097 156.812 0.00287558 0.0061997 2023-10-09T04:09:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:32 smithi181 ceph-mon[196979]: pgmap v445: 89 pgs: 89 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s wr, 262 op/s 2023-10-09T04:09:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:32 smithi163 ceph-mon[221067]: pgmap v445: 89 pgs: 89 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s wr, 262 op/s 2023-10-09T04:09:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:32 smithi163 ceph-mon[213841]: pgmap v445: 89 pgs: 89 active+clean; 132 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s wr, 262 op/s 2023-10-09T04:09:33.359 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 13003 12987 162.097 166.125 0.00148379 0.00609862 2023-10-09T04:09:33.665 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:33.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:33.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:33.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:09:33.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:33.804 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:33.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:33.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:33.958 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:34.061 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:34.362 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 15139 15123 157.24 133.5 0.00739294 0.00635326 2023-10-09T04:09:34.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:34 smithi163 ceph-mon[213841]: pgmap v446: 89 pgs: 89 active+clean; 321 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s 2023-10-09T04:09:34.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:34.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:34 smithi181 ceph-mon[196979]: pgmap v446: 89 pgs: 89 active+clean; 321 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s 2023-10-09T04:09:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:34.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:34 smithi163 ceph-mon[221067]: pgmap v446: 89 pgs: 89 active+clean; 321 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 641 op/s 2023-10-09T04:09:34.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:34.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:35.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:35.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:35.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:35.210 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:35.362 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 17188 17172 153.077 128.062 0.00679851 0.00652679 2023-10-09T04:09:35.581 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:09:35.596 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:09:35.731 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:35.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:35.838 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:35.871 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:35.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:35.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:36.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:36.103 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:36.363 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19353 19337 150.842 135.312 0.00429632 0.00662337 2023-10-09T04:09:36.495 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:36 smithi163 ceph-mon[213841]: pgmap v447: 89 pgs: 89 active+clean; 731 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:09:36.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:36 smithi181 ceph-mon[196979]: pgmap v447: 89 pgs: 89 active+clean; 731 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:09:36.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:36 smithi163 ceph-mon[221067]: pgmap v447: 89 pgs: 89 active+clean; 731 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:09:37.364 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 20732 20716 143.666 86.1875 0.00847463 0.00695483 2023-10-09T04:09:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:37 smithi181 ceph-mon[196979]: from='client.35124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:37.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:37 smithi163 ceph-mon[221067]: from='client.35124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:37.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:37 smithi163 ceph-mon[213841]: from='client.35124 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:09:38.192 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:38.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:38.297 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:38.362 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:38.369 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 22206 22190 138.437 92.125 0.0149742 0.0072137 2023-10-09T04:09:38.435 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-10-09T04:09:35.408205+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":765990816,"num_objects":11703,"num_object_clones":0,"num_object_copies":35109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11703,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":11979,"num_write_kb":749901,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11895,"ondisk_log_size":11895,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4020880,"kb_used_data":2154832,"kb_used_omap":126,"kb_used_meta":1865857,"kb_avail":745940336,"statfs":{"total":767960285184,"available":763842904064,"internally_reserved":0,"allocated":2206547968,"data_stored":2202006118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":1910638203},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":765394944,"num_objects":11679,"num_object_clones":0,"num_object_copies":35037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11679,"num_write_kb":747456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.977641"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":148,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627580+0000","last_change":"2023-10-09T04:03:06.346869+0000","last_active":"2023-10-09T04:09:25.627580+0000","last_peered":"2023-10-09T04:09:25.627580+0000","last_clean":"2023-10-09T04:09:25.627580+0000","last_became_active":"2023-10-09T04:03:06.346455+0000","last_became_peered":"2023-10-09T04:03:06.346455+0000","last_unstale":"2023-10-09T04:09:25.627580+0000","last_undegraded":"2023-10-09T04:09:25.627580+0000","last_fullsized":"2023-10-09T04:09:25.627580+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:41.881666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627990+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:09:25.627990+0000","last_peered":"2023-10-09T04:09:25.627990+0000","last_clean":"2023-10-09T04:09:25.627990+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:09:25.627990+0000","last_undegraded":"2023-10-09T04:09:25.627990+0000","last_fullsized":"2023-10-09T04:09:25.627990+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":120,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628737+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:09:25.628737+0000","last_peered":"2023-10-09T04:09:25.628737+0000","last_clean":"2023-10-09T04:09:25.628737+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:09:25.628737+0000","last_undegraded":"2023-10-09T04:09:25.628737+0000","last_fullsized":"2023-10-09T04:09:25.628737+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628462+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:09:25.628462+0000","last_peered":"2023-10-09T04:09:25.628462+0000","last_clean":"2023-10-09T04:09:25.628462+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:09:25.628462+0000","last_undegraded":"2023-10-09T04:09:25.628462+0000","last_fullsized":"2023-10-09T04:09:25.628462+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629637+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:09:25.629637+0000","last_peered":"2023-10-09T04:09:25.629637+0000","last_clean":"2023-10-09T04:09:25.629637+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:09:25.629637+0000","last_undegraded":"2023-10-09T04:09:25.629637+0000","last_fullsized":"2023-10-09T04:09:25.629637+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":104,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627833+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:09:25.627833+0000","last_peered":"2023-10-09T04:09:25.627833+0000","last_clean":"2023-10-09T04:09:25.627833+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:09:25.627833+0000","last_undegraded":"2023-10-09T04:09:25.627833+0000","last_fullsized":"2023-10-09T04:09:25.627833+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629093+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:09:25.629093+0000","last_peered":"2023-10-09T04:09:25.629093+0000","last_clean":"2023-10-09T04:09:25.629093+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:09:25.629093+0000","last_undegraded":"2023-10-09T04:09:25.629093+0000","last_fullsized":"2023-10-09T04:09:25.629093+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":148,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628024+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:09:25.628024+0000","last_peered":"2023-10-09T04:09:25.628024+0000","last_clean":"2023-10-09T04:09:25.628024+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:09:25.628024+0000","last_undegraded":"2023-10-09T04:09:25.628024+0000","last_fullsized":"2023-10-09T04:09:25.628024+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628616+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:09:25.628616+0000","last_peered":"2023-10-09T04:09:25.628616+0000","last_clean":"2023-10-09T04:09:25.628616+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:09:25.628616+0000","last_undegraded":"2023-10-09T04:09:25.628616+0000","last_fullsized":"2023-10-09T04:09:25.628616+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434087+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:09:25.434087+0000","last_peered":"2023-10-09T04:09:25.434087+0000","last_clean":"2023-10-09T04:09:25.434087+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:09:25.434087+0000","last_undegraded":"2023-10-09T04:09:25.434087+0000","last_fullsized":"2023-10-09T04:09:25.434087+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":133,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434363+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:09:25.434363+0000","last_peered":"2023-10-09T04:09:25.434363+0000","last_clean":"2023-10-09T04:09:25.434363+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:09:25.434363+0000","last_undegraded":"2023-10-09T04:09:25.434363+0000","last_fullsized":"2023-10-09T04:09:25.434363+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:26.677779+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:09:26.677779+0000","last_peered":"2023-10-09T04:09:26.677779+0000","last_clean":"2023-10-09T04:09:26.677779+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:09:26.677779+0000","last_undegraded":"2023-10-09T04:09:26.677779+0000","last_fullsized":"2023-10-09T04:09:26.677779+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628053+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:09:25.628053+0000","last_peered":"2023-10-09T04:09:25.628053+0000","last_clean":"2023-10-09T04:09:25.628053+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:09:25.628053+0000","last_undegraded":"2023-10-09T04:09:25.628053+0000","last_fullsized":"2023-10-09T04:09:25.628053+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628705+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:09:25.628705+0000","last_peered":"2023-10-09T04:09:25.628705+0000","last_clean":"2023-10-09T04:09:25.628705+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:09:25.628705+0000","last_undegraded":"2023-10-09T04:09:25.628705+0000","last_fullsized":"2023-10-09T04:09:25.628705+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434249+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:09:25.434249+0000","last_peered":"2023-10-09T04:09:25.434249+0000","last_clean":"2023-10-09T04:09:25.434249+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:09:25.434249+0000","last_undegraded":"2023-10-09T04:09:25.434249+0000","last_fullsized":"2023-10-09T04:09:25.434249+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":106,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628288+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:09:25.628288+0000","last_peered":"2023-10-09T04:09:25.628288+0000","last_clean":"2023-10-09T04:09:25.628288+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:09:25.628288+0000","last_undegraded":"2023-10-09T04:09:25.628288+0000","last_fullsized":"2023-10-09T04:09:25.628288+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627922+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:09:25.627922+0000","last_peered":"2023-10-09T04:09:25.627922+0000","last_clean":"2023-10-09T04:09:25.627922+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:09:25.627922+0000","last_undegraded":"2023-10-09T04:09:25.627922+0000","last_fullsized":"2023-10-09T04:09:25.627922+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627876+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:09:25.627876+0000","last_peered":"2023-10-09T04:09:25.627876+0000","last_clean":"2023-10-09T04:09:25.627876+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:09:25.627876+0000","last_undegraded":"2023-10-09T04:09:25.627876+0000","last_fullsized":"2023-10-09T04:09:25.627876+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434058+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:09:25.434058+0000","last_peered":"2023-10-09T04:09:25.434058+0000","last_clean":"2023-10-09T04:09:25.434058+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:09:25.434058+0000","last_undegraded":"2023-10-09T04:09:25.434058+0000","last_fullsized":"2023-10-09T04:09:25.434058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628391+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:09:25.628391+0000","last_peered":"2023-10-09T04:09:25.628391+0000","last_clean":"2023-10-09T04:09:25.628391+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:09:25.628391+0000","last_undegraded":"2023-10-09T04:09:25.628391+0000","last_fullsized":"2023-10-09T04:09:25.628391+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":61,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629564+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:09:25.629564+0000","last_peered":"2023-10-09T04:09:25.629564+0000","last_clean":"2023-10-09T04:09:25.629564+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:09:25.629564+0000","last_undegraded":"2023-10-09T04:09:25.629564+0000","last_fullsized":"2023-10-09T04:09:25.629564+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629200+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:09:25.629200+0000","last_peered":"2023-10-09T04:09:25.629200+0000","last_clean":"2023-10-09T04:09:25.629200+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:09:25.629200+0000","last_undegraded":"2023-10-09T04:09:25.629200+0000","last_fullsized":"2023-10-09T04:09:25.629200+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"171'816","reported_seq":829,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:33.595000+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:09:33.595000+0000","last_peered":"2023-10-09T04:09:33.595000+0000","last_clean":"2023-10-09T04:09:33.595000+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:09:33.595000+0000","last_undegraded":"2023-10-09T04:09:33.595000+0000","last_fullsized":"2023-10-09T04:09:33.595000+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":816,"log_dups_size":0,"ondisk_log_size":816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":816,"num_write_kb":52224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":131,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434537+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:09:25.434537+0000","last_peered":"2023-10-09T04:09:25.434537+0000","last_clean":"2023-10-09T04:09:25.434537+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:09:25.434537+0000","last_undegraded":"2023-10-09T04:09:25.434537+0000","last_fullsized":"2023-10-09T04:09:25.434537+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":112,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627447+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:09:25.627447+0000","last_peered":"2023-10-09T04:09:25.627447+0000","last_clean":"2023-10-09T04:09:25.627447+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:09:25.627447+0000","last_undegraded":"2023-10-09T04:09:25.627447+0000","last_fullsized":"2023-10-09T04:09:25.627447+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"171'1022","reported_seq":1035,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:34.829973+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:09:34.829973+0000","last_peered":"2023-10-09T04:09:34.829973+0000","last_clean":"2023-10-09T04:09:34.829973+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:09:34.829973+0000","last_undegraded":"2023-10-09T04:09:34.829973+0000","last_fullsized":"2023-10-09T04:09:34.829973+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":1022,"log_dups_size":0,"ondisk_log_size":1022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1022,"num_write_kb":65408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434514+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:09:25.434514+0000","last_peered":"2023-10-09T04:09:25.434514+0000","last_clean":"2023-10-09T04:09:25.434514+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:09:25.434514+0000","last_undegraded":"2023-10-09T04:09:25.434514+0000","last_fullsized":"2023-10-09T04:09:25.434514+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":146,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627846+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:09:25.627846+0000","last_peered":"2023-10-09T04:09:25.627846+0000","last_clean":"2023-10-09T04:09:25.627846+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:09:25.627846+0000","last_undegraded":"2023-10-09T04:09:25.627846+0000","last_fullsized":"2023-10-09T04:09:25.627846+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"171'604","reported_seq":617,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:32.037338+0000","last_change":"2023-10-09T04:09:20.480625+0000","last_active":"2023-10-09T04:09:32.037338+0000","last_peered":"2023-10-09T04:09:32.037338+0000","last_clean":"2023-10-09T04:09:32.037338+0000","last_became_active":"2023-10-09T04:09:20.480349+0000","last_became_peered":"2023-10-09T04:09:20.480349+0000","last_unstale":"2023-10-09T04:09:32.037338+0000","last_undegraded":"2023-10-09T04:09:32.037338+0000","last_fullsized":"2023-10-09T04:09:32.037338+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":604,"log_dups_size":0,"ondisk_log_size":604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:26.524901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39583744,"num_objects":604,"num_object_clones":0,"num_object_copies":1812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":604,"num_write_kb":38656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":114,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434272+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:09:25.434272+0000","last_peered":"2023-10-09T04:09:25.434272+0000","last_clean":"2023-10-09T04:09:25.434272+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:09:25.434272+0000","last_undegraded":"2023-10-09T04:09:25.434272+0000","last_fullsized":"2023-10-09T04:09:25.434272+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":89,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628348+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:09:25.628348+0000","last_peered":"2023-10-09T04:09:25.628348+0000","last_clean":"2023-10-09T04:09:25.628348+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:09:25.628348+0000","last_undegraded":"2023-10-09T04:09:25.628348+0000","last_fullsized":"2023-10-09T04:09:25.628348+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"171'1041","reported_seq":1054,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:34.831451+0000","last_change":"2023-10-09T04:09:20.480086+0000","last_active":"2023-10-09T04:09:34.831451+0000","last_peered":"2023-10-09T04:09:34.831451+0000","last_clean":"2023-10-09T04:09:34.831451+0000","last_became_active":"2023-10-09T04:09:20.479800+0000","last_became_peered":"2023-10-09T04:09:20.479800+0000","last_unstale":"2023-10-09T04:09:34.831451+0000","last_undegraded":"2023-10-09T04:09:34.831451+0000","last_fullsized":"2023-10-09T04:09:34.831451+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":1041,"log_dups_size":0,"ondisk_log_size":1041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:31.937055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68222976,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1041,"num_write_kb":66624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":130,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628224+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:09:25.628224+0000","last_peered":"2023-10-09T04:09:25.628224+0000","last_clean":"2023-10-09T04:09:25.628224+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:09:25.628224+0000","last_undegraded":"2023-10-09T04:09:25.628224+0000","last_fullsized":"2023-10-09T04:09:25.628224+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629522+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:09:25.629522+0000","last_peered":"2023-10-09T04:09:25.629522+0000","last_clean":"2023-10-09T04:09:25.629522+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:09:25.629522+0000","last_undegraded":"2023-10-09T04:09:25.629522+0000","last_fullsized":"2023-10-09T04:09:25.629522+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"171'631","reported_seq":644,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:32.032275+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:09:32.032275+0000","last_peered":"2023-10-09T04:09:32.032275+0000","last_clean":"2023-10-09T04:09:32.032275+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:09:32.032275+0000","last_undegraded":"2023-10-09T04:09:32.032275+0000","last_fullsized":"2023-10-09T04:09:32.032275+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":631,"log_dups_size":0,"ondisk_log_size":631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41353216,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":631,"num_write_kb":40384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":74,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629440+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:09:25.629440+0000","last_peered":"2023-10-09T04:09:25.629440+0000","last_clean":"2023-10-09T04:09:25.629440+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:09:25.629440+0000","last_undegraded":"2023-10-09T04:09:25.629440+0000","last_fullsized":"2023-10-09T04:09:25.629440+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627965+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:09:25.627965+0000","last_peered":"2023-10-09T04:09:25.627965+0000","last_clean":"2023-10-09T04:09:25.627965+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:09:25.627965+0000","last_undegraded":"2023-10-09T04:09:25.627965+0000","last_fullsized":"2023-10-09T04:09:25.627965+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"171'987","reported_seq":1000,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:34.808949+0000","last_change":"2023-10-09T04:09:20.476840+0000","last_active":"2023-10-09T04:09:34.808949+0000","last_peered":"2023-10-09T04:09:34.808949+0000","last_clean":"2023-10-09T04:09:34.808949+0000","last_became_active":"2023-10-09T04:09:20.476532+0000","last_became_peered":"2023-10-09T04:09:20.476532+0000","last_unstale":"2023-10-09T04:09:34.808949+0000","last_undegraded":"2023-10-09T04:09:34.808949+0000","last_fullsized":"2023-10-09T04:09:34.808949+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":987,"log_dups_size":0,"ondisk_log_size":987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:08.732253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64684032,"num_objects":987,"num_object_clones":0,"num_object_copies":2961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":987,"num_write_kb":63168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":90,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627672+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:09:25.627672+0000","last_peered":"2023-10-09T04:09:25.627672+0000","last_clean":"2023-10-09T04:09:25.627672+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:09:25.627672+0000","last_undegraded":"2023-10-09T04:09:25.627672+0000","last_fullsized":"2023-10-09T04:09:25.627672+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":146,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628203+0000","last_change":"2023-10-09T04:03:06.347744+0000","last_active":"2023-10-09T04:09:25.628203+0000","last_peered":"2023-10-09T04:09:25.628203+0000","last_clean":"2023-10-09T04:09:25.628203+0000","last_became_active":"2023-10-09T04:03:06.347418+0000","last_became_peered":"2023-10-09T04:03:06.347418+0000","last_unstale":"2023-10-09T04:09:25.628203+0000","last_undegraded":"2023-10-09T04:09:25.628203+0000","last_fullsized":"2023-10-09T04:09:25.628203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:18:10.530455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"171'873","reported_seq":886,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:33.588268+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:09:33.588268+0000","last_peered":"2023-10-09T04:09:33.588268+0000","last_clean":"2023-10-09T04:09:33.588268+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:09:33.588268+0000","last_undegraded":"2023-10-09T04:09:33.588268+0000","last_fullsized":"2023-10-09T04:09:33.588268+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":873,"log_dups_size":0,"ondisk_log_size":873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":873,"num_write_kb":55872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627637+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:09:25.627637+0000","last_peered":"2023-10-09T04:09:25.627637+0000","last_clean":"2023-10-09T04:09:25.627637+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:09:25.627637+0000","last_undegraded":"2023-10-09T04:09:25.627637+0000","last_fullsized":"2023-10-09T04:09:25.627637+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628064+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:09:25.628064+0000","last_peered":"2023-10-09T04:09:25.628064+0000","last_clean":"2023-10-09T04:09:25.628064+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:09:25.628064+0000","last_undegraded":"2023-10-09T04:09:25.628064+0000","last_fullsized":"2023-10-09T04:09:25.628064+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"171'313","reported_seq":326,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:30.670140+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:09:30.670140+0000","last_peered":"2023-10-09T04:09:30.670140+0000","last_clean":"2023-10-09T04:09:30.670140+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:09:30.670140+0000","last_undegraded":"2023-10-09T04:09:30.670140+0000","last_fullsized":"2023-10-09T04:09:30.670140+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":313,"log_dups_size":0,"ondisk_log_size":313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":313,"num_write_kb":20032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":135,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627880+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:09:25.627880+0000","last_peered":"2023-10-09T04:09:25.627880+0000","last_clean":"2023-10-09T04:09:25.627880+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:09:25.627880+0000","last_undegraded":"2023-10-09T04:09:25.627880+0000","last_fullsized":"2023-10-09T04:09:25.627880+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628133+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:09:25.628133+0000","last_peered":"2023-10-09T04:09:25.628133+0000","last_clean":"2023-10-09T04:09:25.628133+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:09:25.628133+0000","last_undegraded":"2023-10-09T04:09:25.628133+0000","last_fullsized":"2023-10-09T04:09:25.628133+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"171'1066","reported_seq":1078,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:35.183848+0000","last_change":"2023-10-09T04:09:20.482177+0000","last_active":"2023-10-09T04:09:35.183848+0000","last_peered":"2023-10-09T04:09:35.183848+0000","last_clean":"2023-10-09T04:09:35.183848+0000","last_became_active":"2023-10-09T04:09:20.481901+0000","last_became_peered":"2023-10-09T04:09:20.481901+0000","last_unstale":"2023-10-09T04:09:35.183848+0000","last_undegraded":"2023-10-09T04:09:35.183848+0000","last_fullsized":"2023-10-09T04:09:35.183848+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":1066,"log_dups_size":0,"ondisk_log_size":1066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:17.499123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69861376,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1066,"num_write_kb":68224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":145,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628520+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:09:25.628520+0000","last_peered":"2023-10-09T04:09:25.628520+0000","last_clean":"2023-10-09T04:09:25.628520+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:09:25.628520+0000","last_undegraded":"2023-10-09T04:09:25.628520+0000","last_fullsized":"2023-10-09T04:09:25.628520+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627934+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:09:25.627934+0000","last_peered":"2023-10-09T04:09:25.627934+0000","last_clean":"2023-10-09T04:09:25.627934+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:09:25.627934+0000","last_undegraded":"2023-10-09T04:09:25.627934+0000","last_fullsized":"2023-10-09T04:09:25.627934+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628537+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:09:25.628537+0000","last_peered":"2023-10-09T04:09:25.628537+0000","last_clean":"2023-10-09T04:09:25.628537+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:09:25.628537+0000","last_undegraded":"2023-10-09T04:09:25.628537+0000","last_fullsized":"2023-10-09T04:09:25.628537+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"171'624","reported_seq":637,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:32.041161+0000","last_change":"2023-10-09T04:09:20.479256+0000","last_active":"2023-10-09T04:09:32.041161+0000","last_peered":"2023-10-09T04:09:32.041161+0000","last_clean":"2023-10-09T04:09:32.041161+0000","last_became_active":"2023-10-09T04:09:20.478952+0000","last_became_peered":"2023-10-09T04:09:20.478952+0000","last_unstale":"2023-10-09T04:09:32.041161+0000","last_undegraded":"2023-10-09T04:09:32.041161+0000","last_fullsized":"2023-10-09T04:09:32.041161+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":624,"log_dups_size":0,"ondisk_log_size":624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:40.537104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40894464,"num_objects":624,"num_object_clones":0,"num_object_copies":1872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":624,"num_write_kb":39936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627709+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:09:25.627709+0000","last_peered":"2023-10-09T04:09:25.627709+0000","last_clean":"2023-10-09T04:09:25.627709+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:09:25.627709+0000","last_undegraded":"2023-10-09T04:09:25.627709+0000","last_fullsized":"2023-10-09T04:09:25.627709+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628781+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:09:25.628781+0000","last_peered":"2023-10-09T04:09:25.628781+0000","last_clean":"2023-10-09T04:09:25.628781+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:09:25.628781+0000","last_undegraded":"2023-10-09T04:09:25.628781+0000","last_fullsized":"2023-10-09T04:09:25.628781+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":137,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628449+0000","last_change":"2023-10-09T04:03:37.218452+0000","last_active":"2023-10-09T04:09:25.628449+0000","last_peered":"2023-10-09T04:09:25.628449+0000","last_clean":"2023-10-09T04:09:25.628449+0000","last_became_active":"2023-10-09T04:03:37.218082+0000","last_became_peered":"2023-10-09T04:03:37.218082+0000","last_unstale":"2023-10-09T04:09:25.628449+0000","last_undegraded":"2023-10-09T04:09:25.628449+0000","last_fullsized":"2023-10-09T04:09:25.628449+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:58.393592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"171'615","reported_seq":628,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:32.043478+0000","last_change":"2023-10-09T04:09:20.480244+0000","last_active":"2023-10-09T04:09:32.043478+0000","last_peered":"2023-10-09T04:09:32.043478+0000","last_clean":"2023-10-09T04:09:32.043478+0000","last_became_active":"2023-10-09T04:09:20.479963+0000","last_became_peered":"2023-10-09T04:09:20.479963+0000","last_unstale":"2023-10-09T04:09:32.043478+0000","last_undegraded":"2023-10-09T04:09:32.043478+0000","last_fullsized":"2023-10-09T04:09:32.043478+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":615,"log_dups_size":0,"ondisk_log_size":615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:04:16.202574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40304640,"num_objects":615,"num_object_clones":0,"num_object_copies":1845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":615,"num_write_kb":39360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":155,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628720+0000","last_change":"2023-10-09T04:03:37.215356+0000","last_active":"2023-10-09T04:09:25.628720+0000","last_peered":"2023-10-09T04:09:25.628720+0000","last_clean":"2023-10-09T04:09:25.628720+0000","last_became_active":"2023-10-09T04:03:37.215048+0000","last_became_peered":"2023-10-09T04:03:37.215048+0000","last_unstale":"2023-10-09T04:09:25.628720+0000","last_undegraded":"2023-10-09T04:09:25.628720+0000","last_fullsized":"2023-10-09T04:09:25.628720+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:01.339840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629977+0000","last_change":"2023-10-09T04:03:06.340273+0000","last_active":"2023-10-09T04:09:25.629977+0000","last_peered":"2023-10-09T04:09:25.629977+0000","last_clean":"2023-10-09T04:09:25.629977+0000","last_became_active":"2023-10-09T04:03:06.340007+0000","last_became_peered":"2023-10-09T04:03:06.340007+0000","last_unstale":"2023-10-09T04:09:25.629977+0000","last_undegraded":"2023-10-09T04:09:25.629977+0000","last_fullsized":"2023-10-09T04:09:25.629977+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.228322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":140,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628207+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:09:25.628207+0000","last_peered":"2023-10-09T04:09:25.628207+0000","last_clean":"2023-10-09T04:09:25.628207+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:09:25.628207+0000","last_undegraded":"2023-10-09T04:09:25.628207+0000","last_fullsized":"2023-10-09T04:09:25.628207+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"171'560","reported_seq":573,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:32.043526+0000","last_change":"2023-10-09T04:09:20.475846+0000","last_active":"2023-10-09T04:09:32.043526+0000","last_peered":"2023-10-09T04:09:32.043526+0000","last_clean":"2023-10-09T04:09:32.043526+0000","last_became_active":"2023-10-09T04:09:20.475586+0000","last_became_peered":"2023-10-09T04:09:20.475586+0000","last_unstale":"2023-10-09T04:09:32.043526+0000","last_undegraded":"2023-10-09T04:09:32.043526+0000","last_fullsized":"2023-10-09T04:09:32.043526+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":560,"log_dups_size":0,"ondisk_log_size":560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:06.160979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":560,"num_write_kb":35840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627760+0000","last_change":"2023-10-09T04:03:06.345507+0000","last_active":"2023-10-09T04:09:25.627760+0000","last_peered":"2023-10-09T04:09:25.627760+0000","last_clean":"2023-10-09T04:09:25.627760+0000","last_became_active":"2023-10-09T04:03:06.345146+0000","last_became_peered":"2023-10-09T04:03:06.345146+0000","last_unstale":"2023-10-09T04:09:25.627760+0000","last_undegraded":"2023-10-09T04:09:25.627760+0000","last_fullsized":"2023-10-09T04:09:25.627760+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:07.209488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":511,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434033+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:09:25.434033+0000","last_peered":"2023-10-09T04:09:25.434033+0000","last_clean":"2023-10-09T04:09:25.434033+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:09:25.434033+0000","last_undegraded":"2023-10-09T04:09:25.434033+0000","last_fullsized":"2023-10-09T04:09:25.434033+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":67,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627396+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:09:25.627396+0000","last_peered":"2023-10-09T04:09:25.627396+0000","last_clean":"2023-10-09T04:09:25.627396+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:09:25.627396+0000","last_undegraded":"2023-10-09T04:09:25.627396+0000","last_fullsized":"2023-10-09T04:09:25.627396+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":50,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628396+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:09:25.628396+0000","last_peered":"2023-10-09T04:09:25.628396+0000","last_clean":"2023-10-09T04:09:25.628396+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:09:25.628396+0000","last_undegraded":"2023-10-09T04:09:25.628396+0000","last_fullsized":"2023-10-09T04:09:25.628396+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"171'313","reported_seq":325,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:30.494475+0000","last_change":"2023-10-09T04:09:20.479449+0000","last_active":"2023-10-09T04:09:30.494475+0000","last_peered":"2023-10-09T04:09:30.494475+0000","last_clean":"2023-10-09T04:09:30.494475+0000","last_became_active":"2023-10-09T04:09:20.479062+0000","last_became_peered":"2023-10-09T04:09:20.479062+0000","last_unstale":"2023-10-09T04:09:30.494475+0000","last_undegraded":"2023-10-09T04:09:30.494475+0000","last_fullsized":"2023-10-09T04:09:30.494475+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":313,"log_dups_size":0,"ondisk_log_size":313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:08.154820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":313,"num_write_kb":20032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":151,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.433970+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:09:25.433970+0000","last_peered":"2023-10-09T04:09:25.433970+0000","last_clean":"2023-10-09T04:09:25.433970+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:09:25.433970+0000","last_undegraded":"2023-10-09T04:09:25.433970+0000","last_fullsized":"2023-10-09T04:09:25.433970+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628939+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:09:25.628939+0000","last_peered":"2023-10-09T04:09:25.628939+0000","last_clean":"2023-10-09T04:09:25.628939+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:09:25.628939+0000","last_undegraded":"2023-10-09T04:09:25.628939+0000","last_fullsized":"2023-10-09T04:09:25.628939+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":140,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628674+0000","last_change":"2023-10-09T04:03:06.341993+0000","last_active":"2023-10-09T04:09:25.628674+0000","last_peered":"2023-10-09T04:09:25.628674+0000","last_clean":"2023-10-09T04:09:25.628674+0000","last_became_active":"2023-10-09T04:03:06.341704+0000","last_became_peered":"2023-10-09T04:03:06.341704+0000","last_unstale":"2023-10-09T04:09:25.628674+0000","last_undegraded":"2023-10-09T04:09:25.628674+0000","last_fullsized":"2023-10-09T04:09:25.628674+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:42.382902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"171'330","reported_seq":342,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:30.432018+0000","last_change":"2023-10-09T04:09:20.475783+0000","last_active":"2023-10-09T04:09:30.432018+0000","last_peered":"2023-10-09T04:09:30.432018+0000","last_clean":"2023-10-09T04:09:30.432018+0000","last_became_active":"2023-10-09T04:09:20.475509+0000","last_became_peered":"2023-10-09T04:09:20.475509+0000","last_unstale":"2023-10-09T04:09:30.432018+0000","last_undegraded":"2023-10-09T04:09:30.432018+0000","last_fullsized":"2023-10-09T04:09:30.432018+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":330,"log_dups_size":0,"ondisk_log_size":330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:17.123893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21626880,"num_objects":330,"num_object_clones":0,"num_object_copies":990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":330,"num_write_kb":21120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434318+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:09:25.434318+0000","last_peered":"2023-10-09T04:09:25.434318+0000","last_clean":"2023-10-09T04:09:25.434318+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:09:25.434318+0000","last_undegraded":"2023-10-09T04:09:25.434318+0000","last_fullsized":"2023-10-09T04:09:25.434318+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628597+0000","last_change":"2023-10-09T04:03:06.343141+0000","last_active":"2023-10-09T04:09:25.628597+0000","last_peered":"2023-10-09T04:09:25.628597+0000","last_clean":"2023-10-09T04:09:25.628597+0000","last_became_active":"2023-10-09T04:03:06.342837+0000","last_became_peered":"2023-10-09T04:03:06.342837+0000","last_unstale":"2023-10-09T04:09:25.628597+0000","last_undegraded":"2023-10-09T04:09:25.628597+0000","last_fullsized":"2023-10-09T04:09:25.628597+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:37.219659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":110,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629429+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:09:25.629429+0000","last_peered":"2023-10-09T04:09:25.629429+0000","last_clean":"2023-10-09T04:09:25.629429+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:09:25.629429+0000","last_undegraded":"2023-10-09T04:09:25.629429+0000","last_fullsized":"2023-10-09T04:09:25.629429+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"171'999","reported_seq":1012,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:34.820292+0000","last_change":"2023-10-09T04:09:20.480771+0000","last_active":"2023-10-09T04:09:34.820292+0000","last_peered":"2023-10-09T04:09:34.820292+0000","last_clean":"2023-10-09T04:09:34.820292+0000","last_became_active":"2023-10-09T04:09:20.480530+0000","last_became_peered":"2023-10-09T04:09:20.480530+0000","last_unstale":"2023-10-09T04:09:34.820292+0000","last_undegraded":"2023-10-09T04:09:34.820292+0000","last_fullsized":"2023-10-09T04:09:34.820292+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":999,"log_dups_size":0,"ondisk_log_size":999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:13.028355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65470464,"num_objects":999,"num_object_clones":0,"num_object_copies":2997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":999,"num_write_kb":63936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":75,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434398+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:09:25.434398+0000","last_peered":"2023-10-09T04:09:25.434398+0000","last_clean":"2023-10-09T04:09:25.434398+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:09:25.434398+0000","last_undegraded":"2023-10-09T04:09:25.434398+0000","last_fullsized":"2023-10-09T04:09:25.434398+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":106,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629204+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:09:25.629204+0000","last_peered":"2023-10-09T04:09:25.629204+0000","last_clean":"2023-10-09T04:09:25.629204+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:09:25.629204+0000","last_undegraded":"2023-10-09T04:09:25.629204+0000","last_fullsized":"2023-10-09T04:09:25.629204+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434122+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:09:25.434122+0000","last_peered":"2023-10-09T04:09:25.434122+0000","last_clean":"2023-10-09T04:09:25.434122+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:09:25.434122+0000","last_undegraded":"2023-10-09T04:09:25.434122+0000","last_fullsized":"2023-10-09T04:09:25.434122+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"171'885","reported_seq":898,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:33.594927+0000","last_change":"2023-10-09T04:09:20.476061+0000","last_active":"2023-10-09T04:09:33.594927+0000","last_peered":"2023-10-09T04:09:33.594927+0000","last_clean":"2023-10-09T04:09:33.594927+0000","last_became_active":"2023-10-09T04:09:20.475767+0000","last_became_peered":"2023-10-09T04:09:20.475767+0000","last_unstale":"2023-10-09T04:09:33.594927+0000","last_undegraded":"2023-10-09T04:09:33.594927+0000","last_fullsized":"2023-10-09T04:09:33.594927+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":885,"log_dups_size":0,"ondisk_log_size":885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:05.520013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999360,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":885,"num_write_kb":56640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":99,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.434464+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:09:25.434464+0000","last_peered":"2023-10-09T04:09:25.434464+0000","last_clean":"2023-10-09T04:09:25.434464+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:09:25.434464+0000","last_undegraded":"2023-10-09T04:09:25.434464+0000","last_fullsized":"2023-10-09T04:09:25.434464+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628547+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:09:25.628547+0000","last_peered":"2023-10-09T04:09:25.628547+0000","last_clean":"2023-10-09T04:09:25.628547+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:09:25.628547+0000","last_undegraded":"2023-10-09T04:09:25.628547+0000","last_fullsized":"2023-10-09T04:09:25.628547+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":112,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627713+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:09:25.627713+0000","last_peered":"2023-10-09T04:09:25.627713+0000","last_clean":"2023-10-09T04:09:25.627713+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:09:25.627713+0000","last_undegraded":"2023-10-09T04:09:25.627713+0000","last_fullsized":"2023-10-09T04:09:25.627713+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":114,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T04:09:26.680531+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:09:26.680531+0000","last_peered":"2023-10-09T04:09:26.680531+0000","last_clean":"2023-10-09T04:09:26.680531+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:09:26.680531+0000","last_undegraded":"2023-10-09T04:09:26.680531+0000","last_fullsized":"2023-10-09T04:09:26.680531+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629151+0000","last_change":"2023-10-09T04:03:37.215366+0000","last_active":"2023-10-09T04:09:25.629151+0000","last_peered":"2023-10-09T04:09:25.629151+0000","last_clean":"2023-10-09T04:09:25.629151+0000","last_became_active":"2023-10-09T04:03:37.215054+0000","last_became_peered":"2023-10-09T04:03:37.215054+0000","last_unstale":"2023-10-09T04:09:25.629151+0000","last_undegraded":"2023-10-09T04:09:25.629151+0000","last_fullsized":"2023-10-09T04:09:25.629151+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:38.315273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":132,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628296+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:09:25.628296+0000","last_peered":"2023-10-09T04:09:25.628296+0000","last_clean":"2023-10-09T04:09:25.628296+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:09:25.628296+0000","last_undegraded":"2023-10-09T04:09:25.628296+0000","last_fullsized":"2023-10-09T04:09:25.628296+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":146,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628853+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:09:25.628853+0000","last_peered":"2023-10-09T04:09:25.628853+0000","last_clean":"2023-10-09T04:09:25.628853+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:09:25.628853+0000","last_undegraded":"2023-10-09T04:09:25.628853+0000","last_fullsized":"2023-10-09T04:09:25.628853+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":115,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627968+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:09:25.627968+0000","last_peered":"2023-10-09T04:09:25.627968+0000","last_clean":"2023-10-09T04:09:25.627968+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:09:25.627968+0000","last_undegraded":"2023-10-09T04:09:25.627968+0000","last_fullsized":"2023-10-09T04:09:25.627968+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":146,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627757+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:09:25.627757+0000","last_peered":"2023-10-09T04:09:25.627757+0000","last_clean":"2023-10-09T04:09:25.627757+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:09:25.627757+0000","last_undegraded":"2023-10-09T04:09:25.627757+0000","last_fullsized":"2023-10-09T04:09:25.627757+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":50,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628580+0000","last_change":"2023-10-09T04:06:57.499785+0000","last_active":"2023-10-09T04:09:25.628580+0000","last_peered":"2023-10-09T04:09:25.628580+0000","last_clean":"2023-10-09T04:09:25.628580+0000","last_became_active":"2023-10-09T04:06:57.499419+0000","last_became_peered":"2023-10-09T04:06:57.499419+0000","last_unstale":"2023-10-09T04:09:25.628580+0000","last_undegraded":"2023-10-09T04:09:25.628580+0000","last_fullsized":"2023-10-09T04:09:25.628580+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:14.270664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":127,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.628794+0000","last_change":"2023-10-09T04:06:45.454363+0000","last_active":"2023-10-09T04:09:25.628794+0000","last_peered":"2023-10-09T04:09:25.628794+0000","last_clean":"2023-10-09T04:09:25.628794+0000","last_became_active":"2023-10-09T04:06:45.454065+0000","last_became_peered":"2023-10-09T04:06:45.454065+0000","last_unstale":"2023-10-09T04:09:25.628794+0000","last_undegraded":"2023-10-09T04:09:25.628794+0000","last_fullsized":"2023-10-09T04:09:25.628794+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:21.580027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.629179+0000","last_change":"2023-10-09T04:03:06.343774+0000","last_active":"2023-10-09T04:09:25.629179+0000","last_peered":"2023-10-09T04:09:25.629179+0000","last_clean":"2023-10-09T04:09:25.629179+0000","last_became_active":"2023-10-09T04:03:06.343462+0000","last_became_peered":"2023-10-09T04:03:06.343462+0000","last_unstale":"2023-10-09T04:09:25.629179+0000","last_undegraded":"2023-10-09T04:09:25.629179+0000","last_fullsized":"2023-10-09T04:09:25.629179+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:23.577037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":129,"reported_epoch":170,"state":"active+clean","last_fresh":"2023-10-09T04:09:25.627466+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:09:25.627466+0000","last_peered":"2023-10-09T04:09:25.627466+0000","last_clean":"2023-10-09T04:09:25.627466+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:09:25.627466+0000","last_undegraded":"2023-10-09T04:09:25.627466+0000","last_fullsized":"2023-10-09T04:09:25.627466+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":765394944,"num_objects":11679,"num_object_clones":0,"num_object_copies":35037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11679,"num_write_kb":747456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2145779712,"data_stored":2145779712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11679,"ondisk_log_size":11679,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201484,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":419908,"kb_used_data":114428,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93325244,"statfs":{"total":95995035648,"available":95565049856,"internally_reserved":0,"allocated":117174272,"data_stored":116663659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.77,"15min":0.93500000000000005},"min":{"1min":0.254,"5min":0.219,"15min":0.219},"max":{"1min":5.3520000000000003,"5min":17.398,"15min":17.398},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.605,"5min":1.6679999999999999,"15min":0.90200000000000002},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.7370000000000001,"5min":17.306999999999999,"15min":17.306999999999999},"last":0.64300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":3.2320000000000002,"5min":1.752,"15min":1.0029999999999999},"min":{"1min":0.26200000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.051,"5min":10.051,"15min":10.051},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.2080000000000002,"5min":1.6220000000000001,"15min":1.0089999999999999},"min":{"1min":0.26200000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.7680000000000007,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":1.8100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.427,"15min":1.1739999999999999},"min":{"1min":0.308,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.7119999999999997,"5min":58.615000000000002,"15min":58.615000000000002},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.6279999999999999,"5min":3.5859999999999999,"15min":1.599},"min":{"1min":0.33600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.681,"5min":99.372,"15min":99.372},"last":0.67100000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":3.218,"15min":1.4590000000000001},"min":{"1min":0.33400000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.124000000000001,"5min":99.134,"15min":99.134},"last":1.1399999999999999},{"interface":"front","average":{"1min":2.6480000000000001,"5min":3.367,"15min":1.5329999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.140000000000001,"5min":99.117999999999995,"15min":99.117999999999995},"last":1.1830000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":0.995,"15min":0.68000000000000005},"min":{"1min":0.30499999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.683,"5min":12.683,"15min":12.683},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.8180000000000001,"5min":1.056,"15min":0.70899999999999996},"min":{"1min":0.29199999999999998,"5min":0.28599999999999998,"15min":0.27500000000000002},"max":{"1min":12.442,"5min":12.442,"15min":12.442},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.80000000000000004,"15min":0.60599999999999998},"min":{"1min":0.36199999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6669999999999998,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.91400000000000003,"15min":0.67600000000000005},"min":{"1min":0.311,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4209999999999998,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.46999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.97099999999999997,"15min":0.68700000000000006},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":3.609,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.0289999999999999,"15min":0.73599999999999999},"min":{"1min":0.40600000000000003,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":4.585,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.50700000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463122,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":696052,"kb_used_data":394668,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93049100,"statfs":{"total":95995035648,"available":95282278400,"internally_reserved":0,"allocated":404140032,"data_stored":403645334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:09:18 2023","interfaces":[{"interface":"back","average":{"1min":5.069,"5min":3.194,"15min":1.419},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":54.667000000000002,"5min":78.983999999999995,"15min":78.983999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":5.2439999999999998,"5min":3.8490000000000002,"15min":1.641},"min":{"1min":0.28299999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":54.655000000000001,"5min":78.998000000000005,"15min":78.998000000000005},"last":0.59399999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":5.399,"5min":3.0640000000000001,"15min":1.369},"min":{"1min":0.30599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":57.094000000000001,"5min":57.094000000000001,"15min":57.094000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":5.8369999999999997,"5min":3.1829999999999998,"15min":1.399},"min":{"1min":0.27000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":57.003,"5min":57.003,"15min":57.003},"last":0.67600000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":2.3769999999999998,"15min":1.165},"min":{"1min":0.29299999999999998,"5min":0.218,"15min":0.218},"max":{"1min":3.8849999999999998,"5min":42.195999999999998,"15min":42.195999999999998},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.3759999999999999,"5min":2.4180000000000001,"15min":1.214},"min":{"1min":0.372,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.968,"5min":42.165999999999997,"15min":42.165999999999997},"last":0.73599999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":8.8030000000000008,"5min":3.7559999999999998,"15min":1.6619999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":57.509999999999998,"5min":57.509999999999998,"15min":57.509999999999998},"last":0.92500000000000004},{"interface":"front","average":{"1min":9.3079999999999998,"5min":4.0510000000000002,"15min":1.7869999999999999},"min":{"1min":0.41899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":57.453000000000003,"5min":57.453000000000003,"15min":57.453000000000003},"last":1.089}]},{"osd":4,"last update":"Mon Oct 9 04:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.52500000000000002,"15min":0.48399999999999999},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.72999999999999998,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.85599999999999998,"15min":0.62},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":1.7609999999999999,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.47299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.226,"15min":0.752},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.734,"5min":54.773000000000003,"15min":54.773000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.3999999999999999,"15min":0.84899999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8220000000000001,"5min":54.853999999999999,"15min":54.853999999999999},"last":0.65700000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:09:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.68100000000000005,"15min":0.60699999999999998},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":3.0630000000000002,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.68200000000000005},"min":{"1min":0.36699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.8730000000000002,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.627}]}]},{"osd":1,"up_from":89,"seq":382252089430,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":433500,"kb_used_data":299412,"kb_used_omap":9,"kb_used_meta":134070,"kb_avail":93311652,"statfs":{"total":95995035648,"available":95551131648,"internally_reserved":0,"allocated":306597888,"data_stored":305948923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":137288681},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":2.258,"15min":1},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0389999999999997,"5min":45.811999999999998,"15min":45.811999999999998},"last":0.503},{"interface":"front","average":{"1min":2.7490000000000001,"5min":3.5089999999999999,"15min":1.431},"min":{"1min":0.24099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.4820000000000002,"5min":99.831999999999994,"15min":99.831999999999994},"last":3.04}]},{"osd":2,"last update":"Mon Oct 9 04:09:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":2.7629999999999999,"15min":1.1499999999999999},"min":{"1min":0.22,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.8250000000000002,"5min":121.54600000000001,"15min":121.54600000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":2.0110000000000001,"5min":3.0699999999999998,"15min":1.268},"min":{"1min":0.22900000000000001,"5min":0.186,"15min":0.186},"max":{"1min":8.4450000000000003,"5min":99.899000000000001,"15min":99.899000000000001},"last":2.786}]},{"osd":3,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":2.8119999999999998,"15min":1.1879999999999999},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.56,"5min":42.588999999999999,"15min":42.588999999999999},"last":1.2490000000000001},{"interface":"front","average":{"1min":3.8330000000000002,"5min":4.0110000000000001,"15min":1.5920000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.163,"5min":100.02500000000001,"15min":100.02500000000001},"last":1.6339999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.8050000000000002,"5min":2.581,"15min":1.3220000000000001},"min":{"1min":0.33600000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.5529999999999999,"5min":99.989999999999995,"15min":99.989999999999995},"last":1.577},{"interface":"front","average":{"1min":1.768,"5min":2.5099999999999998,"15min":1.304},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8239999999999998,"5min":100.249,"15min":100.249},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":2.492,"15min":1.3129999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.5780000000000003,"5min":99.933999999999997,"15min":99.933999999999997},"last":1.5409999999999999},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.6960000000000002,"15min":1.3819999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.4779999999999998,"5min":100.34099999999999,"15min":100.34099999999999},"last":0.70399999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.847,"5min":3.0209999999999999,"15min":1.4830000000000001},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.968,"5min":100.309,"15min":100.309},"last":0.67600000000000005},{"interface":"front","average":{"1min":2.044,"5min":2.8319999999999999,"15min":1.4379999999999999},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.9089999999999998,"5min":100.143,"15min":100.143},"last":0.86899999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.8599999999999999,"15min":1.45},"min":{"1min":0.373,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.8490000000000002,"5min":100.214,"15min":100.214},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.7719999999999998,"5min":2.7280000000000002,"15min":1.4219999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":7.8869999999999996,"5min":100.113,"15min":100.113},"last":0.83599999999999997}]}]},{"osd":0,"up_from":84,"seq":360777252956,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":393896,"kb_used_data":285984,"kb_used_omap":28,"kb_used_meta":107875,"kb_avail":93351256,"statfs":{"total":95995035648,"available":95591686144,"internally_reserved":0,"allocated":292847616,"data_stored":292184811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":110464848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:08:44 2023","interfaces":[{"interface":"back","average":{"1min":4.157,"5min":4.2679999999999998,"15min":1.6719999999999999},"min":{"1min":0.19800000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":44.287999999999997,"5min":118.48,"15min":118.48},"last":3.258},{"interface":"front","average":{"1min":4.4889999999999999,"5min":4.5720000000000001,"15min":1.766},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":43.906999999999996,"5min":118.384,"15min":118.384},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:09:14 2023","interfaces":[{"interface":"back","average":{"1min":5.5110000000000001,"5min":4.5860000000000003,"15min":1.8069999999999999},"min":{"1min":0.27300000000000002,"5min":0.221,"15min":0.221},"max":{"1min":25.055,"5min":118.54000000000001,"15min":118.54000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":3.6019999999999999,"5min":5.0190000000000001,"15min":1.952},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":16.515000000000001,"5min":118.727,"15min":118.727},"last":1.645}]},{"osd":3,"last update":"Mon Oct 9 04:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.6959999999999997,"5min":7.4530000000000003,"15min":2.7450000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":44.460000000000001,"5min":118.73999999999999,"15min":118.73999999999999},"last":1.4350000000000001},{"interface":"front","average":{"1min":6.0570000000000004,"5min":7.4199999999999999,"15min":2.7679999999999998},"min":{"1min":0.34899999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":43.972000000000001,"5min":118.584,"15min":118.584},"last":0.32700000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:09:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":3.3340000000000001,"15min":1.508},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":10.023,"5min":118.633,"15min":118.633},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.254,"5min":4.2530000000000001,"15min":1.825},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.202,"5min":118.913,"15min":118.913},"last":1.6759999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:09:31 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":5.734,"15min":2.3279999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.311,"5min":182.68899999999999,"15min":182.68899999999999},"last":1.706},{"interface":"front","average":{"1min":1.425,"5min":5.1989999999999998,"15min":2.1589999999999998},"min":{"1min":0.38900000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.2430000000000003,"5min":182.66300000000001,"15min":182.66300000000001},"last":1.5049999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:09:31 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":3.2200000000000002,"15min":1.514},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.8230000000000004,"5min":118.864,"15min":118.864},"last":1.4690000000000001},{"interface":"front","average":{"1min":2.0409999999999999,"5min":2.8860000000000001,"15min":1.3939999999999999},"min":{"1min":0.40400000000000003,"5min":0.317,"15min":0.317},"max":{"1min":10.092000000000001,"5min":118.70399999999999,"15min":118.70399999999999},"last":0.81200000000000006}]},{"osd":7,"last update":"Mon Oct 9 04:09:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":5.7619999999999996,"15min":2.3559999999999999},"min":{"1min":0.46999999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.0779999999999994,"5min":185.024,"15min":185.024},"last":1.5760000000000001},{"interface":"front","average":{"1min":2.544,"5min":5.1689999999999996,"15min":2.169},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.114000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.42099999999999999}]}]},{"osd":2,"up_from":94,"seq":403726925904,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":713540,"kb_used_data":555516,"kb_used_omap":81,"kb_used_meta":157934,"kb_avail":93031612,"statfs":{"total":95995035648,"available":95264370688,"internally_reserved":0,"allocated":568848384,"data_stored":568220684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":161724634},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:09:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":2.1629999999999998,"15min":0.94499999999999995},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.187},"max":{"1min":12.284000000000001,"5min":12.284000000000001,"15min":12.284000000000001},"last":2.149},{"interface":"front","average":{"1min":4.9489999999999998,"5min":3.1200000000000001,"15min":1.3009999999999999},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":45.979999999999997,"5min":45.979999999999997,"15min":45.979999999999997},"last":1.867}]},{"osd":1,"last update":"Mon Oct 9 04:09:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1219999999999999,"5min":4.024,"15min":1.591},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.221},"max":{"1min":46.049999999999997,"5min":68.009,"15min":68.009},"last":2.5430000000000001},{"interface":"front","average":{"1min":4.944,"5min":4.4809999999999999,"15min":1.738},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":46.012999999999998,"5min":66.975999999999999,"15min":66.975999999999999},"last":1.657}]},{"osd":3,"last update":"Mon Oct 9 04:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.359,"15min":1.0409999999999999},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":12.77,"5min":27.562000000000001,"15min":27.562000000000001},"last":2.2170000000000001},{"interface":"front","average":{"1min":5.4850000000000003,"5min":2.9830000000000001,"15min":1.2470000000000001},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":46.286999999999999,"5min":46.286999999999999,"15min":46.286999999999999},"last":4.4130000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:09:16 2023","interfaces":[{"interface":"back","average":{"1min":4.9210000000000003,"5min":2.6600000000000001,"15min":1.282},"min":{"1min":0.46300000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":46.075000000000003,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.6160000000000001,"15min":0.93600000000000005},"min":{"1min":0.307,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1349999999999998,"5min":14.305,"15min":14.305},"last":1.1379999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:09:16 2023","interfaces":[{"interface":"back","average":{"1min":4.5869999999999997,"5min":2.343,"15min":1.2170000000000001},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":46.265000000000001,"5min":46.265000000000001,"15min":46.265000000000001},"last":2.0550000000000002},{"interface":"front","average":{"1min":4.9880000000000004,"5min":2.7690000000000001,"15min":1.353},"min":{"1min":0.46800000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":46.106000000000002,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.76700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:09:16 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.5660000000000001,"15min":0.94099999999999995},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":8.1850000000000005,"5min":8.1850000000000005,"15min":8.1850000000000005},"last":1.089},{"interface":"front","average":{"1min":4.3040000000000003,"5min":2.319,"15min":1.226},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":46.085000000000001,"5min":46.085000000000001,"15min":46.085000000000001},"last":1.9650000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:09:16 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.9179999999999999,"15min":1.0860000000000001},"min":{"1min":0.45000000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.391,"5min":14.364000000000001,"15min":14.364000000000001},"last":1.1160000000000001},{"interface":"front","average":{"1min":4.3070000000000004,"5min":2.5459999999999998,"15min":1.3220000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":46.241999999999997,"5min":46.241999999999997,"15min":46.241999999999997},"last":1.9019999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762377,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":391408,"kb_used_data":143208,"kb_used_omap":7,"kb_used_meta":248184,"kb_avail":93353744,"statfs":{"total":95995035648,"available":95594233856,"internally_reserved":0,"allocated":146644992,"data_stored":146034261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":254141032},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":4.181,"5min":5.6589999999999998,"15min":2.496},"min":{"1min":0.23200000000000001,"5min":0.185,"15min":0.185},"max":{"1min":13.166,"5min":106.22,"15min":106.22},"last":5.9649999999999999},{"interface":"front","average":{"1min":4.2809999999999997,"5min":5.6660000000000004,"15min":2.4969999999999999},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":11.722,"5min":106.986,"15min":106.986},"last":5.9459999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9990000000000001,"5min":3.8370000000000002,"15min":1.498},"min":{"1min":0.22,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.108000000000001,"5min":106.121,"15min":106.121},"last":0.32100000000000001},{"interface":"front","average":{"1min":4.4619999999999997,"5min":3.319,"15min":1.3400000000000001},"min":{"1min":0.30499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":16.977,"5min":65.015000000000001,"15min":65.015000000000001},"last":0.83499999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":3.726,"5min":4.4749999999999996,"15min":1.7370000000000001},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.205,"5min":106.105,"15min":106.105},"last":0.71299999999999997},{"interface":"front","average":{"1min":2.3460000000000001,"5min":4.6449999999999996,"15min":1.7889999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2170000000000005,"5min":106.70399999999999,"15min":106.70399999999999},"last":0.66500000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0750000000000002,"5min":3.839,"15min":1.675},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":37.652000000000001,"5min":106.85899999999999,"15min":106.85899999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":4.9370000000000003,"5min":2.2290000000000001,"15min":1.1399999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":37.442,"5min":37.442,"15min":37.442},"last":0.35099999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":1.964,"15min":1.05},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.8940000000000001,"5min":12.819000000000001,"15min":12.819000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.625,"5min":2.0139999999999998,"15min":1.4239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.228999999999999,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.65000000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6099999999999999,"5min":3.2210000000000001,"15min":1.8149999999999999},"min":{"1min":0.35499999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.255000000000001,"5min":106.036,"15min":106.036},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.7410000000000001,"5min":3.319,"15min":1.516},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":15.622,"5min":106.741,"15min":106.741},"last":0.63600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":3.492,"15min":1.583},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":15.685,"5min":106.82599999999999,"15min":106.82599999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.96,"5min":3.2330000000000001,"15min":1.478},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.8159999999999998,"5min":106.084,"15min":106.084},"last":0.754}]}]},{"osd":4,"up_from":31,"seq":133143986394,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619896,"kb_used_data":310320,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93125256,"statfs":{"total":95995035648,"available":95360262144,"internally_reserved":0,"allocated":317767680,"data_stored":317268476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:09:14 2023","interfaces":[{"interface":"back","average":{"1min":4.7850000000000001,"5min":2.2149999999999999,"15min":1.0900000000000001},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":63.128,"5min":63.128,"15min":63.128},"last":1.427},{"interface":"front","average":{"1min":4.0149999999999997,"5min":1.9970000000000001,"15min":1.044},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":62.645000000000003,"5min":62.645000000000003,"15min":62.645000000000003},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":5.3109999999999999,"5min":2.3439999999999999,"15min":1.1479999999999999},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":79.075999999999993,"5min":79.075999999999993,"15min":79.075999999999993},"last":1.1299999999999999},{"interface":"front","average":{"1min":5.7249999999999996,"5min":2.427,"15min":1.165},"min":{"1min":0.27000000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":75.314999999999998,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":1.9470000000000001,"15min":1.036},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":8.8030000000000008,"5min":19.356000000000002,"15min":19.356000000000002},"last":2.1840000000000002},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.8380000000000001,"15min":0.98699999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5920000000000005,"5min":19.324000000000002,"15min":19.324000000000002},"last":2.1960000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:08:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.7110000000000001,"15min":0.93100000000000005},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.492,"5min":14.694000000000001,"15min":14.694000000000001},"last":1.488},{"interface":"front","average":{"1min":1.379,"5min":1.716,"15min":0.94399999999999995},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.8840000000000003,"5min":9.0449999999999999,"15min":9.0449999999999999},"last":1.554}]},{"osd":5,"last update":"Mon Oct 9 04:09:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.79000000000000004,"15min":0.58599999999999997},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.4510000000000001,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":1.1080000000000001,"15min":0.71399999999999997},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":3.4460000000000002,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":1.276}]},{"osd":6,"last update":"Mon Oct 9 04:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0589999999999999,"15min":0.72099999999999997},"min":{"1min":0.308,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.6849999999999996,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":1.319},{"interface":"front","average":{"1min":1.1000000000000001,"5min":0.83199999999999996,"15min":0.64300000000000002},"min":{"1min":0.38200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.35999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:09:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.97599999999999998,"15min":0.69999999999999996},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.5190000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":1.524},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.1000000000000001,"15min":0.76700000000000002},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.202,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":1.7170000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724757,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":352680,"kb_used_data":51296,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93392472,"statfs":{"total":95995035648,"available":95633891328,"internally_reserved":0,"allocated":52527104,"data_stored":52039970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":2.3250000000000002,"15min":1.167},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.0119999999999996,"5min":63.037999999999997,"15min":63.037999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.2190000000000001,"5min":2.54,"15min":1.222},"min":{"1min":0.23899999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.4329999999999998,"5min":62.927999999999997,"15min":62.927999999999997},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:38 2023","interfaces":[{"interface":"back","average":{"1min":4.593,"5min":3.1120000000000001,"15min":1.399},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":47.926000000000002,"5min":91.212999999999994,"15min":91.212999999999994},"last":5.5609999999999999},{"interface":"front","average":{"1min":4.8319999999999999,"5min":3.5539999999999998,"15min":1.5680000000000001},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":47.972000000000001,"5min":91.254000000000005,"15min":91.254000000000005},"last":0.376}]},{"osd":2,"last update":"Mon Oct 9 04:09:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":3.2440000000000002,"15min":1.4670000000000001},"min":{"1min":0.255,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.324,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.1779999999999999,"5min":3.1230000000000002,"15min":1.3999999999999999},"min":{"1min":0.36599999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":13.545,"5min":61.154000000000003,"15min":61.154000000000003},"last":0.32200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":3.4319999999999999,"15min":1.534},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.8650000000000002,"5min":91.834999999999994,"15min":91.834999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.5569999999999999,"5min":3.4289999999999998,"15min":1.5469999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.3979999999999997,"5min":91.802000000000007,"15min":91.802000000000007},"last":0.439}]},{"osd":4,"last update":"Mon Oct 9 04:09:27 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.97799999999999998,"15min":0.64500000000000002},"min":{"1min":0.29599999999999999,"5min":0.255,"15min":0.253},"max":{"1min":5.7430000000000003,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.073,"15min":0.69999999999999996},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.26600000000000001},"max":{"1min":4.2869999999999999,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.36699999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:08:43 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.89100000000000001,"15min":0.63900000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2910000000000004,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":0.308},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.88300000000000001,"15min":0.65600000000000003},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.312},"max":{"1min":3.052,"5min":4.3099999999999996,"15min":4.3099999999999996},"last":0.46000000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:09:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.177,"15min":0.78000000000000003},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.0179999999999998,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.194,"15min":0.80500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34599999999999997},"max":{"1min":4.1449999999999996,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":0.51100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":316538880,"data_stored":316538880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":340721664,"data_stored":340721664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":397082624,"data_stored":397082624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":209846272,"data_stored":209846272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":351076352,"data_stored":351076352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":84017152,"data_stored":84017152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":344653824,"data_stored":344653824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":101842944,"data_stored":101842944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:09:38.442 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:09:38.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 26 2023-10-09T04:09:38.500 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:38 smithi163 ceph-mon[213841]: pgmap v448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:09:38.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:38.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:38.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:38.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:38 smithi163 ceph-mon[221067]: pgmap v448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:09:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:38 smithi181 ceph-mon[196979]: pgmap v448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:09:38.816 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:38.834 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:38.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:39.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:39.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:39.170 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:39.310 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:09:39.310 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:09:39.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:39.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:39.371 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 23656 23640 134.076 90.625 0.00553621 0.00745002 2023-10-09T04:09:39.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:39.498 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:39.525 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:09:39.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1933874381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:39.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:39.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1933874381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:39.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:39.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1933874381' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:09:40.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:40.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:40.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:40.176 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:40.371 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 25578 25562 132.913 120.125 0.00441697 0.00751952 2023-10-09T04:09:40.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: pgmap v449: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:09:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:40 smithi181 ceph-mon[196979]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: pgmap v449: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T04:09:40.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:09:40.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[213841]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:09:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:09:40] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-10-09T04:09:40.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: pgmap v449: 89 pgs: 89 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: failed to encode map e172 with expected crc 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:09:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:40 smithi163 ceph-mon[221067]: failed to encode map e172 with expected crc 2023-10-09T04:09:41.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:41.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:41.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:41.294 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:41.336 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:41.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:41.371 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 27176 27160 130.374 99.875 0.00267524 0.00756977 2023-10-09T04:09:41.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:41.466 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:41.519 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:41.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:41.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:41.652 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:41.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:41.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:41.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:41.869 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:42.180 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:42.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:42.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:42.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:09:42.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:42.325 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:42.376 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 28610 28594 127.428 89.625 0.00963505 0.00784086 2023-10-09T04:09:42.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:42.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:42.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:42.390 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:42.491 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:42.520 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:42.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:42.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:42 smithi181 ceph-mon[196979]: pgmap v451: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T04:09:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:42.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:42.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:42.714 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:42.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:42.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:42.785 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:42 smithi163 ceph-mon[213841]: pgmap v451: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T04:09:42.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:42 smithi163 ceph-mon[221067]: pgmap v451: 89 pgs: 89 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T04:09:42.866 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:09:42.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:42.904 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:09:42.907 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:42.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:09:42.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:42.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:43.003 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:09:43.052 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:09:43.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:43.093 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:09:43.120 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:43.129 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:09:43.164 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:09:43.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:43.178 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:09:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:43.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:43.238 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:43.281 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:43.311 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:43.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:43.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:43.377 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 30119 30103 125.208 94.3125 0.00388713 0.00798229 2023-10-09T04:09:43.475 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:43.510 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:43.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:43.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:43.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:43.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:43.747 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:43.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3203129419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:43.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3203129419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:43.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3203129419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:44.088 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:09:44.378 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 31455 31439 122.605 83.5 0.00676987 0.00815012 2023-10-09T04:09:44.540 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:44 smithi163 ceph-mon[213841]: pgmap v452: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T04:09:44.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:44.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:44.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:44.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:44 smithi163 ceph-mon[221067]: pgmap v452: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T04:09:44.859 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:44 smithi181 ceph-mon[196979]: pgmap v452: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T04:09:44.861 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:44.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:44.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:45.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:45.033 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:45.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:45.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:45.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:45.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:45.240 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:45.379 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 32716 32700 120.024 78.8125 0.00425224 0.00832626 2023-10-09T04:09:45.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:45.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:45.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:45.659 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:45.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:45.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:45.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:45.913 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:46.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:46.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:46.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:46.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:46.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:46.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:46.362 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:46.379 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 34209 34193 118.541 93.3125 0.00886119 0.00842772 2023-10-09T04:09:46.438 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:46.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:46.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:46.574 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 7 pg_num to 26 2023-10-09T04:09:46.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:46.671 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:46.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:46.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:46.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:46 smithi163 ceph-mon[221067]: pgmap v453: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T04:09:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1335292107' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:09:46.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:46 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:09:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:46 smithi163 ceph-mon[213841]: pgmap v453: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T04:09:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1335292107' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:09:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:46 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:09:46.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:46.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:46 smithi181 ceph-mon[196979]: pgmap v453: 89 pgs: 89 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T04:09:46.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1335292107' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:09:46.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:46 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:09:46.867 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:46.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:46.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:46.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:47.034 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:47.098 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:47.379 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 35652 35636 117.051 90.1875 0.00703523 0.00853791 2023-10-09T04:09:47.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:09:47.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[213841]: failed to encode map e173 with expected crc 2023-10-09T04:09:47.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:09:47.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[213841]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:09:47.802 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[213841]: failed to encode map e173 with expected crc 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[213841]: failed to encode map e173 with expected crc 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[221067]: failed to encode map e173 with expected crc 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[221067]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[221067]: failed to encode map e173 with expected crc 2023-10-09T04:09:47.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[221067]: failed to encode map e173 with expected crc 2023-10-09T04:09:47.804 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:09:47.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:47.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:47.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:48.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:47 smithi181 ceph-mon[196979]: failed to encode map e173 with expected crc 2023-10-09T04:09:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:09:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:47 smithi181 ceph-mon[196979]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:09:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:47 smithi181 ceph-mon[196979]: failed to encode map e173 with expected crc 2023-10-09T04:09:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:47 smithi181 ceph-mon[196979]: failed to encode map e173 with expected crc 2023-10-09T04:09:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:09:48.051 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:48.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:48.118 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:48.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:48.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:48.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:48.279 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:48.331 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:48.377 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:48.379 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:09:48.380023+0000 min lat: 0.0010512 max lat: 0.230195 avg lat: 0.00870108 2023-10-09T04:09:48.380 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:09:48.380 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 36621 36605 114.23 60.5625 0.0235222 0.00870108 2023-10-09T04:09:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:48.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:48.482 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:48.507 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:48.549 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:48.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:48.562 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:09:48.563 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:09:48.563 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:09:48.563 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:09:48.616 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:48.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:48.679 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:48 smithi181 ceph-mon[196979]: pgmap v455: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:09:48.679 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:48 smithi181 ceph-mon[196979]: failed to encode map e174 with expected crc 2023-10-09T04:09:48.679 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:09:48.679 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:48 smithi181 ceph-mon[196979]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:09:48.681 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:48.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:48.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:48.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[221067]: pgmap v455: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[221067]: failed to encode map e174 with expected crc 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[221067]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[213841]: pgmap v455: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[213841]: failed to encode map e174 with expected crc 2023-10-09T04:09:48.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:09:48.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:48 smithi163 ceph-mon[213841]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:09:48.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:09:48.854 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:48.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:09:48.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:48.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:09:49.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:49.031 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:49.299 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:49.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:49.379 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 36801 36785 109.332 11.25 0.00629789 0.00914169 2023-10-09T04:09:49.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:49.440 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:49.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:49.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:49.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:49.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:49.638 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:49.688 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:49.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:49.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:09:49.851 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:50.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:09:50.382 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 37976 37960 107.69 73.4375 0.0269372 0.0092792 2023-10-09T04:09:50.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:09:50.509 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.509 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.509 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: pgmap v458: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:09:50.510 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[213841]: 7.9 deep-scrub starts 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.511 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:50.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:50.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: pgmap v458: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:09:50.512 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:50 smithi163 ceph-mon[221067]: 7.9 deep-scrub starts 2023-10-09T04:09:50.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:50.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:09:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: failed to encode map e175 with expected crc 2023-10-09T04:09:50.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:50.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:50.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: pgmap v458: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:09:50.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:50 smithi181 ceph-mon[196979]: 7.9 deep-scrub starts 2023-10-09T04:09:50.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:50.647 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:50.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:50.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:50.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:09:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:09:50] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-10-09T04:09:50.879 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:51.382 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 39151 39135 106.203 73.4375 0.00446916 0.00940603 2023-10-09T04:09:51.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:51 smithi163 ceph-mon[213841]: 7.8 deep-scrub starts 2023-10-09T04:09:51.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:51 smithi163 ceph-mon[221067]: 7.8 deep-scrub starts 2023-10-09T04:09:51.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:51.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:51.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:51 smithi181 ceph-mon[196979]: 7.8 deep-scrub starts 2023-10-09T04:09:51.772 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:51.802 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:51.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:51.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:51.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:51.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:09:52.000 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:52.053 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:52.056 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:09:52.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:52.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:09:52.162 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:52.248 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:52.385 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 40620 40604 105.589 91.8125 0.0158443 0.00946352 2023-10-09T04:09:52.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:52.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:52.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:52.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:52 smithi163 ceph-mon[213841]: pgmap v459: 99 pgs: 1 peering, 5 unknown, 93 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T04:09:52.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:52 smithi163 ceph-mon[213841]: 7.9 deep-scrub ok 2023-10-09T04:09:52.423 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:52 smithi163 ceph-mon[221067]: pgmap v459: 99 pgs: 1 peering, 5 unknown, 93 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T04:09:52.423 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:52 smithi163 ceph-mon[221067]: 7.9 deep-scrub ok 2023-10-09T04:09:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:52 smithi181 ceph-mon[196979]: pgmap v459: 99 pgs: 1 peering, 5 unknown, 93 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T04:09:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:52 smithi181 ceph-mon[196979]: 7.9 deep-scrub ok 2023-10-09T04:09:52.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:52.560 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:09:52.598 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:52.612 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:09:52.650 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:09:52.693 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:09:52.741 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:09:52.773 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:09:52.812 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:09:52.851 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:09:52.857 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.857 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.857 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.857 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.857 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.857 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.858 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.858 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:09:52.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:52.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:53.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:53.074 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:53.093 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:09:53.093 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:09:53.093 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.26767549228509036 2023-10-09T04:09:53.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.26767549228509036 2023-10-09T04:09:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[213841]: 7.2 scrub starts 2023-10-09T04:09:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[213841]: 7.5 scrub starts 2023-10-09T04:09:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1020517397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[213841]: 7.2 scrub ok 2023-10-09T04:09:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3044555314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:53.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[221067]: 7.2 scrub starts 2023-10-09T04:09:53.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[221067]: 7.5 scrub starts 2023-10-09T04:09:53.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1020517397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:53.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:53.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[221067]: 7.2 scrub ok 2023-10-09T04:09:53.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3044555314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:53.349 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:09:53.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 42041 42025 104.918 88.8125 0.00909544 0.00952597 2023-10-09T04:09:53.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:53 smithi181 ceph-mon[196979]: 7.2 scrub starts 2023-10-09T04:09:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:53 smithi181 ceph-mon[196979]: 7.5 scrub starts 2023-10-09T04:09:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1020517397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:53 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:09:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:53 smithi181 ceph-mon[196979]: 7.2 scrub ok 2023-10-09T04:09:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3044555314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:09:54.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 43542 43526 104.491 93.8125 0.00778965 0.00956572 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: 7.0 scrub starts 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: 7.1 scrub starts 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: failed to encode map e176 with expected crc 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: failed to encode map e176 with expected crc 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: 7.0 scrub ok 2023-10-09T04:09:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[221067]: pgmap v461: 99 pgs: 1 peering, 1 unknown, 97 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: 7.0 scrub starts 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: 7.1 scrub starts 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: failed to encode map e176 with expected crc 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: failed to encode map e176 with expected crc 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: 7.0 scrub ok 2023-10-09T04:09:54.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:54 smithi163 ceph-mon[213841]: pgmap v461: 99 pgs: 1 peering, 1 unknown, 97 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T04:09:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: 7.0 scrub starts 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: 7.1 scrub starts 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: failed to encode map e176 with expected crc 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: failed to encode map e176 with expected crc 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: 7.0 scrub ok 2023-10-09T04:09:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:54 smithi181 ceph-mon[196979]: pgmap v461: 99 pgs: 1 peering, 1 unknown, 97 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T04:09:55.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 44891 44875 103.744 84.3125 0.00470817 0.00963349 2023-10-09T04:09:55.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:55 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:09:55.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:55.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:55 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:09:55.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:55 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:09:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:55.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:55.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:55.901 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:56.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:56.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:56.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:56.223 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:56.370 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:56.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:56.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:56.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 45289 45273 100.931 24.875 0.00222362 0.00962335 2023-10-09T04:09:56.455 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:56.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:56.506 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:56.551 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:09:56.551 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:09:56.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:56.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:56.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:56.685 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:56.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: pgmap v462: 99 pgs: 99 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 27 KiB/s, 0 objects/s recovering 2023-10-09T04:09:56.725 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[221067]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: pgmap v462: 99 pgs: 99 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 27 KiB/s, 0 objects/s recovering 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:09:56.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:09:56.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.727 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:56 smithi163 ceph-mon[213841]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:56.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: pgmap v462: 99 pgs: 99 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 27 KiB/s, 0 objects/s recovering 2023-10-09T04:09:56.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:09:56.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:09:56.733 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.734 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:56 smithi181 ceph-mon[196979]: failed to encode map e177 with expected crc 2023-10-09T04:09:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:56.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:56.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:09:56.865 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:57.173 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:09:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:57.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:09:57.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:09:57.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:57.303 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:57.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 45289 45273 97.4544 0 - 0.00962335 2023-10-09T04:09:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:57 smithi181 ceph-mon[196979]: failed to encode map e178 with expected crc 2023-10-09T04:09:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:57 smithi181 ceph-mon[196979]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:09:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:57 smithi181 ceph-mon[196979]: failed to encode map e178 with expected crc 2023-10-09T04:09:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:09:57.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:57.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:57.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[213841]: failed to encode map e178 with expected crc 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[213841]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[213841]: failed to encode map e178 with expected crc 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[221067]: failed to encode map e178 with expected crc 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[221067]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[221067]: failed to encode map e178 with expected crc 2023-10-09T04:09:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:09:57.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:57.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:57.961 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:57.999 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:58.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:09:58.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:58.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:58.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:09:58.137 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:58.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:09:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:58.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:09:58.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:09:58.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:58.322 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:58.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 45711 45695 95.0865 13.1875 0.00237981 0.0103207 2023-10-09T04:09:58.432 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:58.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:58.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:58.570 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:58.758 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: failed to encode map e178 with expected crc 2023-10-09T04:09:58.758 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: pgmap v465: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 39 MiB/s wr, 617 op/s 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[221067]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: failed to encode map e178 with expected crc 2023-10-09T04:09:58.759 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: pgmap v465: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 39 MiB/s wr, 617 op/s 2023-10-09T04:09:58.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:09:58.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:09:58.760 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:58 smithi163 ceph-mon[213841]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: failed to encode map e178 with expected crc 2023-10-09T04:09:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: pgmap v465: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 39 MiB/s wr, 617 op/s 2023-10-09T04:09:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:09:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:09:58.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:58 smithi181 ceph-mon[196979]: failed to encode map e179 with expected crc 2023-10-09T04:09:58.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:58.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:58.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:58.966 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:59.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:59.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:59.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:09:59.146 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:59.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:09:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:59.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:09:59.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:09:59.329 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:59.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:09:59.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:59.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:09:59.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 45711 45695 92.0224 0 - 0.0103207 2023-10-09T04:09:59.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:09:59.511 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 0.267675 (4486) 2023-10-09T04:09:59.527 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:09:59.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2330466476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.26767549228509036}]: dispatch 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:59.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2330466476' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.26767549228509036}]': finished 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[213841]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2330466476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.26767549228509036}]: dispatch 2023-10-09T04:09:59.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: failed to encode map e180 with expected crc 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2330466476' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.26767549228509036}]': finished 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:09:59.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:09:59 smithi163 ceph-mon[221067]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:09:59.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:09:59.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:09:59.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:09:59.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:00.006 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:00.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: failed to encode map e180 with expected crc 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: failed to encode map e180 with expected crc 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: failed to encode map e180 with expected crc 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: failed to encode map e180 with expected crc 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2330466476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.26767549228509036}]: dispatch 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: failed to encode map e180 with expected crc 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2330466476' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.26767549228509036}]': finished 2023-10-09T04:10:00.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:10:00.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:09:59 smithi181 ceph-mon[196979]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:10:00.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 45768 45752 89.2606 1.78125 0.00185291 0.0109184 2023-10-09T04:10:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: pgmap v468: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 33 MiB/s wr, 524 op/s 2023-10-09T04:10:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:10:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[221067]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:10:00.835 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:10:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:10:00] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-10-09T04:10:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: pgmap v468: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 33 MiB/s wr, 524 op/s 2023-10-09T04:10:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:10:00.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:00 smithi163 ceph-mon[213841]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:10:00.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:00.837 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:00.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:00.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:00.971 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: pgmap v468: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 33 MiB/s wr, 524 op/s 2023-10-09T04:10:00.971 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: failed to encode map e181 with expected crc 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:10:00.972 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:00 smithi181 ceph-mon[196979]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:10:00.975 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:01.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:01.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:01.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:01.157 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:01.197 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:10:01.197 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:10:01.201 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:01.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:01.223 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:01.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:01.335 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:01.378 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:01.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:01.386 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 45768 45752 86.5585 0 - 0.0109184 2023-10-09T04:10:01.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:01.536 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:01.541 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[213841]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.541 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[213841]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.541 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[213841]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.541 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[213841]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.541 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[213841]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:10:01.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:01.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:01.631 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:01 smithi181 ceph-mon[196979]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.631 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:01 smithi181 ceph-mon[196979]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.631 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:01 smithi181 ceph-mon[196979]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.632 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:01 smithi181 ceph-mon[196979]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.632 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:01 smithi181 ceph-mon[196979]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:10:01.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:01.790 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:01.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[221067]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[221067]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[221067]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[221067]: failed to encode map e182 with expected crc 2023-10-09T04:10:01.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:01 smithi163 ceph-mon[221067]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:10:01.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:01.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:01.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:02.031 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:02.228 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:02.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:02.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:02.368 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:02.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 46621 46605 85.5814 26.6562 0.00225615 0.011637 2023-10-09T04:10:02.470 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:02.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:02.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:02.668 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: pgmap v471: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s; 1402/136845 objects degraded (1.025%); 714/136845 objects misplaced (0.522%); 11 MiB/s, 177 objects/s recovering 2023-10-09T04:10:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 1402/136845 objects degraded (1.025%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: pgmap v471: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s; 1402/136845 objects degraded (1.025%); 714/136845 objects misplaced (0.522%); 11 MiB/s, 177 objects/s recovering 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: failed to encode map e183 with expected crc 2023-10-09T04:10:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:02 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 1402/136845 objects degraded (1.025%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:10:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: pgmap v471: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s; 1402/136845 objects degraded (1.025%); 714/136845 objects misplaced (0.522%); 11 MiB/s, 177 objects/s recovering 2023-10-09T04:10:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: failed to encode map e183 with expected crc 2023-10-09T04:10:03.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:02 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 1402/136845 objects degraded (1.025%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:10:03.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 47817 47801 85.2721 74.75 0.00697858 0.0117228 2023-10-09T04:10:03.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:03.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:03.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:10:03.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:03.663 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:04.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:03 smithi163 ceph-mon[221067]: pgmap v473: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s; 1402/137730 objects degraded (1.018%); 714/137730 objects misplaced (0.518%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T04:10:04.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:04.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:04.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:03 smithi163 ceph-mon[213841]: pgmap v473: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s; 1402/137730 objects degraded (1.018%); 714/137730 objects misplaced (0.518%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T04:10:04.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:04.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:04.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:04.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:03 smithi181 ceph-mon[196979]: pgmap v473: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s; 1402/137730 objects degraded (1.018%); 714/137730 objects misplaced (0.518%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T04:10:04.272 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:04.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 48907 48891 84.796 68.125 0.00270721 0.0116967 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:05.059 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:10:05.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:05.060 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:05.089 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:05.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:05.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:05.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:10:05.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.232 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.233 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.233 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: failed to encode map e184 with expected crc 2023-10-09T04:10:05.233 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:05.233 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:05.239 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:05.313 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:05.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:05.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 49614 49598 83.6994 44.1875 0.0116114 0.0119432 2023-10-09T04:10:05.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:05.472 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:05.474 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:10:05.513 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:10:05.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:05.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:05.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:05.564 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:10:05.613 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:10:05.650 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:10:05.654 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:05.687 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:10:05.739 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:10:05.750 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:05.766 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.779 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.780 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:10:05.866 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:10:05.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:10:05.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:05 smithi181 ceph-mon[196979]: failed to encode map e185 with expected crc 2023-10-09T04:10:05.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:05 smithi181 ceph-mon[196979]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:10:05.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:05 smithi181 ceph-mon[196979]: failed to encode map e185 with expected crc 2023-10-09T04:10:05.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:05 smithi181 ceph-mon[196979]: failed to encode map e185 with expected crc 2023-10-09T04:10:05.919 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:05.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:06.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:06.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[213841]: failed to encode map e185 with expected crc 2023-10-09T04:10:06.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[213841]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:10:06.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[213841]: failed to encode map e185 with expected crc 2023-10-09T04:10:06.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[213841]: failed to encode map e185 with expected crc 2023-10-09T04:10:06.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[213841]: pgmap v476: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 2 peering, 90 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 22 MiB/s wr, 350 op/s; 7313/141561 objects degraded (5.166%); 2185/141561 objects misplaced (1.544%); 27 MiB/s, 433 objects/s recovering 2023-10-09T04:10:06.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/768610765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:06.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[221067]: failed to encode map e185 with expected crc 2023-10-09T04:10:06.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[221067]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:10:06.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[221067]: failed to encode map e185 with expected crc 2023-10-09T04:10:06.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[221067]: failed to encode map e185 with expected crc 2023-10-09T04:10:06.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[221067]: pgmap v476: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 2 peering, 90 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 22 MiB/s wr, 350 op/s; 7313/141561 objects degraded (5.166%); 2185/141561 objects misplaced (1.544%); 27 MiB/s, 433 objects/s recovering 2023-10-09T04:10:06.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/768610765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:06.080 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:06.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:05 smithi181 ceph-mon[196979]: pgmap v476: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 2 peering, 90 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 22 MiB/s wr, 350 op/s; 7313/141561 objects degraded (5.166%); 2185/141561 objects misplaced (1.544%); 27 MiB/s, 433 objects/s recovering 2023-10-09T04:10:06.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/768610765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:06.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 50560 50544 83.053 59.125 0.00233835 0.0120021 2023-10-09T04:10:06.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:06.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:06.833 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:06.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:06.905 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:06.932 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:10:06.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:06.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:07.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:07.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2518309255' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:07.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:06 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:07.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2518309255' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:07.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:06 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:07.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2518309255' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:07.146 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:06 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:07.149 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:07.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:07.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:07.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:07.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:07.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 51710 51694 82.7664 71.875 0.00628743 0.0120766 2023-10-09T04:10:07.388 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:08.133 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:08.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:08.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:08.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:08.168 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 7313/141561 objects degraded (5.166%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[221067]: pgmap v478: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s; 11519/149286 objects degraded (7.716%); 3308/149286 objects misplaced (2.216%); 18 MiB/s, 291 objects/s recovering 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:08.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:10:08.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 7313/141561 objects degraded (5.166%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:08.170 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:07 smithi163 ceph-mon[213841]: pgmap v478: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s; 11519/149286 objects degraded (7.716%); 3308/149286 objects misplaced (2.216%); 18 MiB/s, 291 objects/s recovering 2023-10-09T04:10:08.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: failed to encode map e186 with expected crc 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 7313/141561 objects degraded (5.166%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:07 smithi181 ceph-mon[196979]: pgmap v478: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s; 11519/149286 objects degraded (7.716%); 3308/149286 objects misplaced (2.216%); 18 MiB/s, 291 objects/s recovering 2023-10-09T04:10:08.289 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:08.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:08.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:08.389 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:10:08.388029+0000 min lat: 0.0010512 max lat: 2.18233 avg lat: 0.012112 2023-10-09T04:10:08.389 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:10:08.389 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 52864 52848 82.5004 72.125 0.0084247 0.012112 2023-10-09T04:10:08.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:08.497 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:08.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:08.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:08.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:08.687 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:08.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:08.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:08.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:08.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:10:08.875 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:09.387 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 53261 53245 81.0944 24.8125 0.011512 0.01232 2023-10-09T04:10:09.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:09 smithi181 ceph-mon[196979]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:09 smithi181 ceph-mon[196979]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:10:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:09 smithi181 ceph-mon[196979]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:09 smithi181 ceph-mon[196979]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[221067]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[221067]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[221067]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[221067]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[213841]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[213841]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:10:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[213841]: failed to encode map e187 with expected crc 2023-10-09T04:10:09.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:09 smithi163 ceph-mon[213841]: failed to encode map e187 with expected crc 2023-10-09T04:10:10.169 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:10:10.391 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 54380 54364 80.8221 69.9375 0.00946917 0.0123672 2023-10-09T04:10:10.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:10.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:10.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:10.512 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:10.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[221067]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[221067]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[221067]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:10:10.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[221067]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[221067]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.546 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[221067]: pgmap v481: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 11519/152049 objects degraded (7.576%); 3308/152049 objects misplaced (2.176%); 24 MiB/s, 379 objects/s recovering 2023-10-09T04:10:10.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[213841]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[213841]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[213841]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:10:10.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[213841]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[213841]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.550 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:10 smithi163 ceph-mon[213841]: pgmap v481: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 11519/152049 objects degraded (7.576%); 3308/152049 objects misplaced (2.176%); 24 MiB/s, 379 objects/s recovering 2023-10-09T04:10:10.604 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:10 smithi181 ceph-mon[196979]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:10 smithi181 ceph-mon[196979]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:10 smithi181 ceph-mon[196979]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:10:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:10 smithi181 ceph-mon[196979]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:10 smithi181 ceph-mon[196979]: failed to encode map e188 with expected crc 2023-10-09T04:10:10.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:10 smithi181 ceph-mon[196979]: pgmap v481: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 11519/152049 objects degraded (7.576%); 3308/152049 objects misplaced (2.176%); 24 MiB/s, 379 objects/s recovering 2023-10-09T04:10:10.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:10.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:10.835 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:10:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:10:10] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-10-09T04:10:10.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:10.917 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:10.979 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:10.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:11.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:11.130 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:11.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:11.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:11.297 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:11.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:11.336 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:11.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:11.391 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 55482 55466 80.5443 68.875 0.0263883 0.0124097 2023-10-09T04:10:11.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:11.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:11.423 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:10:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:10:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:55512 7.a 7:5a6609ed:::benchmark_data_smithi163_346082_object55511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2023-10-09T04:10:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:11.416372+0000", 2023-10-09T04:10:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0050287409999999998, 2023-10-09T04:10:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050614240000000001, 2023-10-09T04:10:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:10:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 55512 2023-10-09T04:10:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.416372+0000", 2023-10-09T04:10:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.416372+0000", 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.416370+0000", 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.416402+0000", 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2089999999999999e-05 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.416425+0000", 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2480000000000002e-05 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.416434+0000", 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5829999999999996e-06 2023-10-09T04:10:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.417634+0000", 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012000800000000001 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.417777+0000", 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014252500000000001 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.417915+0000", 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137799 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:11.420409+0000", 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002493781 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:11.431 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:11.432 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:10:11.432 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 1 2023-10-09T04:10:11.432 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:10:11.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:11.588 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:11.841 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:11.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:11.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:11.984 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:12.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:12.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:12.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:12.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:12.242 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:12.391 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 56636 56620 80.353 72.125 0.00804827 0.0124416 2023-10-09T04:10:12.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:12 smithi163 ceph-mon[221067]: pgmap v482: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s; 10117/159234 objects degraded (6.354%); 2841/159234 objects misplaced (1.784%); 23 MiB/s, 361 objects/s recovering 2023-10-09T04:10:12.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 10117/159234 objects degraded (6.354%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:12.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3215577092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:10:12.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:12 smithi163 ceph-mon[213841]: pgmap v482: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s; 10117/159234 objects degraded (6.354%); 2841/159234 objects misplaced (1.784%); 23 MiB/s, 361 objects/s recovering 2023-10-09T04:10:12.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 10117/159234 objects degraded (6.354%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:12.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3215577092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:10:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:12 smithi181 ceph-mon[196979]: pgmap v482: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s; 10117/159234 objects degraded (6.354%); 2841/159234 objects misplaced (1.784%); 23 MiB/s, 361 objects/s recovering 2023-10-09T04:10:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 10117/159234 objects degraded (6.354%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3215577092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:10:13.106 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:13.109 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:13.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:13.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:10:13.215 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:13.247 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:13.271 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:10:13.311 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:13.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:13.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:13.391 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 57774 57758 80.148 71.125 0.0263465 0.0124702 2023-10-09T04:10:13.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:13.539 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3215577092' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:10:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[221067]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T04:10:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[221067]: failed to encode map e189 with expected crc 2023-10-09T04:10:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[221067]: failed to encode map e189 with expected crc 2023-10-09T04:10:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[221067]: pgmap v484: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 10117/164037 objects degraded (6.168%); 2841/164037 objects misplaced (1.732%); 19 MiB/s, 311 objects/s recovering 2023-10-09T04:10:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3215577092' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:10:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[213841]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T04:10:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[213841]: failed to encode map e189 with expected crc 2023-10-09T04:10:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[213841]: failed to encode map e189 with expected crc 2023-10-09T04:10:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:14 smithi163 ceph-mon[213841]: pgmap v484: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 10117/164037 objects degraded (6.168%); 2841/164037 objects misplaced (1.732%); 19 MiB/s, 311 objects/s recovering 2023-10-09T04:10:14.391 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 59071 59055 80.1677 81.0625 0.00511003 0.0124619 2023-10-09T04:10:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3215577092' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:10:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:14 smithi181 ceph-mon[196979]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T04:10:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:14 smithi181 ceph-mon[196979]: failed to encode map e189 with expected crc 2023-10-09T04:10:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:14 smithi181 ceph-mon[196979]: failed to encode map e189 with expected crc 2023-10-09T04:10:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:14 smithi181 ceph-mon[196979]: pgmap v484: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 10117/164037 objects degraded (6.168%); 2841/164037 objects misplaced (1.732%); 19 MiB/s, 311 objects/s recovering 2023-10-09T04:10:15.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:15.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:15.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:15.245 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:15.392 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 59430 59414 78.9402 22.4375 0.00223858 0.0126478 2023-10-09T04:10:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:15 smithi181 ceph-mon[196979]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:15 smithi181 ceph-mon[196979]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:15 smithi181 ceph-mon[196979]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T04:10:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:15 smithi181 ceph-mon[196979]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[213841]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[213841]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[213841]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T04:10:15.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[213841]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.561 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[221067]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[221067]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[221067]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T04:10:15.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:15 smithi163 ceph-mon[221067]: failed to encode map e190 with expected crc 2023-10-09T04:10:15.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:15.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:15.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:15.836 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:16.392 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 60777 60761 79.0493 84.1875 0.0664741 0.0126449 2023-10-09T04:10:16.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:16.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:16.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:16.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:16.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[213841]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[213841]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T04:10:16.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[213841]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[213841]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[213841]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[213841]: pgmap v487: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10117/170847 objects degraded (5.922%); 2841/170847 objects misplaced (1.663%); 19 MiB/s, 311 objects/s recovering 2023-10-09T04:10:16.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[221067]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[221067]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T04:10:16.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[221067]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[221067]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[221067]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.570 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:16 smithi163 ceph-mon[221067]: pgmap v487: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10117/170847 objects degraded (5.922%); 2841/170847 objects misplaced (1.663%); 19 MiB/s, 311 objects/s recovering 2023-10-09T04:10:16.571 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:16.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:16 smithi181 ceph-mon[196979]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:16 smithi181 ceph-mon[196979]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T04:10:16.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:16 smithi181 ceph-mon[196979]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:16 smithi181 ceph-mon[196979]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:16 smithi181 ceph-mon[196979]: failed to encode map e191 with expected crc 2023-10-09T04:10:16.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:16 smithi181 ceph-mon[196979]: pgmap v487: 99 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 10117/170847 objects degraded (5.922%); 2841/170847 objects misplaced (1.663%); 19 MiB/s, 311 objects/s recovering 2023-10-09T04:10:17.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:10:17.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:17.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:17.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:17.204 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:17.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:17.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:17.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:17.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:17.393 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 61758 61742 78.6859 61.3125 0.00305824 0.0126946 2023-10-09T04:10:17.404 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:17.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:17.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 10117/170847 objects degraded (5.922%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:17.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:17 smithi181 ceph-mon[196979]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T04:10:17.467 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:17.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:17.564 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:10:17.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:17.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 10117/170847 objects degraded (5.922%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:17 smithi163 ceph-mon[221067]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T04:10:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 10117/170847 objects degraded (5.922%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:17 smithi163 ceph-mon[213841]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T04:10:17.582 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:10:17.595 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:17.603 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:10:17.625 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:10:17.640 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:17.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:17.649 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:10:17.669 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:10:17.698 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:10:17.718 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:10:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.720 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.720 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:10:17.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:17.772 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:17.817 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:17.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:17.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:17.963 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:18.125 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:18.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:18.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:18.255 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:18.268 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:10:18.319 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:18.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:18.393 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 61953 61937 77.3569 12.1875 0.0148628 0.0129228 2023-10-09T04:10:18.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:18.469 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: pgmap v489: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 10117/179556 objects degraded (5.634%); 1579/179556 objects misplaced (0.879%); 19 MiB/s, 302 objects/s recovering 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3860451344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/478761654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:10:18.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:10:18.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:18.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:10:18.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:18 smithi181 ceph-mon[196979]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T04:10:18.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:18.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: pgmap v489: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 10117/179556 objects degraded (5.634%); 1579/179556 objects misplaced (0.879%); 19 MiB/s, 302 objects/s recovering 2023-10-09T04:10:18.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3860451344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/478761654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:10:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:10:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:18.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:10:18.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[213841]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T04:10:18.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: failed to encode map e192 with expected crc 2023-10-09T04:10:18.586 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: pgmap v489: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 10117/179556 objects degraded (5.634%); 1579/179556 objects misplaced (0.879%); 19 MiB/s, 302 objects/s recovering 2023-10-09T04:10:18.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3860451344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:18.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:18.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:18.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/478761654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:10:18.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:10:18.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:18.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:10:18.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:18 smithi163 ceph-mon[221067]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T04:10:18.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:18.711 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:18.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:18.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:18.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:18.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:18.895 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:19.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:19.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:19.394 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 63089 63073 77.2302 71 0.00717615 0.0129439 2023-10-09T04:10:19.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:19.499 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:10:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:10:19.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:10:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:10:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: failed to encode map e193 with expected crc 2023-10-09T04:10:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:10:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:10:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:20.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:20.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:20.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:20.284 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:20.330 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:20.394 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 64259 64243 77.1512 73.125 0.0181141 0.0129571 2023-10-09T04:10:20.402 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:20.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:20.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:20.560 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:20 smithi163 ceph-mon[213841]: pgmap v491: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 10117/181872 objects degraded (5.563%); 1579/181872 objects misplaced (0.868%); 15 MiB/s, 244 objects/s recovering 2023-10-09T04:10:20.561 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:20 smithi163 ceph-mon[221067]: pgmap v491: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 10117/181872 objects degraded (5.563%); 1579/181872 objects misplaced (0.868%); 15 MiB/s, 244 objects/s recovering 2023-10-09T04:10:20.571 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:20.684 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:20.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:20 smithi181 ceph-mon[196979]: pgmap v491: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 10117/181872 objects degraded (5.563%); 1579/181872 objects misplaced (0.868%); 15 MiB/s, 244 objects/s recovering 2023-10-09T04:10:20.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:20.830 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:20.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:10:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:10:20] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-10-09T04:10:20.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:20.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:20.959 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:21.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:21.097 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:21.259 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:21.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:21.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:21.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:21.394 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 65535 65519 77.2 79.75 0.0088388 0.0129482 2023-10-09T04:10:21.452 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:21.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:21.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:21.752 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:10:21.752 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:10:21.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:21.875 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:21.960 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:21.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:22.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:22.165 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:22.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:10:22.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:10:22.394 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 66753 66737 77.18 76.125 0.00223691 0.0129507 2023-10-09T04:10:22.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:22 smithi163 ceph-mon[221067]: pgmap v492: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 8634/188805 objects degraded (4.573%); 2162/188805 objects misplaced (1.145%); 22 MiB/s, 350 objects/s recovering 2023-10-09T04:10:22.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:22 smithi163 ceph-mon[221067]: 7.8 scrub starts 2023-10-09T04:10:22.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 8634/188805 objects degraded (4.573%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:22.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:22 smithi163 ceph-mon[213841]: pgmap v492: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 8634/188805 objects degraded (4.573%); 2162/188805 objects misplaced (1.145%); 22 MiB/s, 350 objects/s recovering 2023-10-09T04:10:22.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:22 smithi163 ceph-mon[213841]: 7.8 scrub starts 2023-10-09T04:10:22.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 8634/188805 objects degraded (4.573%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:22 smithi181 ceph-mon[196979]: pgmap v492: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 8634/188805 objects degraded (4.573%); 2162/188805 objects misplaced (1.145%); 22 MiB/s, 350 objects/s recovering 2023-10-09T04:10:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:22 smithi181 ceph-mon[196979]: 7.8 scrub starts 2023-10-09T04:10:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 8634/188805 objects degraded (4.573%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:23.396 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 67645 67629 76.7906 55.75 0.0124309 0.0130174 2023-10-09T04:10:23.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:23.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:23.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:10:23.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:23.570 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:23.590 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:23 smithi163 ceph-mon[213841]: 7.18 scrub starts 2023-10-09T04:10:23.590 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:23 smithi163 ceph-mon[221067]: 7.18 scrub starts 2023-10-09T04:10:23.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:23.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:23.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:23.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:23.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:23 smithi181 ceph-mon[196979]: 7.18 scrub starts 2023-10-09T04:10:23.801 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:23.960 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:23.992 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:23.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:23.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:24.013 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:24.098 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:24.155 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:24.225 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:24.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:24.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:24.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:24.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:24.363 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:24.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 68226 68210 76.0682 36.3125 0.00337519 0.0130042 2023-10-09T04:10:24.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:24.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:24.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:24.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:24.534 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:10:24.567 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: pgmap v493: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 8634/193515 objects degraded (4.462%); 2162/193515 objects misplaced (1.117%); 17 MiB/s, 270 objects/s recovering 2023-10-09T04:10:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1590749234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: pgmap v493: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 8634/193515 objects degraded (4.462%); 2162/193515 objects misplaced (1.117%); 17 MiB/s, 270 objects/s recovering 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1590749234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:10:24.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: pgmap v493: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 8634/193515 objects degraded (4.462%); 2162/193515 objects misplaced (1.117%); 17 MiB/s, 270 objects/s recovering 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: failed to encode map e194 with expected crc 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1590749234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:10:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:10:25.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 69243 69227 75.8486 63.5625 0.0102926 0.013179 2023-10-09T04:10:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[213841]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[213841]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:10:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[213841]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:10:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[213841]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[213841]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[221067]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[221067]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:10:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[221067]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:10:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[221067]: failed to encode map e195 with expected crc 2023-10-09T04:10:25.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:25 smithi163 ceph-mon[221067]: failed to encode map e195 with expected crc 2023-10-09T04:10:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:25 smithi181 ceph-mon[196979]: failed to encode map e195 with expected crc 2023-10-09T04:10:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:25 smithi181 ceph-mon[196979]: failed to encode map e195 with expected crc 2023-10-09T04:10:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:25 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:10:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:25 smithi181 ceph-mon[196979]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:10:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:25 smithi181 ceph-mon[196979]: failed to encode map e195 with expected crc 2023-10-09T04:10:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:25 smithi181 ceph-mon[196979]: failed to encode map e195 with expected crc 2023-10-09T04:10:26.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:26.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:26.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:26.370 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:26.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 70127 70111 75.4936 55.25 0.0139559 0.0132413 2023-10-09T04:10:26.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:26 smithi163 ceph-mon[221067]: 7.10 scrub starts 2023-10-09T04:10:26.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:26 smithi163 ceph-mon[221067]: 7.1 scrub starts 2023-10-09T04:10:26.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:26 smithi163 ceph-mon[221067]: pgmap v496: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s; 5704/200283 objects degraded (2.848%); 4335/200283 objects misplaced (2.164%); 14 MiB/s, 225 objects/s recovering 2023-10-09T04:10:26.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:26 smithi163 ceph-mon[213841]: 7.10 scrub starts 2023-10-09T04:10:26.839 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:26 smithi163 ceph-mon[213841]: 7.1 scrub starts 2023-10-09T04:10:26.839 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:26 smithi163 ceph-mon[213841]: pgmap v496: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s; 5704/200283 objects degraded (2.848%); 4335/200283 objects misplaced (2.164%); 14 MiB/s, 225 objects/s recovering 2023-10-09T04:10:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:26 smithi181 ceph-mon[196979]: 7.10 scrub starts 2023-10-09T04:10:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:26 smithi181 ceph-mon[196979]: 7.1 scrub starts 2023-10-09T04:10:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:26 smithi181 ceph-mon[196979]: pgmap v496: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s; 5704/200283 objects degraded (2.848%); 4335/200283 objects misplaced (2.164%); 14 MiB/s, 225 objects/s recovering 2023-10-09T04:10:27.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 71091 71075 75.2353 60.25 0.00294803 0.0132714 2023-10-09T04:10:27.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5704/200283 objects degraded (2.848%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:27.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5704/200283 objects degraded (2.848%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:27.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:27.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:27.909 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:27.945 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5704/200283 objects degraded (2.848%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:27.947 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:28.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:28.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:28.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:28.143 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:28.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:28.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:28.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:28.317 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:28.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:10:28.395941+0000 min lat: 0.0010512 max lat: 2.18233 avg lat: 0.0133295 2023-10-09T04:10:28.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:10:28.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 71894 71878 74.818 50.1875 0.0036352 0.0133295 2023-10-09T04:10:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:28 smithi163 ceph-mon[221067]: 7.13 scrub starts 2023-10-09T04:10:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:28 smithi163 ceph-mon[221067]: pgmap v497: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 5704/208380 objects degraded (2.737%); 4335/208380 objects misplaced (2.080%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:10:28.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:28 smithi163 ceph-mon[213841]: 7.13 scrub starts 2023-10-09T04:10:28.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:28 smithi163 ceph-mon[213841]: pgmap v497: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 5704/208380 objects degraded (2.737%); 4335/208380 objects misplaced (2.080%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:10:28.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:28.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:28.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:29.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:28 smithi181 ceph-mon[196979]: 7.13 scrub starts 2023-10-09T04:10:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:28 smithi181 ceph-mon[196979]: pgmap v497: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 5704/208380 objects degraded (2.737%); 4335/208380 objects misplaced (2.080%); 13 MiB/s, 202 objects/s recovering 2023-10-09T04:10:29.028 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:29.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 72911 72895 74.6335 63.5625 0.00182399 0.0133855 2023-10-09T04:10:29.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:10:29.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:29.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:29.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:29.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:29 smithi163 ceph-mon[213841]: 7.8 scrub starts 2023-10-09T04:10:29.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:29 smithi163 ceph-mon[221067]: 7.8 scrub starts 2023-10-09T04:10:29.786 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:29.844 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:29.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:29.851 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:29 smithi181 ceph-mon[196979]: 7.8 scrub starts 2023-10-09T04:10:29.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:29.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:30.039 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:30.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:30.347 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:10:30.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:30.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:30.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 73823 73807 74.3491 57 0.0172185 0.0134438 2023-10-09T04:10:30.396 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:10:30.441 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:10:30.463 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:30.485 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:10:30.497 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:30.535 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:10:30.550 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:30.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:30.586 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:10:30.620 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:10:30.632 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:30 smithi163 ceph-mon[221067]: pgmap v498: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 5704/210828 objects degraded (2.706%); 3492/210828 objects misplaced (1.656%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:10:30.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3585139131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:30.633 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:30 smithi163 ceph-mon[213841]: pgmap v498: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 5704/210828 objects degraded (2.706%); 3492/210828 objects misplaced (1.656%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:10:30.633 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3585139131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:30.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:30.662 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:10:30.681 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.681 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.682 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:10:30.750 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:30 smithi181 ceph-mon[196979]: pgmap v498: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 5704/210828 objects degraded (2.706%); 3492/210828 objects misplaced (1.656%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:10:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3585139131' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:30.792 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181790 6.12 6:483ee06b:::benchmark_data_smithi163_287034_object73527:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:10:30.925 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.714039727, 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2147281759999999, 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181790 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206414+0000", 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.855e-06 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206419+0000", 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7840000000000003e-06 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206423+0000", 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5449999999999997e-06 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207972+0000", 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001549031 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207978+0000", 2023-10-09T04:10:30.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9889999999999997e-06 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388116+0000", 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.180137601 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for rw locks", 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388262+0000", 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145642 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408226+0000", 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019964064 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408317+0000", 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1717000000000003e-05 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:10:30.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408461+0000", 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014404700000000001 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418786+0000", 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010325051 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420556+0000", 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017692529999999999 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420629+0000", 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2847000000000002e-05 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420693+0000", 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4324000000000003e-05 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.421141+0000", 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044771999999999998 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181893 6.12 6:48514e0f:::benchmark_data_smithi163_287034_object73630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.71395211500001, 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2117616529999999, 2023-10-09T04:10:30.934 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181893 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206502+0000", 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0099999999999998e-06 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206513+0000", 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206517+0000", 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5499999999999996e-06 2023-10-09T04:10:30.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208079+0000", 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015617529999999999 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208087+0000", 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5059999999999998e-06 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395090+0000", 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1870032690000001 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395194+0000", 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010448 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395313+0000", 2023-10-09T04:10:30.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011805 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400612+0000", 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052990609999999999 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407098+0000", 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064860760000000003 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417540+0000", 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010442168999999999 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417604+0000", 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4577000000000002e-05 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418262+0000", 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00065729299999999998 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181924 6.12 6:4956f571:::benchmark_data_smithi163_287034_object73661:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.71391097700001, 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.212897227, 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.942 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181924 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206543+0000", 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0640000000000001e-06 2023-10-09T04:10:30.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206549+0000", 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6949999999999997e-06 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206554+0000", 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208118+0000", 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015642570000000001 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208123+0000", 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6370000000000004e-06 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397380+0000", 2023-10-09T04:10:30.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1892568610000001 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397556+0000", 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017565199999999999 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397712+0000", 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015616 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400663+0000", 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002950664 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407160+0000", 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064973219999999998 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418821+0000", 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011661227 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418878+0000", 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6666999999999998e-05 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419438+0000", 2023-10-09T04:10:30.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056037000000000003 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181928 6.12 6:4a3131f6:::benchmark_data_smithi163_287034_object73665:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.71388024699999, 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2134125929999999, 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.950 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181928 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206574+0000", 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0389999999999999e-06 2023-10-09T04:10:30.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206579+0000", 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206585+0000", 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6330000000000001e-06 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208144+0000", 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015595439999999999 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208151+0000", 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400687+0000", 2023-10-09T04:10:30.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.192535828 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400844+0000", 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015750000000000001 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.401023+0000", 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017885000000000001 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407172+0000", 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006149374 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407223+0000", 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0392999999999997e-05 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419479+0000", 2023-10-09T04:10:30.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01225652 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419533+0000", 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3735e-05 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419985+0000", 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045140000000000002 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.957 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181971 6.13 6:ce89c71c:::benchmark_data_smithi163_287034_object73708:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70417346400001, 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.344004741, 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181971 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:10:30.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216277+0000", 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216286+0000", 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7050000000000005e-06 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216564+0000", 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027831900000000002 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216577+0000", 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.29e-05 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217701+0000", 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001124002 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217710+0000", 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6999999999999997e-06 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.538610+0000", 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3209002599999999 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.538722+0000", 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112537 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.539153+0000", 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000430362 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542629+0000", 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034758459999999999 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.545992+0000", 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033635409999999998 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552225+0000", 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062325189999999997 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.559513+0000", 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072881919999999998 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.559580+0000", 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7296000000000001e-05 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560283+0000", 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070328800000000005 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181975 6.13 6:caf04be6:::benchmark_data_smithi163_287034_object73712:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70352708499999, 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3577086709999999, 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.966 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.967 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.967 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.967 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.967 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181975 2023-10-09T04:10:30.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216924+0000", 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216930+0000", 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8719999999999999e-06 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216938+0000", 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8380000000000008e-06 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216944+0000", 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4310000000000002e-06 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217731+0000", 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078680899999999997 2023-10-09T04:10:30.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217736+0000", 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7459999999999997e-06 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542662+0000", 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3249255600000001 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542848+0000", 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000186174 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.543402+0000", 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000553489 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552236+0000", 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0088346299999999996 2023-10-09T04:10:30.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552280+0000", 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3537e-05 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560349+0000", 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080691140000000005 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.573767+0000", 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013417861 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.573826+0000", 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9379999999999997e-05 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574634+0000", 2023-10-09T04:10:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080766300000000002 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181980 6.13 6:cdaa0fdf:::benchmark_data_smithi163_287034_object73717:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.703503337, 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3655362260000001, 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181980 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216948+0000", 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216951+0000", 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6579999999999998e-06 2023-10-09T04:10:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216958+0000", 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5350000000000001e-06 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216963+0000", 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3650000000000003e-06 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217756+0000", 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00079339899999999997 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217762+0000", 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6180000000000003e-06 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546040+0000", 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.328277887 2023-10-09T04:10:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546236+0000", 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019608900000000001 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546794+0000", 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055758299999999995 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552243+0000", 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054494230000000001 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552318+0000", 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5481000000000005e-05 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566793+0000", 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014474614 2023-10-09T04:10:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581590+0000", 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014796659 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581645+0000", 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5427999999999999e-05 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582485+0000", 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083988900000000002 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182015 6.13 6:cae1eea5:::benchmark_data_smithi163_287034_object73752:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70348481600001, 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3664138910000001, 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182015 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216970+0000", 2023-10-09T04:10:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5299999999999999e-06 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216975+0000", 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3689999999999998e-06 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216979+0000", 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5589999999999999e-06 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217782+0000", 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080258200000000003 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217788+0000", 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.502e-06 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.548975+0000", 2023-10-09T04:10:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3311876169999999 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.549135+0000", 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015948399999999999 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.549566+0000", 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043101400000000002 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552286+0000", 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027206219999999998 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552369+0000", 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3139000000000005e-05 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566902+0000", 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014533001 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582622+0000", 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015720073000000001 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582685+0000", 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2966999999999995e-05 2023-10-09T04:10:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583381+0000", 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000695814 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182032 6.13 6:cff9d57a:::benchmark_data_smithi163_287034_object73769:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70346899699999, 2023-10-09T04:10:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3672618480000001, 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182032 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:10:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216985+0000", 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1789999999999998e-06 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216990+0000", 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3109999999999996e-06 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216995+0000", 2023-10-09T04:10:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9280000000000001e-06 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217808+0000", 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00081240700000000002 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217813+0000", 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4009999999999996e-06 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552391+0000", 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3345780999999999 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552538+0000", 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014700800000000001 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.553135+0000", 2023-10-09T04:10:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059664400000000002 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563747+0000", 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010612430000000001 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566840+0000", 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030923819999999999 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574748+0000", 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079079549999999995 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583498+0000", 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087501799999999998 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583544+0000", 2023-10-09T04:10:31.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6165999999999998e-05 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584245+0000", 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070112899999999999 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182044 6.13 6:c8672a95:::benchmark_data_smithi163_287034_object73781:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.001 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70345315500001, 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.367953709, 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182044 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217001+0000", 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9290000000000001e-06 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217007+0000", 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1700000000000002e-06 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217011+0000", 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3850000000000004e-06 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217834+0000", 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082231700000000003 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217839+0000", 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:10:31.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556028+0000", 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.338188543 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556217+0000", 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189452 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556804+0000", 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058719899999999995 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563759+0000", 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069549829999999997 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566968+0000", 2023-10-09T04:10:31.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003208598 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582584+0000", 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015615655000000001 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584293+0000", 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017097309999999999 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584348+0000", 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4750999999999998e-05 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584953+0000", 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060460600000000005 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182130 6.13 6:ce90ea3b:::benchmark_data_smithi163_287034_object73867:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70343714699999, 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.376683477, 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182130 2023-10-09T04:10:31.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217017+0000", 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.464e-06 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217021+0000", 2023-10-09T04:10:31.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6940000000000001e-06 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217024+0000", 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2760000000000001e-06 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217860+0000", 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083570200000000002 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217865+0000", 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5400000000000003e-06 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560446+0000", 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.342580544 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560636+0000", 2023-10-09T04:10:31.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019010500000000001 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.561233+0000", 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059681600000000001 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566854+0000", 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056207490000000004 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567010+0000", 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015621499999999999 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583462+0000", 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016452205000000001 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.592834+0000", 2023-10-09T04:10:31.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093721150000000003 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.592892+0000", 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7810999999999999e-05 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593699+0000", 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080657999999999999 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.017 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182187 6.13 6:c86ec581:::benchmark_data_smithi163_287034_object73924:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70342575000001, 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3896110020000001, 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.018 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182187 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:10:31.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217028+0000", 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.906e-06 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217031+0000", 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.095e-06 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217034+0000", 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6529999999999998e-06 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217885+0000", 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00085116200000000003 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217891+0000", 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4920000000000001e-06 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563798+0000", 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3459071169999999 2023-10-09T04:10:31.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563972+0000", 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017474299999999999 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.564543+0000", 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00057071800000000001 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567022+0000", 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024785779999999999 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567079+0000", 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7258e-05 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597561+0000", 2023-10-09T04:10:31.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.030482183999999999 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.605598+0000", 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080370560000000008 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.605665+0000", 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7268999999999996e-05 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.606637+0000", 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097197400000000002 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182188 6.13 6:cef9c490:::benchmark_data_smithi163_287034_object73925:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.703415151, 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4008471199999999, 2023-10-09T04:10:31.026 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182188 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217039+0000", 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8160000000000001e-06 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217044+0000", 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.169e-06 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217048+0000", 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1980000000000003e-06 2023-10-09T04:10:31.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217910+0000", 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086211000000000002 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217915+0000", 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2850000000000002e-06 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567119+0000", 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3492036760000001 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567314+0000", 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000195353 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567922+0000", 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060742399999999996 2023-10-09T04:10:31.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574701+0000", 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067789490000000003 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.577940+0000", 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032392279999999998 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601664+0000", 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023723782999999998 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.616941+0000", 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015276853 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617000+0000", 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9185999999999999e-05 2023-10-09T04:10:31.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617884+0000", 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088436199999999997 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182235 6.13 6:cbfff063:::benchmark_data_smithi163_287034_object73972:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.034 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.703400991, 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4014693499999999, 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182235 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217053+0000", 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3099999999999999e-06 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217059+0000", 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3759999999999999e-06 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217063+0000", 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.318e-06 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217936+0000", 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00087284899999999998 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217941+0000", 2023-10-09T04:10:31.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.643e-06 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570245+0000", 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3523035450000001 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570415+0000", 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016968699999999999 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570978+0000", 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056366499999999998 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.577951+0000", 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069725350000000002 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581550+0000", 2023-10-09T04:10:31.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035993399999999999 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601696+0000", 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020146045000000001 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617929+0000", 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016232449999999999 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617984+0000", 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5387999999999999e-05 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618521+0000", 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053653999999999996 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182259 6.13 6:ca05850d:::benchmark_data_smithi163_287034_object73996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70338540200001, 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4019898450000001, 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182259 2023-10-09T04:10:31.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217066+0000", 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217069+0000", 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.227e-06 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217074+0000", 2023-10-09T04:10:31.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6829999999999996e-06 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217079+0000", 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2880000000000003e-06 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217961+0000", 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088278599999999999 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217967+0000", 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574785+0000", 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3568176510000001 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574950+0000", 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000164869 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.575563+0000", 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061337800000000004 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582540+0000", 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069775280000000002 2023-10-09T04:10:31.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593712+0000", 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011171096 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.606689+0000", 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012977087 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618561+0000", 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011872297 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618604+0000", 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3266999999999998e-05 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619057+0000", 2023-10-09T04:10:31.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045241999999999998 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182286 6.13 6:c82cebba:::benchmark_data_smithi163_287034_object74023:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70336982399999, 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4024858179999999, 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.051 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182286 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217089+0000", 2023-10-09T04:10:31.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1029999999999996e-06 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217096+0000", 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4010000000000002e-06 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217101+0000", 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217987+0000", 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088635799999999996 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217993+0000", 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8619999999999999e-06 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.578004+0000", 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.360011112 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.578616+0000", 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061207500000000001 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.579171+0000", 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055508900000000004 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583427+0000", 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.004255549 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593719+0000", 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010292066000000001 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.610148+0000", 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016429684 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619093+0000", 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0089444580000000006 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619134+0000", 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0794000000000003e-05 2023-10-09T04:10:31.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619568+0000", 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043460500000000002 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182384 6.13 6:cc63ec28:::benchmark_data_smithi163_287034_object74121:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:10:31.059 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70334811000001, 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4029806759999999, 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182384 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:10:31.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217106+0000", 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1689999999999999e-06 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217113+0000", 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9739999999999996e-06 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217117+0000", 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4499999999999997e-06 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218013+0000", 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00089540900000000001 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218019+0000", 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584986+0000", 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3669674590000001 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.585163+0000", 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000176995 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.064 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.585767+0000", 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060421699999999999 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593726+0000", 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079584719999999994 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597514+0000", 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037886840000000001 2023-10-09T04:10:31.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.614382+0000", 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016867727999999999 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619605+0000", 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005223073 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619647+0000", 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2032e-05 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.620085+0000", 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043757399999999998 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182402 6.13 6:cc13d46a:::benchmark_data_smithi163_287034_object74139:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70333128799999, 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.407380751, 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182402 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.068 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217123+0000", 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0609999999999998e-06 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217128+0000", 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5969999999999999e-06 2023-10-09T04:10:31.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217133+0000", 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.34e-06 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218039+0000", 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090607999999999997 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218044+0000", 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.589640+0000", 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3715955420000001 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.589766+0000", 2023-10-09T04:10:31.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126234 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.590263+0000", 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000496529 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597468+0000", 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072057340000000001 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601593+0000", 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041247039999999999 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.620123+0000", 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018529468 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.623617+0000", 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034940879999999998 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.623677+0000", 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0062000000000002e-05 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624502+0000", 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000824952 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.075 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182403 6.13 6:ccb27798:::benchmark_data_smithi163_287034_object74140:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70331462199999, 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.407780327, 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.076 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182403 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:10:31.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217137+0000", 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217139+0000", 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2689999999999998e-06 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217145+0000", 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7980000000000002e-06 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217150+0000", 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4229999999999998e-06 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218064+0000", 2023-10-09T04:10:31.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00091458200000000005 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218070+0000", 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4979999999999997e-06 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593753+0000", 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.375683465 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593914+0000", 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016051499999999999 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.594483+0000", 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056889400000000002 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597523+0000", 2023-10-09T04:10:31.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030404329999999999 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601631+0000", 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041084379999999998 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624529+0000", 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.022897438999999999 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624548+0000", 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8601999999999999e-05 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624576+0000", 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8637999999999999e-05 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624918+0000", 2023-10-09T04:10:31.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00034171100000000002 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182407 6.13 6:cd9f07b1:::benchmark_data_smithi163_287034_object74144:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 139.70329870899999, 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4088424019999999, 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:10:31.084 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182407 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:31.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217155+0000", 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2699999999999999e-06 2023-10-09T04:10:31.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217161+0000", 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217166+0000", 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6439999999999999e-06 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218093+0000", 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092731699999999999 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218100+0000", 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7399999999999998e-06 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597985+0000", 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.37988534 2023-10-09T04:10:31.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.598147+0000", 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000162189 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.598729+0000", 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058185599999999995 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601789+0000", 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003059712 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.603630+0000", 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001841051 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624943+0000", 2023-10-09T04:10:31.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021312827999999999 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625489+0000", 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00054580099999999995 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625535+0000", 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6610000000000003e-05 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:31.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625996+0000", 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000460715 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:31.092 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:10:31.093 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:10:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:10:30] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-10-09T04:10:31.395 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 74837 74821 74.175 63.375 0.00800048 0.0134765 2023-10-09T04:10:31.616 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:10:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:31 smithi181 ceph-mon[196979]: 7.18 scrub starts 2023-10-09T04:10:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3901554220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:32.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:31 smithi163 ceph-mon[213841]: 7.18 scrub starts 2023-10-09T04:10:32.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3901554220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:32.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:32.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:31 smithi163 ceph-mon[221067]: 7.18 scrub starts 2023-10-09T04:10:32.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3901554220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:32.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:32.398 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 75792 75776 73.9487 59.6875 0.00879516 0.0135183 2023-10-09T04:10:32.460 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:32.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:32.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:32.604 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:32.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:32.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:32.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:32.799 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: 7.3 scrub starts 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: pgmap v499: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5704/217716 objects degraded (2.620%); 3492/217716 objects misplaced (1.604%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5704/217716 objects degraded (2.620%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: 7.3 scrub starts 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: pgmap v499: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5704/217716 objects degraded (2.620%); 3492/217716 objects misplaced (1.604%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5704/217716 objects degraded (2.620%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:32.868 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:32.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: 7.3 scrub starts 2023-10-09T04:10:32.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: pgmap v499: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 5704/217716 objects degraded (2.620%); 3492/217716 objects misplaced (1.604%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:10:32.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:10:32.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:32.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:10:32.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: failed to encode map e196 with expected crc 2023-10-09T04:10:32.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5704/217716 objects degraded (2.620%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:32.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:32.978 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:33.047 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:33.396 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 76892 76876 73.8681 68.75 0.0194173 0.0135328 2023-10-09T04:10:33.521 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:33.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:33.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:33.669 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:33.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:33.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:33.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:33.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:33 smithi163 ceph-mon[213841]: pgmap v501: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 5704/222024 objects degraded (2.569%); 3492/222024 objects misplaced (1.573%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T04:10:33.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:33 smithi163 ceph-mon[221067]: pgmap v501: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 5704/222024 objects degraded (2.569%); 3492/222024 objects misplaced (1.573%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T04:10:33.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:33.882 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:33 smithi181 ceph-mon[196979]: pgmap v501: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 5704/222024 objects degraded (2.569%); 3492/222024 objects misplaced (1.573%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T04:10:33.884 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:33.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:10:33.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:33.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:34.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:34.085 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:34.153 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:34.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:34.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:34.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:34.354 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:34.396 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 77833 77817 73.6401 58.8125 0.0168968 0.0135748 2023-10-09T04:10:34.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:34.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:34.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:34.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:34.766 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:34.870 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:34.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:34.977 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:35.008 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:35.049 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:35.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:35.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:35.152 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:35.192 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:35.257 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:35.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:35.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:35.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[213841]: 7.10 scrub starts 2023-10-09T04:10:35.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[213841]: 7.1 scrub starts 2023-10-09T04:10:35.280 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:35.281 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:35.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:35.397 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 78742 78726 73.3877 56.8125 0.0160601 0.0136215 2023-10-09T04:10:35.411 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:35.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:35.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:35 smithi181 ceph-mon[196979]: 7.10 scrub starts 2023-10-09T04:10:35.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:35 smithi181 ceph-mon[196979]: 7.1 scrub starts 2023-10-09T04:10:35.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:35.478 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:35.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:35.565 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[221067]: 7.10 scrub starts 2023-10-09T04:10:35.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[221067]: 7.1 scrub starts 2023-10-09T04:10:35.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:35.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:35.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:35.614 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:35.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:35.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:35.708 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:35.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:35.859 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:35.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:10:35.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:35.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:36.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:36.101 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:36.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:36.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:36.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:36.294 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:36.331 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:10:36.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:36.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:36 smithi181 ceph-mon[196979]: pgmap v502: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 5704/228546 objects degraded (2.496%); 2787/228546 objects misplaced (1.219%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:10:36.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:36.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:36.396 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:36 smithi163 ceph-mon[221067]: pgmap v502: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 5704/228546 objects degraded (2.496%); 2787/228546 objects misplaced (1.219%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:10:36.397 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:36 smithi163 ceph-mon[213841]: pgmap v502: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 5704/228546 objects degraded (2.496%); 2787/228546 objects misplaced (1.219%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:10:36.398 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 79718 79702 73.2054 61 0.0131641 0.0136557 2023-10-09T04:10:36.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:36.485 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:36.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:36.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:36.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:36.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:36.680 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:36.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:36.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:36.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:36.913 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:37.307 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:10:37.398 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 80493 80477 72.8465 48.4375 0.00203827 0.0136759 2023-10-09T04:10:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2223060763' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:10:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5704/228546 objects degraded (2.496%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:37.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2223060763' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:10:37.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5704/228546 objects degraded (2.496%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:37.711 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:37.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:37.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2223060763' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:10:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5704/228546 objects degraded (2.496%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:37.814 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:37.841 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:38.235 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:10:38.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:38.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:38.288 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:38.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:38.389 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:38.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 80519 80503 71.8238 1.625 0.00241057 0.0137141 2023-10-09T04:10:38.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[213841]: 7.8 scrub starts 2023-10-09T04:10:38.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[213841]: failed to encode map e197 with expected crc 2023-10-09T04:10:38.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2223060763' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:10:38.413 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[213841]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:10:38.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[213841]: pgmap v504: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5704/236772 objects degraded (2.409%); 2787/236772 objects misplaced (1.177%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:10:38.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[221067]: 7.8 scrub starts 2023-10-09T04:10:38.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[221067]: failed to encode map e197 with expected crc 2023-10-09T04:10:38.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2223060763' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:10:38.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[221067]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:10:38.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:38 smithi163 ceph-mon[221067]: pgmap v504: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5704/236772 objects degraded (2.409%); 2787/236772 objects misplaced (1.177%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:10:38.442 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:38.676 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:38.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:38.743 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:10:38.743 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:10:38.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80498 7.13 7.ee42a673 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.423886596, 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.42391890799999998, 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "delayed", 2023-10-09T04:10:38.744 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:38.745 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:38.745 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:38.745 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80498 2023-10-09T04:10:38.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.746 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:38.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:38.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:10:38.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:38.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:10:38.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317838+0000", 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317881+0000", 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3013000000000003e-05 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317895+0000", 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3732e-05 2023-10-09T04:10:38.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317903+0000", 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5209999999999997e-06 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318091+0000", 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188829 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318101+0000", 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.55e-06 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323224+0000", 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051229439999999999 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:38.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323236+0000", 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992e-05 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80508 7.13 7.45c1d9f3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:10:38.753 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.42381048300000002, 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.42393605099999998, 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "delayed", 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:38.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80508 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:38.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317916+0000", 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317944+0000", 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7642000000000001e-05 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317952+0000", 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3070000000000003e-06 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:38.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317958+0000", 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1290000000000001e-06 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318152+0000", 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019322999999999999 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318158+0000", 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7700000000000004e-06 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323259+0000", 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051002950000000003 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323278+0000", 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814999999999999e-05 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:38.761 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80516 7.13 7.b9032593 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.42376372699999998, 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.42397385599999998, 2023-10-09T04:10:38.762 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "delayed", 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80516 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:38.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:38.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317963+0000", 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317983+0000", 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9938000000000001e-05 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317990+0000", 2023-10-09T04:10:38.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7850000000000003e-06 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317993+0000", 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3680000000000002e-06 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318180+0000", 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018708599999999999 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318187+0000", 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2589999999999997e-06 2023-10-09T04:10:38.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323302+0000", 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051155979999999998 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:38.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323310+0000", 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0660000000000004e-06 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.770 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80551 7.13 7.d734a193 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.271253299, 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27155731700000002, 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "delayed", 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:38.771 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80551 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:10:38.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:38.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:10:38.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:38.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470472+0000", 2023-10-09T04:10:38.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:10:38.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470498+0000", 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5672000000000001e-05 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:38.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470514+0000", 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5846999999999999e-05 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470522+0000", 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9629999999999998e-06 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470576+0000", 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4360999999999997e-05 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:38.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470594+0000", 2023-10-09T04:10:38.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7754999999999999e-05 2023-10-09T04:10:38.778 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.778 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:38.778 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.779 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80553 7.3 7.f2360603 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:10:38.779 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:10:38.779 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.26806317600000001, 2023-10-09T04:10:38.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.268437392, 2023-10-09T04:10:38.779 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:38.779 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "delayed", 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80553 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:38.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:38.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473663+0000", 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:10:38.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473684+0000", 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1585000000000001e-05 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:38.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473699+0000", 2023-10-09T04:10:38.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4456e-05 2023-10-09T04:10:38.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:38.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473705+0000", 2023-10-09T04:10:38.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6630000000000004e-06 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473767+0000", 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1607999999999997e-05 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473779+0000", 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2330999999999999e-05 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.786 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80558 7.13 7.c62f5c13 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.26216730300000002, 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26261092899999999, 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "delayed", 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80558 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:10:38.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:10:38.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479559+0000", 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:10:38.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:10:38.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479582+0000", 2023-10-09T04:10:38.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2878999999999999e-05 2023-10-09T04:10:38.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:10:38.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479594+0000", 2023-10-09T04:10:38.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1328000000000001e-05 2023-10-09T04:10:38.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:10:38.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479601+0000", 2023-10-09T04:10:38.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3690000000000001e-06 2023-10-09T04:10:38.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479661+0000", 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0053000000000003e-05 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479673+0000", 2023-10-09T04:10:38.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1877e-05 2023-10-09T04:10:38.794 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.794 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:10:38.794 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.794 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:10:38.794 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:10:38.795 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 6 2023-10-09T04:10:38.795 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:10:38.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:38.796 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:38 smithi181 ceph-mon[196979]: 7.8 scrub starts 2023-10-09T04:10:38.796 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:38 smithi181 ceph-mon[196979]: failed to encode map e197 with expected crc 2023-10-09T04:10:38.796 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2223060763' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:10:38.796 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:38 smithi181 ceph-mon[196979]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:10:38.797 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:38 smithi181 ceph-mon[196979]: pgmap v504: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5704/236772 objects degraded (2.409%); 2787/236772 objects misplaced (1.177%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:10:38.834 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:39.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 80598 80582 70.8824 4.9375 0.00293327 0.0138409 2023-10-09T04:10:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: 7.1 scrub starts 2023-10-09T04:10:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:10:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[213841]: 7.1 scrub ok 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: 7.1 scrub starts 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:39 smithi163 ceph-mon[221067]: 7.1 scrub ok 2023-10-09T04:10:39.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: 7.1 scrub starts 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: failed to encode map e198 with expected crc 2023-10-09T04:10:39.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:39 smithi181 ceph-mon[196979]: 7.1 scrub ok 2023-10-09T04:10:39.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:39.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:40.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:40.098 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:40.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:40.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:40.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:40.273 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:40.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:40.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:40.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:10:40.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: 7.13 starting backfill to osd.2 from (0'0,179'2620] MIN to 196'3689 2023-10-09T04:10:40.322 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: 7.3 starting backfill to osd.2 from (0'0,179'2634] MIN to 196'3730 2023-10-09T04:10:40.322 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: 7.15 starting backfill to osd.2 from (0'0,179'2540] MIN to 196'3648 2023-10-09T04:10:40.322 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[213841]: pgmap v507: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 4206/238548 objects degraded (1.763%); 3792/238548 objects misplaced (1.590%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:10:40.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 80790 80774 70.0651 12 0.0052635 0.0140226 2023-10-09T04:10:40.421 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:40.451 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:40.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: 7.13 starting backfill to osd.2 from (0'0,179'2620] MIN to 196'3689 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: 7.3 starting backfill to osd.2 from (0'0,179'2634] MIN to 196'3730 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: 7.15 starting backfill to osd.2 from (0'0,179'2540] MIN to 196'3648 2023-10-09T04:10:40.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:40 smithi181 ceph-mon[196979]: pgmap v507: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 4206/238548 objects degraded (1.763%); 3792/238548 objects misplaced (1.590%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: failed to encode map e199 with expected crc 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: 7.13 starting backfill to osd.2 from (0'0,179'2620] MIN to 196'3689 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: 7.3 starting backfill to osd.2 from (0'0,179'2634] MIN to 196'3730 2023-10-09T04:10:40.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: 7.15 starting backfill to osd.2 from (0'0,179'2540] MIN to 196'3648 2023-10-09T04:10:40.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:40 smithi163 ceph-mon[221067]: pgmap v507: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 4206/238548 objects degraded (1.763%); 3792/238548 objects misplaced (1.590%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:10:40.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:10:40.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:40.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:40.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:40.886 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:10:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:10:40] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-10-09T04:10:40.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:40.989 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:41.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:41.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:41.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:41.163 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:41.205 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:41.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:41.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:41.336 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:41.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:41.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:41.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:41.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 80790 80774 69.1059 0 - 0.0140226 2023-10-09T04:10:41.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:41.510 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:41.689 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:41.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:41.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:41.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:41.841 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:41.844 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:41.886 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:41.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:41.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:41.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:42.018 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:42.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:42.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:42.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:42.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:42.226 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:10:42.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:42.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:42.271 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:10:42.314 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:10:42.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:10:42.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:42.383 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:42.397 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:10:42.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 15 81040 81025 68.3844 7.84375 0.0128713 0.0145494 2023-10-09T04:10:42.430 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:10:42.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:42.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:42.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:42.467 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:10:42.498 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:10:42.520 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:10:42.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:10:42.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:42.615 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:42.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:42 smithi181 ceph-mon[196979]: pgmap v508: 99 pgs: 3 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19 MiB/s wr, 307 op/s; 3315/241611 objects degraded (1.372%); 11433/241611 objects misplaced (4.732%); 363 KiB/s, 5 objects/s recovering 2023-10-09T04:10:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/135866980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:10:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:42 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:10:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1083481840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3315/241611 objects degraded (1.372%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:42.769 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[213841]: pgmap v508: 99 pgs: 3 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19 MiB/s wr, 307 op/s; 3315/241611 objects degraded (1.372%); 11433/241611 objects misplaced (4.732%); 363 KiB/s, 5 objects/s recovering 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/135866980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1083481840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3315/241611 objects degraded (1.372%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[221067]: pgmap v508: 99 pgs: 3 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19 MiB/s wr, 307 op/s; 3315/241611 objects degraded (1.372%); 11433/241611 objects misplaced (4.732%); 363 KiB/s, 5 objects/s recovering 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/135866980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1083481840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:42.770 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3315/241611 objects degraded (1.372%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:42.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:42.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:43.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:43.151 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:43.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 81849 81833 68.1461 50.5 0.00899532 0.0146701 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[221067]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[221067]: failed to encode map e200 with expected crc 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[221067]: failed to encode map e200 with expected crc 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:10:43.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[213841]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:10:43.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[213841]: failed to encode map e200 with expected crc 2023-10-09T04:10:43.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:43 smithi163 ceph-mon[213841]: failed to encode map e200 with expected crc 2023-10-09T04:10:44.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:43 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:43 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:10:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:43 smithi181 ceph-mon[196979]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:10:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:43 smithi181 ceph-mon[196979]: failed to encode map e200 with expected crc 2023-10-09T04:10:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:43 smithi181 ceph-mon[196979]: failed to encode map e200 with expected crc 2023-10-09T04:10:44.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 82391 82375 67.6954 33.875 0.00257531 0.0146734 2023-10-09T04:10:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: pgmap v510: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 5870/242421 objects degraded (2.421%); 8646/242421 objects misplaced (3.567%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:10:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: osdmap e201: 8 total, 8 up, 8 in 2023-10-09T04:10:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[213841]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: pgmap v510: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 5870/242421 objects degraded (2.421%); 8646/242421 objects misplaced (3.567%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:10:44.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: osdmap e201: 8 total, 8 up, 8 in 2023-10-09T04:10:44.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: failed to encode map e201 with expected crc 2023-10-09T04:10:44.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:44 smithi163 ceph-mon[221067]: failed to encode map e201 with expected crc 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: pgmap v510: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 5870/242421 objects degraded (2.421%); 8646/242421 objects misplaced (3.567%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: failed to encode map e201 with expected crc 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: osdmap e201: 8 total, 8 up, 8 in 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: failed to encode map e201 with expected crc 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: failed to encode map e201 with expected crc 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: failed to encode map e201 with expected crc 2023-10-09T04:10:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:44 smithi181 ceph-mon[196979]: failed to encode map e201 with expected crc 2023-10-09T04:10:45.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:45.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:45.404 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 83013 82997 67.3212 38.875 0.00971505 0.0148491 2023-10-09T04:10:45.492 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:45.523 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:45.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[213841]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[213841]: osdmap e202: 8 total, 8 up, 8 in 2023-10-09T04:10:45.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[213841]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[213841]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:45.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[221067]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[221067]: osdmap e202: 8 total, 8 up, 8 in 2023-10-09T04:10:45.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[221067]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:45 smithi163 ceph-mon[221067]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:45.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:45.689 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:45.721 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:45.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:45.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:45 smithi181 ceph-mon[196979]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:45 smithi181 ceph-mon[196979]: osdmap e202: 8 total, 8 up, 8 in 2023-10-09T04:10:45.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:45 smithi181 ceph-mon[196979]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.777 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:45 smithi181 ceph-mon[196979]: failed to encode map e202 with expected crc 2023-10-09T04:10:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:45.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:45.879 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:45.962 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:10:45.963 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:10:45.963 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:46.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 83958 83942 67.2137 59.0625 0.0120123 0.014873 2023-10-09T04:10:46.551 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:46 smithi163 ceph-mon[213841]: pgmap v513: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 5870/244131 objects degraded (2.404%); 7641/244131 objects misplaced (3.130%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:10:46.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:46 smithi163 ceph-mon[221067]: pgmap v513: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 5870/244131 objects degraded (2.404%); 7641/244131 objects misplaced (3.130%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:10:46.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:46.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:46.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:46.789 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:47.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:46 smithi181 ceph-mon[196979]: pgmap v513: 99 pgs: 1 peering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 5870/244131 objects degraded (2.404%); 7641/244131 objects misplaced (3.130%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:10:47.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:47.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:47.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:47.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:10:47.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:10:47.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 84828 84812 67.0512 54.375 0.0129418 0.0149087 2023-10-09T04:10:47.432 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:47.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4285888493' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:10:47.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:10:47.567 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5870/244131 objects degraded (2.404%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:47.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4285888493' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:10:47.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:10:47.574 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5870/244131 objects degraded (2.404%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:47.582 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:47.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:47.598 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:10:47.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:10:47.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:47.825 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:48.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4285888493' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:10:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:10:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5870/244131 objects degraded (2.404%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:48.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:48.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:48.406 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:10:48.407257+0000 min lat: 0.0010512 max lat: 5.87729 avg lat: 0.0149706 2023-10-09T04:10:48.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:10:48.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 85545 85529 66.7733 44.8125 0.00955936 0.0149706 2023-10-09T04:10:48.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:48.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:10:48.542 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:48.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[221067]: pgmap v514: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 481 op/s; 4750/250275 objects degraded (1.898%); 8625/250275 objects misplaced (3.446%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:10:48.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:10:48.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[221067]: osdmap e203: 8 total, 8 up, 8 in 2023-10-09T04:10:48.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[221067]: failed to encode map e203 with expected crc 2023-10-09T04:10:48.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[221067]: failed to encode map e203 with expected crc 2023-10-09T04:10:48.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[221067]: failed to encode map e203 with expected crc 2023-10-09T04:10:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[213841]: pgmap v514: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 481 op/s; 4750/250275 objects degraded (1.898%); 8625/250275 objects misplaced (3.446%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:10:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:10:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[213841]: osdmap e203: 8 total, 8 up, 8 in 2023-10-09T04:10:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[213841]: failed to encode map e203 with expected crc 2023-10-09T04:10:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[213841]: failed to encode map e203 with expected crc 2023-10-09T04:10:48.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:48 smithi163 ceph-mon[213841]: failed to encode map e203 with expected crc 2023-10-09T04:10:48.677 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:48.698 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:48.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:48.741 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:48.786 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:48.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:48.882 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:49.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:48 smithi181 ceph-mon[196979]: pgmap v514: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 481 op/s; 4750/250275 objects degraded (1.898%); 8625/250275 objects misplaced (3.446%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:10:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:10:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:48 smithi181 ceph-mon[196979]: osdmap e203: 8 total, 8 up, 8 in 2023-10-09T04:10:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:48 smithi181 ceph-mon[196979]: failed to encode map e203 with expected crc 2023-10-09T04:10:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:48 smithi181 ceph-mon[196979]: failed to encode map e203 with expected crc 2023-10-09T04:10:49.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:48 smithi181 ceph-mon[196979]: failed to encode map e203 with expected crc 2023-10-09T04:10:49.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:49.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:49.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:49.271 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:49.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:49.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:49.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 86291 86275 66.5247 46.625 0.0316259 0.015025 2023-10-09T04:10:49.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:49.457 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:49.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:49.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:49.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:49.662 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:49.721 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:49.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:49.828 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:49.868 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:50.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 87051 87035 66.2927 47.5 0.0022644 0.0150719 2023-10-09T04:10:50.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:50.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:50.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:50 smithi163 ceph-mon[213841]: pgmap v516: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 530 op/s; 4750/251976 objects degraded (1.885%); 8625/251976 objects misplaced (3.423%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:10:50.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:50.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:50.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:50 smithi163 ceph-mon[221067]: pgmap v516: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 530 op/s; 4750/251976 objects degraded (1.885%); 8625/251976 objects misplaced (3.423%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:10:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:10:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:50 smithi181 ceph-mon[196979]: pgmap v516: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 530 op/s; 4750/251976 objects degraded (1.885%); 8625/251976 objects misplaced (3.423%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:10:50.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:50.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:50.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:50.939 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:50.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:50.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:10:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:10:50] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:10:51.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:51.114 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:51.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:51.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:51.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:51.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:51.303 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:51.362 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:51.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:51.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:51.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 87920 87904 66.1484 54.3125 0.0194895 0.0151123 2023-10-09T04:10:51.472 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:51.509 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:51.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:51.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:51.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:51.764 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:52.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:52.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:52.280 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:52.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:52.407 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 88832 88816 66.0395 57 0.00904337 0.0151306 2023-10-09T04:10:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:52.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:52.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:52.568 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:52.620 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:52.670 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[213841]: pgmap v517: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 4750/258123 objects degraded (1.840%); 8462/258123 objects misplaced (3.278%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:10:52.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4750/258123 objects degraded (1.840%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:52.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[213841]: failed to encode map e204 with expected crc 2023-10-09T04:10:52.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[213841]: failed to encode map e204 with expected crc 2023-10-09T04:10:52.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[213841]: osdmap e204: 8 total, 8 up, 8 in 2023-10-09T04:10:52.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[221067]: pgmap v517: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 4750/258123 objects degraded (1.840%); 8462/258123 objects misplaced (3.278%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:10:52.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4750/258123 objects degraded (1.840%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:52.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[221067]: failed to encode map e204 with expected crc 2023-10-09T04:10:52.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[221067]: failed to encode map e204 with expected crc 2023-10-09T04:10:52.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:52 smithi163 ceph-mon[221067]: osdmap e204: 8 total, 8 up, 8 in 2023-10-09T04:10:52.685 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:10:52.685 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:10:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:52 smithi181 ceph-mon[196979]: pgmap v517: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 4750/258123 objects degraded (1.840%); 8462/258123 objects misplaced (3.278%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:10:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4750/258123 objects degraded (1.840%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:10:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:52 smithi181 ceph-mon[196979]: failed to encode map e204 with expected crc 2023-10-09T04:10:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:52 smithi181 ceph-mon[196979]: failed to encode map e204 with expected crc 2023-10-09T04:10:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:52 smithi181 ceph-mon[196979]: osdmap e204: 8 total, 8 up, 8 in 2023-10-09T04:10:53.378 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:53.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:53.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 89450 89434 65.7148 38.625 0.0608129 0.0152078 2023-10-09T04:10:53.482 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:53.515 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:53.577 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:53.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:53.682 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:53.715 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:53.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:53 smithi181 ceph-mon[196979]: failed to encode map e204 with expected crc 2023-10-09T04:10:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:53 smithi181 ceph-mon[196979]: osdmap e205: 8 total, 8 up, 8 in 2023-10-09T04:10:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:53 smithi181 ceph-mon[196979]: failed to encode map e205 with expected crc 2023-10-09T04:10:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/452123827' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:53 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:53.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:53.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:53.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[213841]: failed to encode map e204 with expected crc 2023-10-09T04:10:53.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[213841]: osdmap e205: 8 total, 8 up, 8 in 2023-10-09T04:10:53.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[213841]: failed to encode map e205 with expected crc 2023-10-09T04:10:53.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/452123827' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:53.789 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:53.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[221067]: failed to encode map e204 with expected crc 2023-10-09T04:10:53.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[221067]: osdmap e205: 8 total, 8 up, 8 in 2023-10-09T04:10:53.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[221067]: failed to encode map e205 with expected crc 2023-10-09T04:10:53.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/452123827' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:53.790 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:53 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:10:53.878 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:53.939 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:54.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:10:54.160 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:10:54.213 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:10:54.259 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:10:54.300 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:10:54.301 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:10:54.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:54.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:54.376 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:10:54.402 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:10:54.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:10:54.410 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 90262 90246 65.5409 50.75 0.00242595 0.0152465 2023-10-09T04:10:54.420 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:10:54.432 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:54.440 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:10:54.441 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.442 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:10:54.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:10:54.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:54.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:10:54.480 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:10:54.514 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: pgmap v520: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 2197/262362 objects degraded (0.837%); 11005/262362 objects misplaced (4.195%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/464111598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: osdmap e206: 8 total, 8 up, 8 in 2023-10-09T04:10:54.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:54 smithi181 ceph-mon[196979]: failed to encode map e206 with expected crc 2023-10-09T04:10:54.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: pgmap v520: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 2197/262362 objects degraded (0.837%); 11005/262362 objects misplaced (4.195%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/464111598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: osdmap e206: 8 total, 8 up, 8 in 2023-10-09T04:10:54.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[221067]: failed to encode map e206 with expected crc 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: failed to encode map e205 with expected crc 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: pgmap v520: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 2197/262362 objects degraded (0.837%); 11005/262362 objects misplaced (4.195%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/464111598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:10:54.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:10:54.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:10:54.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: osdmap e206: 8 total, 8 up, 8 in 2023-10-09T04:10:54.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:54 smithi163 ceph-mon[213841]: failed to encode map e206 with expected crc 2023-10-09T04:10:54.612 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:54.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:54.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:54.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:54.884 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:55.410 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 91002 90986 65.3192 46.25 0.0202734 0.0153042 2023-10-09T04:10:55.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:55.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:55.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:10:55.904 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:56.183 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:10:56.183 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:10:56.184 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:10:56.184 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:10:56.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:10:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:56.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:10:56.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:56.372 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:56.410 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 91860 91844 65.1863 53.625 0.00914344 0.0153347 2023-10-09T04:10:56.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:56.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:56.521 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:56 smithi163 ceph-mon[213841]: pgmap v522: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 2197/266457 objects degraded (0.825%); 11005/266457 objects misplaced (4.130%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:10:56.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:56.562 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:56.604 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:56.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:56.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:56.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:56.778 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:56 smithi163 ceph-mon[221067]: pgmap v522: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 2197/266457 objects degraded (0.825%); 11005/266457 objects misplaced (4.130%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:10:56.787 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:56.915 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:56.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:56.969 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:56 smithi181 ceph-mon[196979]: pgmap v522: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 2197/266457 objects degraded (0.825%); 11005/266457 objects misplaced (4.130%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:10:56.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:10:57.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:57.076 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:57.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:57.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:57.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:57.356 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:57.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 92665 92649 65.019 50.3125 0.00731463 0.0153736 2023-10-09T04:10:57.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:57.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:57.472 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:10:57.553 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:10:57.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:10:57.611 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/745097691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:10:57.612 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/266457 objects degraded (0.825%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:10:57.612 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/745097691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:10:57.612 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/266457 objects degraded (0.825%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:10:57.628 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:57.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:10:57.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:10:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:57.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:10:57.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:57.871 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/745097691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:10:57.871 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/266457 objects degraded (0.825%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:10:57.893 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:58.105 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:58.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:58.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:58.236 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:58.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:58.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:58.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:10:58.381 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:10:58.406 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:58.411 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 90 1 93267 93266 64.7252 38.5625 0.239276 0.0154378 2023-10-09T04:10:58.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:10:58.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:58.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:10:58.551 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:58.578 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:58.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[221067]: pgmap v523: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 2197/270243 objects degraded (0.813%); 11194/270243 objects misplaced (4.142%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:10:58.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/745097691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:10:58.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[221067]: osdmap e207: 8 total, 8 up, 8 in 2023-10-09T04:10:58.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[221067]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[221067]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[221067]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[213841]: pgmap v523: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 2197/270243 objects degraded (0.813%); 11194/270243 objects misplaced (4.142%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/745097691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[213841]: osdmap e207: 8 total, 8 up, 8 in 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[213841]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[213841]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:10:58 smithi163 ceph-mon[213841]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:10:58.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:58.787 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:58 smithi181 ceph-mon[196979]: pgmap v523: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 2197/270243 objects degraded (0.813%); 11194/270243 objects misplaced (4.142%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:10:58.787 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/745097691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:10:58.787 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:58 smithi181 ceph-mon[196979]: osdmap e207: 8 total, 8 up, 8 in 2023-10-09T04:10:58.787 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:58 smithi181 ceph-mon[196979]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.787 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:58 smithi181 ceph-mon[196979]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.788 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:10:58 smithi181 ceph-mon[196979]: failed to encode map e207 with expected crc 2023-10-09T04:10:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:58.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:58.799 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:58.857 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:58.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:10:58.928 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:59.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:10:59.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:59.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:10:59.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:10:59.167 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:59.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:10:59.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:59.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:10:59.241 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:10:59.242 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:10:59.242 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:10:59.242 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:10:59.242 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:10:59.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:10:59.362 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0605 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 93267 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 64.7252 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 38.8304 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 194.812 2023-10-09T04:10:59.412 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1035 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 621.309 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3117 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0154385 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0727496 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 5.87729 2023-10-09T04:10:59.413 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.0010512 2023-10-09T04:10:59.418 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:10:59.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:10:59.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:59.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:10:59.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:10:59.762 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:10:59.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:10:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:10:59.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:10:59.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:10:59.992 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:00.091 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:00.390 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:00.488 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:00.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:00.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:00.736 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:00.773 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:00 smithi163 ceph-mon[213841]: pgmap v525: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 2197/275823 objects degraded (0.797%); 11111/275823 objects misplaced (4.028%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:11:00.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:11:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:11:00] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:11:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:00 smithi163 ceph-mon[221067]: pgmap v525: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 2197/275823 objects degraded (0.797%); 11111/275823 objects misplaced (4.028%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:11:00.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:00 smithi181 ceph-mon[196979]: pgmap v525: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 2197/275823 objects degraded (0.797%); 11111/275823 objects misplaced (4.028%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:11:00.931 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:00.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:01.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:01.068 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:01.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:01.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:01.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:01.254 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:01.536 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:01.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:01.576 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:01.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:01.621 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:01.657 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:01.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:01.707 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:01.712 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:01.745 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:01.787 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:01.825 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.831 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:11:01.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4187064590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:01.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4187064590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:01.840 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4187064590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:01.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:01.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:01.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:01.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:02.014 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:02.065 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:02.065 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:02.595 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:11:02.631 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[213841]: pgmap v526: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 744 op/s; 2197/278301 objects degraded (0.789%); 11214/278301 objects misplaced (4.029%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:11:02.631 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2789759199' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:02.632 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:02.632 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/278301 objects degraded (0.789%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:02.632 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[221067]: pgmap v526: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 744 op/s; 2197/278301 objects degraded (0.789%); 11214/278301 objects misplaced (4.029%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:11:02.632 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2789759199' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:02.632 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:02.633 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/278301 objects degraded (0.789%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:02.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:02 smithi181 ceph-mon[196979]: pgmap v526: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 744 op/s; 2197/278301 objects degraded (0.789%); 11214/278301 objects misplaced (4.029%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:11:02.845 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2789759199' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:02.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:02 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:02.846 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/278301 objects degraded (0.789%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:02.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:11:02.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:02.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:02.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:03.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:03.086 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:03.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:03.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:03.317 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:03.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:03.355 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:03.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:03.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:03.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:03.588 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:03.693 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:03 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:03.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:03 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:11:03.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:03 smithi181 ceph-mon[196979]: osdmap e208: 8 total, 8 up, 8 in 2023-10-09T04:11:03.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:03 smithi181 ceph-mon[196979]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:03 smithi181 ceph-mon[196979]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.694 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:03 smithi181 ceph-mon[196979]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[213841]: osdmap e208: 8 total, 8 up, 8 in 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[213841]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[213841]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[213841]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[221067]: osdmap e208: 8 total, 8 up, 8 in 2023-10-09T04:11:03.887 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[221067]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[221067]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:03 smithi163 ceph-mon[221067]: failed to encode map e208 with expected crc 2023-10-09T04:11:03.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:03.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:04.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:04.039 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:04.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:04.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:04.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:04.284 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:04.384 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:04.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:04.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:04.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:04.560 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:04.648 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:11:04.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:04.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:04.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:04 smithi181 ceph-mon[196979]: pgmap v528: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 700 op/s; 2197/276303 objects degraded (0.795%); 11105/276303 objects misplaced (4.019%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:11:04.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:04.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1342296903' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:11:04.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:11:04.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[213841]: pgmap v528: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 700 op/s; 2197/276303 objects degraded (0.795%); 11105/276303 objects misplaced (4.019%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:11:04.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:04.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1342296903' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:11:04.817 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:11:04.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:04.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[221067]: pgmap v528: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 700 op/s; 2197/276303 objects degraded (0.795%); 11105/276303 objects misplaced (4.019%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:11:04.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:04.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1342296903' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:11:04.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:11:04.886 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:04.987 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:04.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:05.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:11:05.072 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:05.072 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:05.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:05.170 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:05.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:05.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:05.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:05.445 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:05.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:05.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:05.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:05.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:11:05.686 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:05.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:05 smithi181 ceph-mon[196979]: failed to encode map e209 with expected crc 2023-10-09T04:11:05.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:11:05.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:05 smithi181 ceph-mon[196979]: osdmap e209: 8 total, 8 up, 8 in 2023-10-09T04:11:05.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:05 smithi181 ceph-mon[196979]: failed to encode map e209 with expected crc 2023-10-09T04:11:05.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:05 smithi181 ceph-mon[196979]: pgmap v530: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 129 B/s rd, 15 MiB/s wr, 556 op/s; 2197/272277 objects degraded (0.807%); 11105/272277 objects misplaced (4.079%); 444 KiB/s, 6 objects/s recovering 2023-10-09T04:11:05.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:05.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[213841]: failed to encode map e209 with expected crc 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[213841]: osdmap e209: 8 total, 8 up, 8 in 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[213841]: failed to encode map e209 with expected crc 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[213841]: pgmap v530: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 129 B/s rd, 15 MiB/s wr, 556 op/s; 2197/272277 objects degraded (0.807%); 11105/272277 objects misplaced (4.079%); 444 KiB/s, 6 objects/s recovering 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[221067]: failed to encode map e209 with expected crc 2023-10-09T04:11:05.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:11:05.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[221067]: osdmap e209: 8 total, 8 up, 8 in 2023-10-09T04:11:05.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[221067]: failed to encode map e209 with expected crc 2023-10-09T04:11:05.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:05 smithi163 ceph-mon[221067]: pgmap v530: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 129 B/s rd, 15 MiB/s wr, 556 op/s; 2197/272277 objects degraded (0.807%); 11105/272277 objects misplaced (4.079%); 444 KiB/s, 6 objects/s recovering 2023-10-09T04:11:05.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:05.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:05.897 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:05.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:11:05.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:06.019 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:06.075 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:06.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:06.184 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:06.222 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:06.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:06.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:06.319 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:06.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:06.428 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:06.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:06.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:06.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:06.681 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:06.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:06.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:07.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:07.036 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:07.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:07.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:07.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:07.213 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:07.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:07.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:07.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/272277 objects degraded (0.807%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:07.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/272277 objects degraded (0.807%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:07.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:07.438 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:07.513 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:07.517 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/272277 objects degraded (0.807%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:07.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:07.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:07.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:07.625 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:07.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:07.806 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:08.045 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:08.045 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:08.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:08.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:08.272 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:08.315 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:08.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:08.352 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:08.385 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:08.385 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:08.407 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:08.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:08.435 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:08 smithi163 ceph-mon[221067]: pgmap v531: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 10 MiB/s wr, 633 op/s; 2197/268548 objects degraded (0.818%); 10735/268548 objects misplaced (3.997%); 440 KiB/s, 6 objects/s recovering 2023-10-09T04:11:08.435 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1231285649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:08.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:08 smithi163 ceph-mon[213841]: pgmap v531: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 10 MiB/s wr, 633 op/s; 2197/268548 objects degraded (0.818%); 10735/268548 objects misplaced (3.997%); 440 KiB/s, 6 objects/s recovering 2023-10-09T04:11:08.436 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1231285649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:08.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:08.518 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:08.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:08.560 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:08.591 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:08.617 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.618 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:11:08.655 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:08.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:08 smithi181 ceph-mon[196979]: pgmap v531: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 10 MiB/s wr, 633 op/s; 2197/268548 objects degraded (0.818%); 10735/268548 objects misplaced (3.997%); 440 KiB/s, 6 objects/s recovering 2023-10-09T04:11:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1231285649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:08.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:08.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:08.806 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:11:08.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:08.939 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:09.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:09.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:09.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:09.176 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:09.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:09.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:09.275 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:09.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:09.422 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:09.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:09.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:10.046 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:10.081 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:10.143 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:10.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:10.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:10.280 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:10.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:10.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:10.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:10.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:10.554 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:10 smithi163 ceph-mon[213841]: pgmap v532: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 1.4 MiB/s wr, 703 op/s; 2197/262458 objects degraded (0.837%); 10647/262458 objects misplaced (4.057%); 792 KiB/s, 12 objects/s recovering 2023-10-09T04:11:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:10 smithi163 ceph-mon[221067]: pgmap v532: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 1.4 MiB/s wr, 703 op/s; 2197/262458 objects degraded (0.837%); 10647/262458 objects misplaced (4.057%); 792 KiB/s, 12 objects/s recovering 2023-10-09T04:11:10.705 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:10 smithi181 ceph-mon[196979]: pgmap v532: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 127 B/s rd, 1.4 MiB/s wr, 703 op/s; 2197/262458 objects degraded (0.837%); 10647/262458 objects misplaced (4.057%); 792 KiB/s, 12 objects/s recovering 2023-10-09T04:11:10.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:10.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:10.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:10.865 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:10.934 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:10.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:10.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:10.975 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:11:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:11:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:49821 6.7 6:e6e39e35:::benchmark_data_smithi163_287034_object49820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:05:26.692589+0000", 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 344.27944598699997, 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.212208916, 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 49821 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692589+0000", 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692589+0000", 2023-10-09T04:11:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692588+0000", 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692640+0000", 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2082e-05 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692654+0000", 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3631999999999999e-05 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692660+0000", 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8429999999999998e-06 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692711+0000", 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1452000000000001e-05 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692817+0000", 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010610799999999999 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.692954+0000", 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013706999999999999 2023-10-09T04:11:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.695117+0000", 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021632380000000001 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.696456+0000", 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013387200000000001 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.904705+0000", 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20824909599999999 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.904758+0000", 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2609999999999999e-05 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:05:26.904798+0000", 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0308000000000001e-05 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118877 6.a 6:5646758f:::benchmark_data_smithi163_287034_object10614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 267.96123954699999, 2023-10-09T04:11:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24332414399999999, 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118877 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010794+0000", 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010802+0000", 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3140000000000004e-06 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010818+0000", 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5945000000000001e-05 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010825+0000", 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1659999999999997e-06 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010868+0000", 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2886999999999998e-05 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148662+0000", 2023-10-09T04:11:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.137794378 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148810+0000", 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147527 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.152530+0000", 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003719703 2023-10-09T04:11:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.153038+0000", 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050862599999999998 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254050+0000", 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.101012064 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254090+0000", 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9533999999999998e-05 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254120+0000", 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9903000000000002e-05 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151856 6.d 6:ba33a9cc:::benchmark_data_smithi163_287034_object43593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 223.34507084500001, 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23517902700000001, 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151856 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626966+0000", 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.877e-06 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626972+0000", 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7089999999999999e-06 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626975+0000", 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1319999999999998e-06 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627008+0000", 2023-10-09T04:11:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3343999999999998e-05 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627097+0000", 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8706000000000001e-05 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627179+0000", 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2310000000000003e-05 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.631260+0000", 2023-10-09T04:11:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040813139999999999 2023-10-09T04:11:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794861+0000", 2023-10-09T04:11:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16360079 2023-10-09T04:11:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862048+0000", 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067187075999999998 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862105+0000", 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6997000000000003e-05 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862143+0000", 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8050999999999999e-05 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151866 6.d 6:b4107b04:::benchmark_data_smithi163_287034_object43603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:11:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 223.335114054, 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22535977800000001, 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151866 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636920+0000", 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636926+0000", 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636939+0000", 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2904e-05 2023-10-09T04:11:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636944+0000", 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8879999999999996e-06 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636983+0000", 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8402000000000003e-05 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637122+0000", 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000139307 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637246+0000", 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012406599999999999 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.641603+0000", 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043568410000000002 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794991+0000", 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15338833599999999 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862183+0000", 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067192247999999996 2023-10-09T04:11:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862248+0000", 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4317000000000001e-05 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862281+0000", 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3144e-05 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169261 6.d 6:b3c8e3b4:::benchmark_data_smithi163_287034_object60998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 199.961193313, 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22831853499999999, 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169261 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010841+0000", 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010846+0000", 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1510000000000003e-06 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010858+0000", 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1446e-05 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010864+0000", 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3620000000000002e-06 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010914+0000", 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9552000000000003e-05 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011016+0000", 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102262 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011120+0000", 2023-10-09T04:11:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104622 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020057+0000", 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089365069999999998 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179106+0000", 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15904936 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239056+0000", 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.059950160000000002 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239115+0000", 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8446000000000003e-05 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239160+0000", 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5503999999999997e-05 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169262 6.d 6:b9ed22f3:::benchmark_data_smithi163_287034_object60999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 199.95617641499999, 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23726620200000001, 2023-10-09T04:11:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169262 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:11:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015858+0000", 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015863+0000", 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.87e-06 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015871+0000", 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7810000000000005e-06 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015874+0000", 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6440000000000002e-06 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015992+0000", 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117414 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016078+0000", 2023-10-09T04:11:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6527999999999996e-05 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016159+0000", 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0648000000000004e-05 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020242+0000", 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040829209999999998 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179222+0000", 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15898009699999999 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253062+0000", 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.073840684000000004 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253100+0000", 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7886000000000003e-05 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253125+0000", 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4675999999999999e-05 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205752 6.a 6:5755c8f4:::benchmark_data_smithi163_287034_object97489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:11:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.09980455300001, 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22375711200000001, 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205752 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872230+0000", 2023-10-09T04:11:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872233+0000", 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8949999999999998e-06 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872238+0000", 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2449999999999998e-06 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872241+0000", 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8600000000000001e-06 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872286+0000", 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5157999999999997e-05 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872387+0000", 2023-10-09T04:11:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010013699999999999 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872470+0000", 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3599999999999999e-05 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881422+0000", 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089518930000000007 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881460+0000", 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7812000000000003e-05 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095901+0000", 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.214440778 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095945+0000", 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4332000000000002e-05 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095988+0000", 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2778000000000003e-05 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205754 6.d 6:b347cae0:::benchmark_data_smithi163_287034_object97491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.09978280199999, 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22360279999999999, 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205754 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872255+0000", 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.796e-06 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872261+0000", 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.117e-06 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872265+0000", 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6839999999999998e-06 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877378+0000", 2023-10-09T04:11:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051136020000000001 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877474+0000", 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5499000000000002e-05 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877596+0000", 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012222700000000001 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.882034+0000", 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044384189999999999 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.890599+0000", 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085642140000000005 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891540+0000", 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00094105299999999996 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891569+0000", 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9382000000000001e-05 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095855+0000", 2023-10-09T04:11:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.204286197 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205761 6.a 6:5be756cf:::benchmark_data_smithi163_287034_object97498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.09462188000001, 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.218706593, 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205761 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877412+0000", 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877421+0000", 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4700000000000002e-06 2023-10-09T04:11:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877430+0000", 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8929999999999994e-06 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877434+0000", 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9559999999999996e-06 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877558+0000", 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012344299999999999 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877639+0000", 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1212999999999996e-05 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877720+0000", 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0533999999999998e-05 2023-10-09T04:11:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881593+0000", 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003873479 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881717+0000", 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123604 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096041+0000", 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21432412100000001 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096082+0000", 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1437999999999999e-05 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096120+0000", 2023-10-09T04:11:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7599999999999999e-05 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205765 6.a 6:57307337:::benchmark_data_smithi163_287034_object97502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.883543+0000", 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 145.088492038, 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.212665823, 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205765 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883543+0000", 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883543+0000", 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883542+0000", 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883546+0000", 2023-10-09T04:11:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0899999999999998e-06 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883555+0000", 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.405e-06 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883559+0000", 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.403e-06 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883593+0000", 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3691e-05 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883668+0000", 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4684000000000005e-05 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:11:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.883738+0000", 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0525000000000007e-05 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.887458+0000", 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037203119999999999 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.887470+0000", 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2109e-05 2023-10-09T04:11:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096151+0000", 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20868013199999999 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096177+0000", 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6112999999999999e-05 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096209+0000", 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2277999999999998e-05 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36612 7.0 7:006250d9:::benchmark_data_smithi163_346082_object36611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 82.682021117000005, 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97418880500000005, 2023-10-09T04:11:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36612 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:11:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290012+0000", 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290059+0000", 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6390000000000001e-05 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290079+0000", 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0370999999999999e-05 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290088+0000", 2023-10-09T04:11:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.183e-06 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291399+0000", 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013106369999999999 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291468+0000", 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9116999999999996e-05 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264113+0000", 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97264483000000002 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264203+0000", 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0143999999999996e-05 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:50550 7.0 7:02154344:::benchmark_data_smithi163_346082_object50549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 64.693094306999996, 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25363291799999998, 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 50550 2023-10-09T04:11:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278940+0000", 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278990+0000", 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0781000000000001e-05 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279008+0000", 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7193999999999998e-05 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279016+0000", 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2830000000000001e-06 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279063+0000", 2023-10-09T04:11:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7596000000000003e-05 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279201+0000", 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137634 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279310+0000", 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109231 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.280343+0000", 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001032989 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.280822+0000", 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047909999999999999 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532453+0000", 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25163097600000001 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532526+0000", 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2867000000000006e-05 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532574+0000", 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7599000000000003e-05 2023-10-09T04:11:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:50558 7.0 7:042b1ccf:::benchmark_data_smithi163_346082_object50557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 64.687455516, 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24840627900000001, 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 50558 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:11:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284578+0000", 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284632+0000", 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4066e-05 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284645+0000", 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2251e-05 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284650+0000", 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2100000000000001e-06 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284690+0000", 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9845000000000002e-05 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284837+0000", 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014754199999999999 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284947+0000", 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109949 2023-10-09T04:11:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.285939+0000", 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00099183299999999991 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.286366+0000", 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042700700000000003 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532907+0000", 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24654063800000001 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532954+0000", 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6845999999999998e-05 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532986+0000", 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2450000000000003e-05 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:60833 7.0 7:0345c2d3:::benchmark_data_smithi163_346082_object60832:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:16.434148+0000", 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 54.537886854999996, 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21680829400000001, 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 60833 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434148+0000", 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434148+0000", 2023-10-09T04:11:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434147+0000", 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434202+0000", 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5478e-05 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434216+0000", 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4049999999999999e-05 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434225+0000", 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.388e-06 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434272+0000", 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6925999999999998e-05 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434387+0000", 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115512 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.434508+0000", 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120526 2023-10-09T04:11:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.435589+0000", 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001080739 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.436333+0000", 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074485299999999996 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.650853+0000", 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21451964400000001 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.650906+0000", 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2442999999999999e-05 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:16.650957+0000", 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1146000000000001e-05 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80455 7.0 7:03584563:::benchmark_data_smithi163_346082_object80454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 33.836491246000001, 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31383416800000002, 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80455 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135542+0000", 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135577+0000", 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5281000000000003e-05 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135599+0000", 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2073e-05 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135607+0000", 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.174e-06 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135662+0000", 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4206999999999998e-05 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135784+0000", 2023-10-09T04:11:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121916 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135908+0000", 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124819 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.137658+0000", 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017491220000000001 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.139918+0000", 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022599949999999999 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449290+0000", 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30937226000000001 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449346+0000", 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6529e-05 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449378+0000", 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.18e-05 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80493 7.0 7:04f35a7b:::benchmark_data_smithi163_346082_object80492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 33.791035274999999, 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27444662600000003, 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80493 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.180999+0000", 2023-10-09T04:11:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181023+0000", 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4460999999999998e-05 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181036+0000", 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.277e-05 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181043+0000", 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6320000000000002e-06 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181091+0000", 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8309000000000001e-05 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181190+0000", 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9109000000000003e-05 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181286+0000", 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5846000000000004e-05 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.182553+0000", 2023-10-09T04:11:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001267191 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.182801+0000", 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024814299999999999 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455353+0000", 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27255158600000001 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455406+0000", 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3074e-05 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455447+0000", 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0441000000000001e-05 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80494 7.e 7:79ff37ea:::benchmark_data_smithi163_346082_object80493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 33.522407635999997, 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.00624114, 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80494 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449627+0000", 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449685+0000", 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8378000000000002e-05 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449697+0000", 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2427e-05 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449703+0000", 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449751+0000", 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8476999999999997e-05 2023-10-09T04:11:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449761+0000", 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0378e-05 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441447+0000", 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99168547399999996 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441622+0000", 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017500600000000001 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441827+0000", 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020516999999999999 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451473+0000", 2023-10-09T04:11:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096461649999999999 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452805+0000", 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013312770000000001 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454609+0000", 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018042959999999999 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454649+0000", 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0256999999999998e-05 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.455869+0000", 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001219649 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80509 7.e 7:77548b6a:::benchmark_data_smithi163_346082_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 33.503569669000001, 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0024772179999999, 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80509 2023-10-09T04:11:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468464+0000", 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468512+0000", 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615000000000001e-05 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468527+0000", 2023-10-09T04:11:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4778e-05 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468534+0000", 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2049999999999996e-06 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468595+0000", 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0865000000000001e-05 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468603+0000", 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2439999999999996e-06 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445422+0000", 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97681920499999997 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445635+0000", 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000213382 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445836+0000", 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020043 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451507+0000", 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056710220000000004 2023-10-09T04:11:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452890+0000", 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013838699999999999 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469631+0000", 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01674059 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469678+0000", 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6551999999999997e-05 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.470943+0000", 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00126515 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80511 7.e 7:74ae5f43:::benchmark_data_smithi163_346082_object80510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 33.502007366000001, 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002226042, 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80511 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470027+0000", 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470082+0000", 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5106999999999999e-05 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470093+0000", 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0738e-05 2023-10-09T04:11:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470098+0000", 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5820000000000003e-06 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470142+0000", 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4261000000000002e-05 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470148+0000", 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0170000000000002e-06 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.447968+0000", 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97782021900000005 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448099+0000", 2023-10-09T04:11:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013048700000000001 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448204+0000", 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105266 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451513+0000", 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033096240000000002 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.456169+0000", 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046552060000000003 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471004+0000", 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014834926999999999 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471062+0000", 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8873999999999999e-05 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472254+0000", 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011913939999999999 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80513 7.e 7:768b3fe4:::benchmark_data_smithi163_346082_object80512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 33.500081655999999, 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0014613109999999, 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80513 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471952+0000", 2023-10-09T04:11:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472006+0000", 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3981999999999999e-05 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472018+0000", 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1924e-05 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472023+0000", 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6999999999999999e-06 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472070+0000", 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7218000000000001e-05 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472078+0000", 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8830000000000006e-06 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449809+0000", 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97773131000000002 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449917+0000", 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107543 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.450027+0000", 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010959399999999999 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451621+0000", 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015948360000000001 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.466881+0000", 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015260029 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472336+0000", 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054546960000000002 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472393+0000", 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6829000000000001e-05 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.473415+0000", 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001021852 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:11.126 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:11:11.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:11.127 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:11:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:11:10] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-10-09T04:11:11.223 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:11.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:11.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:11.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:11.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:11.547 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:11.709 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:11.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:11.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:11:11.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:11.879 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:12.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:12.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:12.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:12.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:12.168 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:12.234 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:12.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:12.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:12.268 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:12.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:11:12.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:12.374 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:12.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:12.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:12.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:12.558 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:11:12.560 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:11:12.590 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:12 smithi181 ceph-mon[196979]: pgmap v533: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 115 B/s wr, 734 op/s; 2197/256728 objects degraded (0.856%); 10284/256728 objects misplaced (4.006%); 317 KiB/s, 4 objects/s recovering 2023-10-09T04:11:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1647613679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:12 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3940595446' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:11:12.706 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/256728 objects degraded (0.856%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:12.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:12.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:12.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[213841]: pgmap v533: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 115 B/s wr, 734 op/s; 2197/256728 objects degraded (0.856%); 10284/256728 objects misplaced (4.006%); 317 KiB/s, 4 objects/s recovering 2023-10-09T04:11:12.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1647613679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:12.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:12.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3940595446' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:11:12.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/256728 objects degraded (0.856%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:12.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[221067]: pgmap v533: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 115 B/s wr, 734 op/s; 2197/256728 objects degraded (0.856%); 10284/256728 objects misplaced (4.006%); 317 KiB/s, 4 objects/s recovering 2023-10-09T04:11:12.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1647613679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:12.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:12.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3940595446' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:11:12.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/256728 objects degraded (0.856%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:12.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:12.889 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:12.957 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:12.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:13.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:13.117 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:13.159 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:13.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:13.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:13.289 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:13.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:13.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:13.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:11:13.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[213841]: failed to encode map e210 with expected crc 2023-10-09T04:11:13.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:13.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3940595446' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:11:13.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[213841]: osdmap e210: 8 total, 8 up, 8 in 2023-10-09T04:11:13.547 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[213841]: failed to encode map e210 with expected crc 2023-10-09T04:11:13.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:13.693 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:13.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:13.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:13.816 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:13 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:11:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:13 smithi181 ceph-mon[196979]: failed to encode map e210 with expected crc 2023-10-09T04:11:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:13 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3940595446' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:11:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:13 smithi181 ceph-mon[196979]: osdmap e210: 8 total, 8 up, 8 in 2023-10-09T04:11:13.817 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:13 smithi181 ceph-mon[196979]: failed to encode map e210 with expected crc 2023-10-09T04:11:13.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:11:13.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[221067]: failed to encode map e210 with expected crc 2023-10-09T04:11:13.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:13.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3940595446' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:11:13.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[221067]: osdmap e210: 8 total, 8 up, 8 in 2023-10-09T04:11:13.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:13 smithi163 ceph-mon[221067]: failed to encode map e210 with expected crc 2023-10-09T04:11:13.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:13.921 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:13.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:13.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:13.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:14.046 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:11:14.046 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:11:14.046 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:11:14.047 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:11:14.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:14.095 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:14.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:14.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:14.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:11:14.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:14.606 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:14.699 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:14.746 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:14.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:14.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:14.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:14 smithi163 ceph-mon[221067]: pgmap v535: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 2197/252207 objects degraded (0.871%); 10195/252207 objects misplaced (4.042%); 637 KiB/s, 9 objects/s recovering 2023-10-09T04:11:14.763 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:14 smithi163 ceph-mon[213841]: pgmap v535: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 2197/252207 objects degraded (0.871%); 10195/252207 objects misplaced (4.042%); 637 KiB/s, 9 objects/s recovering 2023-10-09T04:11:14.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:14.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:14 smithi181 ceph-mon[196979]: pgmap v535: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 2197/252207 objects degraded (0.871%); 10195/252207 objects misplaced (4.042%); 637 KiB/s, 9 objects/s recovering 2023-10-09T04:11:14.883 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:11:14.919 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:15.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:15.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:15.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:15.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:15.225 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:15.335 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:15.491 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:15.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:15.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:15.677 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:15.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:15.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:15.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:15.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:15.923 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:16.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:16.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:16.139 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:16.165 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:16.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:16.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:16.375 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:16.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:16.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:11:16.553 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:16.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:16.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:16.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:16.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:16 smithi163 ceph-mon[221067]: pgmap v536: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2197/247980 objects degraded (0.886%); 10195/247980 objects misplaced (4.111%); 563 KiB/s, 8 objects/s recovering 2023-10-09T04:11:16.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:16 smithi163 ceph-mon[213841]: pgmap v536: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2197/247980 objects degraded (0.886%); 10195/247980 objects misplaced (4.111%); 563 KiB/s, 8 objects/s recovering 2023-10-09T04:11:16.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:16 smithi181 ceph-mon[196979]: pgmap v536: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2197/247980 objects degraded (0.886%); 10195/247980 objects misplaced (4.111%); 563 KiB/s, 8 objects/s recovering 2023-10-09T04:11:16.935 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:17.121 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:11:17.121 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:11:17.121 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:11:17.121 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:11:17.121 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:11:17.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/247980 objects degraded (0.886%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:17.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/247980 objects degraded (0.886%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:17.843 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:17.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/247980 objects degraded (0.886%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:17.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:11:17.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:17.922 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:17.961 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:17.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:18.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:18.011 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:18.050 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:18.094 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:18.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:18.129 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:18.146 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:18.159 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.160 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:11:18.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:18.222 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:18.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:18.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:18.334 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:18.370 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:18.424 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:18.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:18.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:18.562 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:18.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:18.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:18.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:18.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:18 smithi181 ceph-mon[196979]: pgmap v537: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 2197/244176 objects degraded (0.900%); 9802/244176 objects misplaced (4.014%); 563 KiB/s, 8 objects/s recovering 2023-10-09T04:11:18.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/515304463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:18.761 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:18 smithi163 ceph-mon[213841]: pgmap v537: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 2197/244176 objects degraded (0.900%); 9802/244176 objects misplaced (4.014%); 563 KiB/s, 8 objects/s recovering 2023-10-09T04:11:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/515304463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:18 smithi163 ceph-mon[221067]: pgmap v537: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 2197/244176 objects degraded (0.900%); 9802/244176 objects misplaced (4.014%); 563 KiB/s, 8 objects/s recovering 2023-10-09T04:11:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/515304463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:18.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:18.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:18.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:18.975 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:19.516 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:19.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:19.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:19.664 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:19.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:19.794 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:19.794 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:19.794 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:19.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:19.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:19.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:19.944 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:20.184 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:20.184 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:20.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:20.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:20.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:20.381 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:20.506 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:20.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:20.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:20.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:20.632 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:11:20.643 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:20.825 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:11:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:11:20] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-10-09T04:11:20.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:20 smithi163 ceph-mon[221067]: pgmap v538: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 2197/238278 objects degraded (0.922%); 9708/238278 objects misplaced (4.074%); 582 KiB/s, 9 objects/s recovering 2023-10-09T04:11:20.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/462338801' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:11:20.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:11:20.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:20 smithi163 ceph-mon[213841]: pgmap v538: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 2197/238278 objects degraded (0.922%); 9708/238278 objects misplaced (4.074%); 582 KiB/s, 9 objects/s recovering 2023-10-09T04:11:20.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/462338801' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:11:20.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:11:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:20 smithi181 ceph-mon[196979]: pgmap v538: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 2197/238278 objects degraded (0.922%); 9708/238278 objects misplaced (4.074%); 582 KiB/s, 9 objects/s recovering 2023-10-09T04:11:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/462338801' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:11:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:11:21.096 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:11:21.190 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:21.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:21.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:11:21.294 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:21.394 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:21.592 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:11:21.592 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:11:21.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:21.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:21.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[221067]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:11:21.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[221067]: osdmap e211: 8 total, 8 up, 8 in 2023-10-09T04:11:21.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[221067]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[221067]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4229902882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:11:21.648 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[213841]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:11:21.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[213841]: osdmap e211: 8 total, 8 up, 8 in 2023-10-09T04:11:21.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[213841]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[213841]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.649 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4229902882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:11:21.685 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:11:21.724 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:21.767 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:21.879 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:21.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:21.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:21 smithi181 ceph-mon[196979]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:21 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:11:21.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:21 smithi181 ceph-mon[196979]: osdmap e211: 8 total, 8 up, 8 in 2023-10-09T04:11:21.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:21 smithi181 ceph-mon[196979]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:21 smithi181 ceph-mon[196979]: failed to encode map e211 with expected crc 2023-10-09T04:11:21.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4229902882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:11:21.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:21.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:22.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:22.111 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:22.662 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:22.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:22.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:22.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:22.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: pgmap v540: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 2197/232794 objects degraded (0.944%); 9386/232794 objects misplaced (4.032%); 653 KiB/s, 10 objects/s recovering 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: failed to encode map e212 with expected crc 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4229902882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: osdmap e212: 8 total, 8 up, 8 in 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: failed to encode map e212 with expected crc 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: failed to encode map e212 with expected crc 2023-10-09T04:11:22.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/232794 objects degraded (0.944%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: pgmap v540: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 2197/232794 objects degraded (0.944%); 9386/232794 objects misplaced (4.032%); 653 KiB/s, 10 objects/s recovering 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: failed to encode map e212 with expected crc 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4229902882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: osdmap e212: 8 total, 8 up, 8 in 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: failed to encode map e212 with expected crc 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: failed to encode map e212 with expected crc 2023-10-09T04:11:22.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/232794 objects degraded (0.944%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:22.968 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: pgmap v540: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 2197/232794 objects degraded (0.944%); 9386/232794 objects misplaced (4.032%); 653 KiB/s, 10 objects/s recovering 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: failed to encode map e212 with expected crc 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4229902882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: osdmap e212: 8 total, 8 up, 8 in 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: failed to encode map e212 with expected crc 2023-10-09T04:11:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: failed to encode map e212 with expected crc 2023-10-09T04:11:23.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/232794 objects degraded (0.944%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:23.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:23.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:23.213 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:23.261 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:23.282 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:23.443 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:23.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:23.554 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:23.591 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:23.714 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:23.715 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:23.743 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:23 smithi181 ceph-mon[196979]: pgmap v542: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 2197/229203 objects degraded (0.959%); 9298/229203 objects misplaced (4.057%); 728 KiB/s, 11 objects/s recovering 2023-10-09T04:11:24.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:24.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:23 smithi163 ceph-mon[221067]: pgmap v542: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 2197/229203 objects degraded (0.959%); 9298/229203 objects misplaced (4.057%); 728 KiB/s, 11 objects/s recovering 2023-10-09T04:11:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:24.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:23 smithi163 ceph-mon[213841]: pgmap v542: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 2197/229203 objects degraded (0.959%); 9298/229203 objects misplaced (4.057%); 728 KiB/s, 11 objects/s recovering 2023-10-09T04:11:24.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:24.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:24.197 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:24.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:24.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:24.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:24.683 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:11:24.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:11:24.717 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:25.029 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3782587349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:25.222 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:25.222 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:25.222 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:25.222 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:25.222 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:25.251 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:25.259 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:25.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:25.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:25.282 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:25.311 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:25.322 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.322 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.322 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.323 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.323 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.323 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.323 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.323 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:11:25.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3782587349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:25.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:25.460 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3782587349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:25.470 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:25.996 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:26.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:26.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:26.139 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:26.193 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:26.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:26.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:26.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:26.325 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:26 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:26 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:26.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:26 smithi181 ceph-mon[196979]: pgmap v543: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2197/225564 objects degraded (0.974%); 9298/225564 objects misplaced (4.122%); 728 KiB/s, 11 objects/s recovering 2023-10-09T04:11:26.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:26.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:26 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:26 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:26.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:26 smithi163 ceph-mon[221067]: pgmap v543: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2197/225564 objects degraded (0.974%); 9298/225564 objects misplaced (4.122%); 728 KiB/s, 11 objects/s recovering 2023-10-09T04:11:26.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:26 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:26.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:26 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:26.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:26 smithi163 ceph-mon[213841]: pgmap v543: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2197/225564 objects degraded (0.974%); 9298/225564 objects misplaced (4.122%); 728 KiB/s, 11 objects/s recovering 2023-10-09T04:11:26.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:26.670 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:26.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:26.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:26.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:26.915 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:27.004 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:11:27.004 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:11:27.005 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:11:27.005 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:11:27.005 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:11:27.053 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:27.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:27.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:27.416 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:11:27.418 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:11:27.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:27.507 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/225564 objects degraded (0.974%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/225564 objects degraded (0.974%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/225564 objects degraded (0.974%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:27.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:11:27.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:27.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:27.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:27.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:27.941 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:28.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:28.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:28.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:28.166 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:28.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:28.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:28.298 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:11:28.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:11:28.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:28.399 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:28.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:28.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:28.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:28.677 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:28 smithi163 ceph-mon[213841]: from='client.54973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:28.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:28 smithi163 ceph-mon[213841]: pgmap v544: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 2197/222351 objects degraded (0.988%); 8957/222351 objects misplaced (4.028%); 352 KiB/s, 5 objects/s recovering 2023-10-09T04:11:28.678 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:28 smithi163 ceph-mon[221067]: from='client.54973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:28.678 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:28 smithi163 ceph-mon[221067]: pgmap v544: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 2197/222351 objects degraded (0.988%); 8957/222351 objects misplaced (4.028%); 352 KiB/s, 5 objects/s recovering 2023-10-09T04:11:28.685 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:28 smithi181 ceph-mon[196979]: from='client.54973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:28.685 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:28 smithi181 ceph-mon[196979]: pgmap v544: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 2197/222351 objects degraded (0.988%); 8957/222351 objects misplaced (4.028%); 352 KiB/s, 5 objects/s recovering 2023-10-09T04:11:28.698 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:28.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:28.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:28.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:29.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:29.033 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:29.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:29.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:29.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:29.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:29.207 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:29.388 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":543,"stamp":"2023-10-09T04:11:25.455881+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4926478264,"num_objects":75188,"num_object_clones":0,"num_object_copies":225564,"num_objects_missing_on_primary":452,"num_objects_missing":452,"num_objects_degraded":2197,"num_objects_misplaced":9298,"num_objects_unfound":0,"num_objects_dirty":75188,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":105233,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10345,"num_bytes_recovered":679086772,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134083,"ondisk_log_size":134083,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17263216,"kb_used_data":14937136,"kb_used_omap":126,"kb_used_meta":2325825,"kb_avail":732698000,"statfs":{"total":767960285184,"available":750282752000,"internally_reserved":0,"allocated":15295627264,"data_stored":15289817346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2381645435},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-406585344,"num_objects":-6204,"num_object_clones":0,"num_object_copies":-18612,"num_objects_missing_on_primary":-91,"num_objects_missing":-91,"num_objects_degraded":0,"num_objects_misplaced":-504,"num_objects_unfound":0,"num_objects_dirty":-6204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6204,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":5963776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003070"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":209,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649196+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:11:20.649196+0000","last_peered":"2023-10-09T04:11:20.649196+0000","last_clean":"2023-10-09T04:11:20.649196+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:11:20.649196+0000","last_undegraded":"2023-10-09T04:11:20.649196+0000","last_fullsized":"2023-10-09T04:11:20.649196+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":147,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644526+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:11:20.644526+0000","last_peered":"2023-10-09T04:11:20.644526+0000","last_clean":"2023-10-09T04:11:20.644526+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:11:20.644526+0000","last_undegraded":"2023-10-09T04:11:20.644526+0000","last_fullsized":"2023-10-09T04:11:20.644526+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":162,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644023+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:11:20.644023+0000","last_peered":"2023-10-09T04:11:20.644023+0000","last_clean":"2023-10-09T04:11:20.644023+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:20.644023+0000","last_undegraded":"2023-10-09T04:11:20.644023+0000","last_fullsized":"2023-10-09T04:11:20.644023+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.622055+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:11:20.622055+0000","last_peered":"2023-10-09T04:11:20.622055+0000","last_clean":"2023-10-09T04:11:20.622055+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:11:20.622055+0000","last_undegraded":"2023-10-09T04:11:20.622055+0000","last_fullsized":"2023-10-09T04:11:20.622055+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":165,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621570+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:11:20.621570+0000","last_peered":"2023-10-09T04:11:20.621570+0000","last_clean":"2023-10-09T04:11:20.621570+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:11:20.621570+0000","last_undegraded":"2023-10-09T04:11:20.621570+0000","last_fullsized":"2023-10-09T04:11:20.621570+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":143,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.618061+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:11:04.618061+0000","last_peered":"2023-10-09T04:11:04.618061+0000","last_clean":"2023-10-09T04:11:04.618061+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:11:04.618061+0000","last_undegraded":"2023-10-09T04:11:04.618061+0000","last_fullsized":"2023-10-09T04:11:04.618061+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"210'4553","reported_seq":7503,"reported_epoch":210,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:20.512752+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:11:20.512752+0000","last_peered":"2023-10-09T04:11:20.512752+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:11:20.512752+0000","last_undegraded":"2023-10-09T04:11:20.512752+0000","last_fullsized":"2023-10-09T04:11:20.512752+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2406,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3382,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2406}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":166,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621972+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:11:20.621972+0000","last_peered":"2023-10-09T04:11:20.621972+0000","last_clean":"2023-10-09T04:11:20.621972+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:11:20.621972+0000","last_undegraded":"2023-10-09T04:11:20.621972+0000","last_fullsized":"2023-10-09T04:11:20.621972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:12.550436+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:11:12.550436+0000","last_peered":"2023-10-09T04:11:12.550436+0000","last_clean":"2023-10-09T04:11:12.550436+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:11:12.550436+0000","last_undegraded":"2023-10-09T04:11:12.550436+0000","last_fullsized":"2023-10-09T04:11:12.550436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"212'4719","reported_seq":8568,"reported_epoch":212,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:11:22.767155+0000","last_change":"2023-10-09T04:10:51.646576+0000","last_active":"2023-10-09T04:11:22.767155+0000","last_peered":"2023-10-09T04:11:22.767155+0000","last_clean":"2023-10-09T04:09:59.283093+0000","last_became_active":"2023-10-09T04:10:40.892310+0000","last_became_peered":"2023-10-09T04:10:40.892310+0000","last_unstale":"2023-10-09T04:11:22.767155+0000","last_undegraded":"2023-10-09T04:11:22.767155+0000","last_fullsized":"2023-10-09T04:10:40.864699+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":452,"num_objects_missing":452,"num_objects_degraded":0,"num_objects_misplaced":1992,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3554,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1013,"num_bytes_recovered":66387968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"0,1,2,3","objects":1},{"shards":"0,1,6,7","objects":1087},{"shards":"0,6","objects":860},{"shards":"1,2,3","objects":452}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":169,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.620591+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:11:04.620591+0000","last_peered":"2023-10-09T04:11:04.620591+0000","last_clean":"2023-10-09T04:11:04.620591+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:11:04.620591+0000","last_undegraded":"2023-10-09T04:11:04.620591+0000","last_fullsized":"2023-10-09T04:11:04.620591+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":106,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.634918+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:11:04.634918+0000","last_peered":"2023-10-09T04:11:04.634918+0000","last_clean":"2023-10-09T04:11:04.634918+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:11:04.634918+0000","last_undegraded":"2023-10-09T04:11:04.634918+0000","last_fullsized":"2023-10-09T04:11:04.634918+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"210'4548","reported_seq":7428,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.541569+0000","last_change":"2023-10-09T04:10:18.287893+0000","last_active":"2023-10-09T04:11:20.541569+0000","last_peered":"2023-10-09T04:11:20.541569+0000","last_clean":"2023-10-09T04:11:20.541569+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:11:20.541569+0000","last_undegraded":"2023-10-09T04:11:20.541569+0000","last_fullsized":"2023-10-09T04:11:20.541569+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4548,"log_dups_size":0,"ondisk_log_size":4548,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155123712,"num_objects":2367,"num_object_clones":0,"num_object_copies":7101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3413,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":172,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635368+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:11:04.635368+0000","last_peered":"2023-10-09T04:11:04.635368+0000","last_clean":"2023-10-09T04:11:04.635368+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:11:04.635368+0000","last_undegraded":"2023-10-09T04:11:04.635368+0000","last_fullsized":"2023-10-09T04:11:04.635368+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644525+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:11:20.644525+0000","last_peered":"2023-10-09T04:11:20.644525+0000","last_clean":"2023-10-09T04:11:20.644525+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:11:20.644525+0000","last_undegraded":"2023-10-09T04:11:20.644525+0000","last_fullsized":"2023-10-09T04:11:20.644525+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"212'4768","reported_seq":7701,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:24.855270+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:11:24.855270+0000","last_peered":"2023-10-09T04:11:24.855270+0000","last_clean":"2023-10-09T04:11:24.855270+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:11:24.855270+0000","last_undegraded":"2023-10-09T04:11:24.855270+0000","last_fullsized":"2023-10-09T04:11:24.855270+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4768,"log_dups_size":0,"ondisk_log_size":4768,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3579,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":176,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649227+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:11:20.649227+0000","last_peered":"2023-10-09T04:11:20.649227+0000","last_clean":"2023-10-09T04:11:20.649227+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:11:20.649227+0000","last_undegraded":"2023-10-09T04:11:20.649227+0000","last_fullsized":"2023-10-09T04:11:20.649227+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649370+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:11:20.649370+0000","last_peered":"2023-10-09T04:11:20.649370+0000","last_clean":"2023-10-09T04:11:20.649370+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:11:20.649370+0000","last_undegraded":"2023-10-09T04:11:20.649370+0000","last_fullsized":"2023-10-09T04:11:20.649370+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"212'4673","reported_seq":4737,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.773026+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:11:22.773026+0000","last_peered":"2023-10-09T04:11:22.773026+0000","last_clean":"2023-10-09T04:11:22.773026+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:11:22.773026+0000","last_undegraded":"2023-10-09T04:11:22.773026+0000","last_fullsized":"2023-10-09T04:11:22.773026+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4673,"log_dups_size":0,"ondisk_log_size":4673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152371200,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3519,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":171,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.634540+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:11:04.634540+0000","last_peered":"2023-10-09T04:11:04.634540+0000","last_clean":"2023-10-09T04:11:04.634540+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:11:04.634540+0000","last_undegraded":"2023-10-09T04:11:04.634540+0000","last_fullsized":"2023-10-09T04:11:04.634540+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":147,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644293+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:11:20.644293+0000","last_peered":"2023-10-09T04:11:20.644293+0000","last_clean":"2023-10-09T04:11:20.644293+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:11:20.644293+0000","last_undegraded":"2023-10-09T04:11:20.644293+0000","last_fullsized":"2023-10-09T04:11:20.644293+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"212'4516","reported_seq":4579,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.798373+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:11:22.798373+0000","last_peered":"2023-10-09T04:11:22.798373+0000","last_clean":"2023-10-09T04:11:22.798373+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:11:22.798373+0000","last_undegraded":"2023-10-09T04:11:22.798373+0000","last_fullsized":"2023-10-09T04:11:22.798373+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147226624,"num_objects":2247,"num_object_clones":0,"num_object_copies":6741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3399,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.651199+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:11:21.651199+0000","last_peered":"2023-10-09T04:11:21.651199+0000","last_clean":"2023-10-09T04:11:21.651199+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:11:21.651199+0000","last_undegraded":"2023-10-09T04:11:21.651199+0000","last_fullsized":"2023-10-09T04:11:21.651199+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":168,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.617969+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:11:04.617969+0000","last_peered":"2023-10-09T04:11:04.617969+0000","last_clean":"2023-10-09T04:11:04.617969+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:11:04.617969+0000","last_undegraded":"2023-10-09T04:11:04.617969+0000","last_fullsized":"2023-10-09T04:11:04.617969+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"210'4568","reported_seq":7507,"reported_epoch":210,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:20.556857+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:11:20.556857+0000","last_peered":"2023-10-09T04:11:20.556857+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:11:20.556857+0000","last_undegraded":"2023-10-09T04:11:20.556857+0000","last_fullsized":"2023-10-09T04:11:20.556857+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4568,"log_dups_size":0,"ondisk_log_size":4568,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160595968,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2451,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3453,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2451}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":152,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.634842+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:11:04.634842+0000","last_peered":"2023-10-09T04:11:04.634842+0000","last_clean":"2023-10-09T04:11:04.634842+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:11:04.634842+0000","last_undegraded":"2023-10-09T04:11:04.634842+0000","last_fullsized":"2023-10-09T04:11:04.634842+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.622060+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:11:20.622060+0000","last_peered":"2023-10-09T04:11:20.622060+0000","last_clean":"2023-10-09T04:11:20.622060+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:11:20.622060+0000","last_undegraded":"2023-10-09T04:11:20.622060+0000","last_fullsized":"2023-10-09T04:11:20.622060+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"212'4664","reported_seq":4728,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.756726+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:11:22.756726+0000","last_peered":"2023-10-09T04:11:22.756726+0000","last_clean":"2023-10-09T04:11:22.756726+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:11:22.756726+0000","last_undegraded":"2023-10-09T04:11:22.756726+0000","last_fullsized":"2023-10-09T04:11:22.756726+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4664,"log_dups_size":0,"ondisk_log_size":4664,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3502,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":102,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621416+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:11:20.621416+0000","last_peered":"2023-10-09T04:11:20.621416+0000","last_clean":"2023-10-09T04:11:20.621416+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:11:20.621416+0000","last_undegraded":"2023-10-09T04:11:20.621416+0000","last_fullsized":"2023-10-09T04:11:20.621416+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":175,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:12.552685+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:11:12.552685+0000","last_peered":"2023-10-09T04:11:12.552685+0000","last_clean":"2023-10-09T04:11:12.552685+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:11:12.552685+0000","last_undegraded":"2023-10-09T04:11:12.552685+0000","last_fullsized":"2023-10-09T04:11:12.552685+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"212'6973","reported_seq":7027,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:23.565758+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:11:23.565758+0000","last_peered":"2023-10-09T04:11:23.565758+0000","last_clean":"2023-10-09T04:11:23.565758+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:11:23.565758+0000","last_undegraded":"2023-10-09T04:11:23.565758+0000","last_fullsized":"2023-10-09T04:11:23.565758+0000","mapping_epoch":167,"log_start":"171'100","ondisk_log_start":"171'100","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6873,"log_dups_size":0,"ondisk_log_size":6873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308215808,"num_objects":4703,"num_object_clones":0,"num_object_copies":14109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6973,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":170,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.634921+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:11:04.634921+0000","last_peered":"2023-10-09T04:11:04.634921+0000","last_clean":"2023-10-09T04:11:04.634921+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:11:04.634921+0000","last_undegraded":"2023-10-09T04:11:04.634921+0000","last_fullsized":"2023-10-09T04:11:04.634921+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":151,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.617767+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:11:04.617767+0000","last_peered":"2023-10-09T04:11:04.617767+0000","last_clean":"2023-10-09T04:11:04.617767+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:11:04.617767+0000","last_undegraded":"2023-10-09T04:11:04.617767+0000","last_fullsized":"2023-10-09T04:11:04.617767+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"212'7037","reported_seq":7091,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:24.943593+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:11:24.943593+0000","last_peered":"2023-10-09T04:11:24.943593+0000","last_clean":"2023-10-09T04:11:24.943593+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:11:24.943593+0000","last_undegraded":"2023-10-09T04:11:24.943593+0000","last_fullsized":"2023-10-09T04:11:24.943593+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296550400,"num_objects":4525,"num_object_clones":0,"num_object_copies":13575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7037,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":171,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635825+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:11:04.635825+0000","last_peered":"2023-10-09T04:11:04.635825+0000","last_clean":"2023-10-09T04:11:04.635825+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:11:04.635825+0000","last_undegraded":"2023-10-09T04:11:04.635825+0000","last_fullsized":"2023-10-09T04:11:04.635825+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649720+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:11:20.649720+0000","last_peered":"2023-10-09T04:11:20.649720+0000","last_clean":"2023-10-09T04:11:20.649720+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:11:20.649720+0000","last_undegraded":"2023-10-09T04:11:20.649720+0000","last_fullsized":"2023-10-09T04:11:20.649720+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"212'4681","reported_seq":4852,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.789261+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:11:22.789261+0000","last_peered":"2023-10-09T04:11:22.789261+0000","last_clean":"2023-10-09T04:11:22.789261+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:11:22.789261+0000","last_undegraded":"2023-10-09T04:11:22.789261+0000","last_fullsized":"2023-10-09T04:11:22.789261+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":4681,"log_dups_size":0,"ondisk_log_size":4681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155779072,"num_objects":2377,"num_object_clones":0,"num_object_copies":7131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2295,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":153,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635204+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:11:04.635204+0000","last_peered":"2023-10-09T04:11:04.635204+0000","last_clean":"2023-10-09T04:11:04.635204+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:11:04.635204+0000","last_undegraded":"2023-10-09T04:11:04.635204+0000","last_fullsized":"2023-10-09T04:11:04.635204+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.643990+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:11:20.643990+0000","last_peered":"2023-10-09T04:11:20.643990+0000","last_clean":"2023-10-09T04:11:20.643990+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:11:20.643990+0000","last_undegraded":"2023-10-09T04:11:20.643990+0000","last_fullsized":"2023-10-09T04:11:20.643990+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"212'4595","reported_seq":4663,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:24.768582+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:11:24.768582+0000","last_peered":"2023-10-09T04:11:24.768582+0000","last_clean":"2023-10-09T04:11:24.768582+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:11:24.768582+0000","last_undegraded":"2023-10-09T04:11:24.768582+0000","last_fullsized":"2023-10-09T04:11:24.768582+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4595,"log_dups_size":0,"ondisk_log_size":4595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149225472,"num_objects":2277,"num_object_clones":0,"num_object_copies":6831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3455,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":169,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.617970+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:11:04.617970+0000","last_peered":"2023-10-09T04:11:04.617970+0000","last_clean":"2023-10-09T04:11:04.617970+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:11:04.617970+0000","last_undegraded":"2023-10-09T04:11:04.617970+0000","last_fullsized":"2023-10-09T04:11:04.617970+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":167,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621681+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:11:20.621681+0000","last_peered":"2023-10-09T04:11:20.621681+0000","last_clean":"2023-10-09T04:11:20.621681+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:11:20.621681+0000","last_undegraded":"2023-10-09T04:11:20.621681+0000","last_fullsized":"2023-10-09T04:11:20.621681+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"212'7006","reported_seq":7059,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.830150+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:11:22.830150+0000","last_peered":"2023-10-09T04:11:22.830150+0000","last_clean":"2023-10-09T04:11:22.830150+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:11:22.830150+0000","last_undegraded":"2023-10-09T04:11:22.830150+0000","last_fullsized":"2023-10-09T04:11:22.830150+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7006,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":116,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621622+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:11:20.621622+0000","last_peered":"2023-10-09T04:11:20.621622+0000","last_clean":"2023-10-09T04:11:20.621622+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:11:20.621622+0000","last_undegraded":"2023-10-09T04:11:20.621622+0000","last_fullsized":"2023-10-09T04:11:20.621622+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649180+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:11:20.649180+0000","last_peered":"2023-10-09T04:11:20.649180+0000","last_clean":"2023-10-09T04:11:20.649180+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:11:20.649180+0000","last_undegraded":"2023-10-09T04:11:20.649180+0000","last_fullsized":"2023-10-09T04:11:20.649180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"210'6821","reported_seq":6889,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:11:20.558987+0000","last_change":"2023-10-09T04:10:38.444053+0000","last_active":"2023-10-09T04:11:20.558987+0000","last_peered":"2023-10-09T04:11:20.558987+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:11:20.558987+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314376192,"num_objects":4797,"num_object_clones":0,"num_object_copies":14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6821,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":2197},{"shards":"0,5","objects":2600}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649285+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:11:20.649285+0000","last_peered":"2023-10-09T04:11:20.649285+0000","last_clean":"2023-10-09T04:11:20.649285+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:11:20.649285+0000","last_undegraded":"2023-10-09T04:11:20.649285+0000","last_fullsized":"2023-10-09T04:11:20.649285+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649434+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:11:20.649434+0000","last_peered":"2023-10-09T04:11:20.649434+0000","last_clean":"2023-10-09T04:11:20.649434+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:11:20.649434+0000","last_undegraded":"2023-10-09T04:11:20.649434+0000","last_fullsized":"2023-10-09T04:11:20.649434+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"212'7003","reported_seq":7058,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:23.547192+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:11:23.547192+0000","last_peered":"2023-10-09T04:11:23.547192+0000","last_clean":"2023-10-09T04:11:23.547192+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:11:23.547192+0000","last_undegraded":"2023-10-09T04:11:23.547192+0000","last_fullsized":"2023-10-09T04:11:23.547192+0000","mapping_epoch":167,"log_start":"171'100","ondisk_log_start":"171'100","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302514200,"num_objects":4617,"num_object_clones":0,"num_object_copies":13851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4617,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7003,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649425+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:11:20.649425+0000","last_peered":"2023-10-09T04:11:20.649425+0000","last_clean":"2023-10-09T04:11:20.649425+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:11:20.649425+0000","last_undegraded":"2023-10-09T04:11:20.649425+0000","last_fullsized":"2023-10-09T04:11:20.649425+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649225+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:11:20.649225+0000","last_peered":"2023-10-09T04:11:20.649225+0000","last_clean":"2023-10-09T04:11:20.649225+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:11:20.649225+0000","last_undegraded":"2023-10-09T04:11:20.649225+0000","last_fullsized":"2023-10-09T04:11:20.649225+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"210'6695","reported_seq":6747,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.576547+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:11:20.576547+0000","last_peered":"2023-10-09T04:11:20.576547+0000","last_clean":"2023-10-09T04:11:20.576547+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:11:20.576547+0000","last_undegraded":"2023-10-09T04:11:20.576547+0000","last_fullsized":"2023-10-09T04:11:20.576547+0000","mapping_epoch":167,"log_start":"171'300","ondisk_log_start":"171'300","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310312960,"num_objects":4735,"num_object_clones":0,"num_object_copies":14205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6695,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":176,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:12.550253+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:11:12.550253+0000","last_peered":"2023-10-09T04:11:12.550253+0000","last_clean":"2023-10-09T04:11:12.550253+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:11:12.550253+0000","last_undegraded":"2023-10-09T04:11:12.550253+0000","last_fullsized":"2023-10-09T04:11:12.550253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644619+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:11:20.644619+0000","last_peered":"2023-10-09T04:11:20.644619+0000","last_clean":"2023-10-09T04:11:20.644619+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:11:20.644619+0000","last_undegraded":"2023-10-09T04:11:20.644619+0000","last_fullsized":"2023-10-09T04:11:20.644619+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"212'4682","reported_seq":4869,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:25.249735+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:11:25.249735+0000","last_peered":"2023-10-09T04:11:25.249735+0000","last_clean":"2023-10-09T04:11:25.249735+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:11:25.249735+0000","last_undegraded":"2023-10-09T04:11:25.249735+0000","last_fullsized":"2023-10-09T04:11:25.249735+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":4682,"log_dups_size":0,"ondisk_log_size":4682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151781376,"num_objects":2316,"num_object_clones":0,"num_object_copies":6948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1029,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644585+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:11:20.644585+0000","last_peered":"2023-10-09T04:11:20.644585+0000","last_clean":"2023-10-09T04:11:20.644585+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:11:20.644585+0000","last_undegraded":"2023-10-09T04:11:20.644585+0000","last_fullsized":"2023-10-09T04:11:20.644585+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":186,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.618055+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:11:04.618055+0000","last_peered":"2023-10-09T04:11:04.618055+0000","last_clean":"2023-10-09T04:11:04.618055+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:04.618055+0000","last_undegraded":"2023-10-09T04:11:04.618055+0000","last_fullsized":"2023-10-09T04:11:04.618055+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":149,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644122+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:11:20.644122+0000","last_peered":"2023-10-09T04:11:20.644122+0000","last_clean":"2023-10-09T04:11:20.644122+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:11:20.644122+0000","last_undegraded":"2023-10-09T04:11:20.644122+0000","last_fullsized":"2023-10-09T04:11:20.644122+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"212'4651","reported_seq":4715,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.738954+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:11:22.738954+0000","last_peered":"2023-10-09T04:11:22.738954+0000","last_clean":"2023-10-09T04:11:22.738954+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:11:22.738954+0000","last_undegraded":"2023-10-09T04:11:22.738954+0000","last_fullsized":"2023-10-09T04:11:22.738954+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4651,"log_dups_size":0,"ondisk_log_size":4651,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153944064,"num_objects":2349,"num_object_clones":0,"num_object_copies":7047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3497,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":173,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.618016+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:11:04.618016+0000","last_peered":"2023-10-09T04:11:04.618016+0000","last_clean":"2023-10-09T04:11:04.618016+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:11:04.618016+0000","last_undegraded":"2023-10-09T04:11:04.618016+0000","last_fullsized":"2023-10-09T04:11:04.618016+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649434+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:11:20.649434+0000","last_peered":"2023-10-09T04:11:20.649434+0000","last_clean":"2023-10-09T04:11:20.649434+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:11:20.649434+0000","last_undegraded":"2023-10-09T04:11:20.649434+0000","last_fullsized":"2023-10-09T04:11:20.649434+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":197,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644390+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:11:20.644390+0000","last_peered":"2023-10-09T04:11:20.644390+0000","last_clean":"2023-10-09T04:11:20.644390+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:11:20.644390+0000","last_undegraded":"2023-10-09T04:11:20.644390+0000","last_fullsized":"2023-10-09T04:11:20.644390+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"212'4713","reported_seq":4777,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.661236+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:11:22.661236+0000","last_peered":"2023-10-09T04:11:22.661236+0000","last_clean":"2023-10-09T04:11:22.661236+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:11:22.661236+0000","last_undegraded":"2023-10-09T04:11:22.661236+0000","last_fullsized":"2023-10-09T04:11:22.661236+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4713,"log_dups_size":0,"ondisk_log_size":4713,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152895488,"num_objects":2333,"num_object_clones":0,"num_object_copies":6999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3551,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.641910+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:11:20.641910+0000","last_peered":"2023-10-09T04:11:20.641910+0000","last_clean":"2023-10-09T04:11:20.641910+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:11:20.641910+0000","last_undegraded":"2023-10-09T04:11:20.641910+0000","last_fullsized":"2023-10-09T04:11:20.641910+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":181,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.643764+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:11:20.643764+0000","last_peered":"2023-10-09T04:11:20.643764+0000","last_clean":"2023-10-09T04:11:20.643764+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:11:20.643764+0000","last_undegraded":"2023-10-09T04:11:20.643764+0000","last_fullsized":"2023-10-09T04:11:20.643764+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":182,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644693+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:11:20.644693+0000","last_peered":"2023-10-09T04:11:20.644693+0000","last_clean":"2023-10-09T04:11:20.644693+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:11:20.644693+0000","last_undegraded":"2023-10-09T04:11:20.644693+0000","last_fullsized":"2023-10-09T04:11:20.644693+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"212'4489","reported_seq":4552,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.567548+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:11:22.567548+0000","last_peered":"2023-10-09T04:11:22.567548+0000","last_clean":"2023-10-09T04:11:22.567548+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:11:22.567548+0000","last_undegraded":"2023-10-09T04:11:22.567548+0000","last_fullsized":"2023-10-09T04:11:22.567548+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4489,"log_dups_size":0,"ondisk_log_size":4489,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154632192,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3371,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":193,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.617831+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:11:04.617831+0000","last_peered":"2023-10-09T04:11:04.617831+0000","last_clean":"2023-10-09T04:11:04.617831+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:11:04.617831+0000","last_undegraded":"2023-10-09T04:11:04.617831+0000","last_fullsized":"2023-10-09T04:11:04.617831+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":550,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635346+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:11:04.635346+0000","last_peered":"2023-10-09T04:11:04.635346+0000","last_clean":"2023-10-09T04:11:04.635346+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:11:04.635346+0000","last_undegraded":"2023-10-09T04:11:04.635346+0000","last_fullsized":"2023-10-09T04:11:04.635346+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":106,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.617933+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:11:04.617933+0000","last_peered":"2023-10-09T04:11:04.617933+0000","last_clean":"2023-10-09T04:11:04.617933+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:11:04.617933+0000","last_undegraded":"2023-10-09T04:11:04.617933+0000","last_fullsized":"2023-10-09T04:11:04.617933+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649620+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:11:20.649620+0000","last_peered":"2023-10-09T04:11:20.649620+0000","last_clean":"2023-10-09T04:11:20.649620+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:11:20.649620+0000","last_undegraded":"2023-10-09T04:11:20.649620+0000","last_fullsized":"2023-10-09T04:11:20.649620+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"210'4482","reported_seq":6826,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.469013+0000","last_change":"2023-10-09T04:10:53.276397+0000","last_active":"2023-10-09T04:11:20.469013+0000","last_peered":"2023-10-09T04:11:20.469013+0000","last_clean":"2023-10-09T04:11:20.469013+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:11:20.469013+0000","last_undegraded":"2023-10-09T04:11:20.469013+0000","last_fullsized":"2023-10-09T04:11:20.469013+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4482,"log_dups_size":0,"ondisk_log_size":4482,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157843456,"num_objects":2408,"num_object_clones":0,"num_object_copies":7224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3366,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":190,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635330+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:11:04.635330+0000","last_peered":"2023-10-09T04:11:04.635330+0000","last_clean":"2023-10-09T04:11:04.635330+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:11:04.635330+0000","last_undegraded":"2023-10-09T04:11:04.635330+0000","last_fullsized":"2023-10-09T04:11:04.635330+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621672+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:11:20.621672+0000","last_peered":"2023-10-09T04:11:20.621672+0000","last_clean":"2023-10-09T04:11:20.621672+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:11:20.621672+0000","last_undegraded":"2023-10-09T04:11:20.621672+0000","last_fullsized":"2023-10-09T04:11:20.621672+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":200,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621900+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:11:20.621900+0000","last_peered":"2023-10-09T04:11:20.621900+0000","last_clean":"2023-10-09T04:11:20.621900+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:11:20.621900+0000","last_undegraded":"2023-10-09T04:11:20.621900+0000","last_fullsized":"2023-10-09T04:11:20.621900+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"210'4740","reported_seq":4865,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.541294+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:11:20.541294+0000","last_peered":"2023-10-09T04:11:20.541294+0000","last_clean":"2023-10-09T04:11:20.541294+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:11:20.541294+0000","last_undegraded":"2023-10-09T04:11:20.541294+0000","last_fullsized":"2023-10-09T04:11:20.541294+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":4740,"log_dups_size":0,"ondisk_log_size":4740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158924800,"num_objects":2425,"num_object_clones":0,"num_object_copies":7275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2193,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":173,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635282+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:11:04.635282+0000","last_peered":"2023-10-09T04:11:04.635282+0000","last_clean":"2023-10-09T04:11:04.635282+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:11:04.635282+0000","last_undegraded":"2023-10-09T04:11:04.635282+0000","last_fullsized":"2023-10-09T04:11:04.635282+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":181,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644668+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:11:20.644668+0000","last_peered":"2023-10-09T04:11:20.644668+0000","last_clean":"2023-10-09T04:11:20.644668+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:11:20.644668+0000","last_undegraded":"2023-10-09T04:11:20.644668+0000","last_fullsized":"2023-10-09T04:11:20.644668+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":152,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621528+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:11:20.621528+0000","last_peered":"2023-10-09T04:11:20.621528+0000","last_clean":"2023-10-09T04:11:20.621528+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:11:20.621528+0000","last_undegraded":"2023-10-09T04:11:20.621528+0000","last_fullsized":"2023-10-09T04:11:20.621528+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"210'4698","reported_seq":7755,"reported_epoch":210,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:20.520932+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:11:20.520932+0000","last_peered":"2023-10-09T04:11:20.520932+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:11:20.520932+0000","last_undegraded":"2023-10-09T04:11:20.520932+0000","last_fullsized":"2023-10-09T04:11:20.520932+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4698,"log_dups_size":0,"ondisk_log_size":4698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160497664,"num_objects":2449,"num_object_clones":0,"num_object_copies":7347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2449,"num_objects_unfound":0,"num_objects_dirty":2449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3527,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2449}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":114,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.634720+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:11:04.634720+0000","last_peered":"2023-10-09T04:11:04.634720+0000","last_clean":"2023-10-09T04:11:04.634720+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:11:04.634720+0000","last_undegraded":"2023-10-09T04:11:04.634720+0000","last_fullsized":"2023-10-09T04:11:04.634720+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":148,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644948+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:11:20.644948+0000","last_peered":"2023-10-09T04:11:20.644948+0000","last_clean":"2023-10-09T04:11:20.644948+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:11:20.644948+0000","last_undegraded":"2023-10-09T04:11:20.644948+0000","last_fullsized":"2023-10-09T04:11:20.644948+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.635411+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:11:04.635411+0000","last_peered":"2023-10-09T04:11:04.635411+0000","last_clean":"2023-10-09T04:11:04.635411+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:11:04.635411+0000","last_undegraded":"2023-10-09T04:11:04.635411+0000","last_fullsized":"2023-10-09T04:11:04.635411+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"212'4696","reported_seq":6903,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:23.512334+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:11:23.512334+0000","last_peered":"2023-10-09T04:11:23.512334+0000","last_clean":"2023-10-09T04:11:23.512334+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:11:23.512334+0000","last_undegraded":"2023-10-09T04:11:23.512334+0000","last_fullsized":"2023-10-09T04:11:23.512334+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":4696,"log_dups_size":0,"ondisk_log_size":4696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150142976,"num_objects":2291,"num_object_clones":0,"num_object_copies":6873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2219,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":138,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.634842+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:11:04.634842+0000","last_peered":"2023-10-09T04:11:04.634842+0000","last_clean":"2023-10-09T04:11:04.634842+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:11:04.634842+0000","last_undegraded":"2023-10-09T04:11:04.634842+0000","last_fullsized":"2023-10-09T04:11:04.634842+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621987+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:11:20.621987+0000","last_peered":"2023-10-09T04:11:20.621987+0000","last_clean":"2023-10-09T04:11:20.621987+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:11:20.621987+0000","last_undegraded":"2023-10-09T04:11:20.621987+0000","last_fullsized":"2023-10-09T04:11:20.621987+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649510+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:11:20.649510+0000","last_peered":"2023-10-09T04:11:20.649510+0000","last_clean":"2023-10-09T04:11:20.649510+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:11:20.649510+0000","last_undegraded":"2023-10-09T04:11:20.649510+0000","last_fullsized":"2023-10-09T04:11:20.649510+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":155,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.621721+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:11:20.621721+0000","last_peered":"2023-10-09T04:11:20.621721+0000","last_clean":"2023-10-09T04:11:20.621721+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:11:20.621721+0000","last_undegraded":"2023-10-09T04:11:20.621721+0000","last_fullsized":"2023-10-09T04:11:20.621721+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.642022+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:11:20.642022+0000","last_peered":"2023-10-09T04:11:20.642022+0000","last_clean":"2023-10-09T04:11:20.642022+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:11:20.642022+0000","last_undegraded":"2023-10-09T04:11:20.642022+0000","last_fullsized":"2023-10-09T04:11:20.642022+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"212'4703","reported_seq":4769,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:24.921374+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:11:24.921374+0000","last_peered":"2023-10-09T04:11:24.921374+0000","last_clean":"2023-10-09T04:11:24.921374+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:11:24.921374+0000","last_undegraded":"2023-10-09T04:11:24.921374+0000","last_fullsized":"2023-10-09T04:11:24.921374+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4703,"log_dups_size":0,"ondisk_log_size":4703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147783680,"num_objects":2255,"num_object_clones":0,"num_object_copies":6765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3563,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":171,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.618057+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:11:04.618057+0000","last_peered":"2023-10-09T04:11:04.618057+0000","last_clean":"2023-10-09T04:11:04.618057+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:11:04.618057+0000","last_undegraded":"2023-10-09T04:11:04.618057+0000","last_fullsized":"2023-10-09T04:11:04.618057+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649466+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:11:20.649466+0000","last_peered":"2023-10-09T04:11:20.649466+0000","last_clean":"2023-10-09T04:11:20.649466+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:11:20.649466+0000","last_undegraded":"2023-10-09T04:11:20.649466+0000","last_fullsized":"2023-10-09T04:11:20.649466+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"212'4693","reported_seq":4754,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.775200+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:11:22.775200+0000","last_peered":"2023-10-09T04:11:22.775200+0000","last_clean":"2023-10-09T04:11:22.775200+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:11:22.775200+0000","last_undegraded":"2023-10-09T04:11:22.775200+0000","last_fullsized":"2023-10-09T04:11:22.775200+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155516928,"num_objects":2373,"num_object_clones":0,"num_object_copies":7119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3531,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":154,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-09T04:11:04.618125+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:11:04.618125+0000","last_peered":"2023-10-09T04:11:04.618125+0000","last_clean":"2023-10-09T04:11:04.618125+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:11:04.618125+0000","last_undegraded":"2023-10-09T04:11:04.618125+0000","last_fullsized":"2023-10-09T04:11:04.618125+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649304+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:11:20.649304+0000","last_peered":"2023-10-09T04:11:20.649304+0000","last_clean":"2023-10-09T04:11:20.649304+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:11:20.649304+0000","last_undegraded":"2023-10-09T04:11:20.649304+0000","last_fullsized":"2023-10-09T04:11:20.649304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.642381+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:11:20.642381+0000","last_peered":"2023-10-09T04:11:20.642381+0000","last_clean":"2023-10-09T04:11:20.642381+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:11:20.642381+0000","last_undegraded":"2023-10-09T04:11:20.642381+0000","last_fullsized":"2023-10-09T04:11:20.642381+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.641980+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:11:20.641980+0000","last_peered":"2023-10-09T04:11:20.641980+0000","last_clean":"2023-10-09T04:11:20.641980+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:11:20.641980+0000","last_undegraded":"2023-10-09T04:11:20.641980+0000","last_fullsized":"2023-10-09T04:11:20.641980+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":201,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.622058+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:11:20.622058+0000","last_peered":"2023-10-09T04:11:20.622058+0000","last_clean":"2023-10-09T04:11:20.622058+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:11:20.622058+0000","last_undegraded":"2023-10-09T04:11:20.622058+0000","last_fullsized":"2023-10-09T04:11:20.622058+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.648997+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:11:20.648997+0000","last_peered":"2023-10-09T04:11:20.648997+0000","last_clean":"2023-10-09T04:11:20.648997+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:11:20.648997+0000","last_undegraded":"2023-10-09T04:11:20.648997+0000","last_fullsized":"2023-10-09T04:11:20.648997+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4925882392,"num_objects":75164,"num_object_clones":0,"num_object_copies":225492,"num_objects_missing_on_primary":452,"num_objects_missing":452,"num_objects_degraded":2197,"num_objects_misplaced":9298,"num_objects_unfound":0,"num_objects_dirty":75164,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104933,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10335,"num_bytes_recovered":677314560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15193878528,"data_stored":15193866312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133867,"ondisk_log_size":133867,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201506,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2116400,"kb_used_data":1806824,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91628752,"statfs":{"total":95995035648,"available":93827842048,"internally_reserved":0,"allocated":1850187776,"data_stored":1849561374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.825,"15min":0.94899999999999995},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":3.181,"5min":17.398,"15min":17.398},"last":0.317},{"interface":"front","average":{"1min":1.087,"5min":1.7829999999999999,"15min":0.93999999999999995},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.3630000000000004,"5min":17.306999999999999,"15min":17.306999999999999},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:10:47 2023","interfaces":[{"interface":"back","average":{"1min":11.227,"5min":3.9630000000000001,"15min":1.736},"min":{"1min":0.28499999999999998,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":245.28700000000001,"5min":245.28700000000001,"15min":245.28700000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":11.473000000000001,"5min":3.8130000000000002,"15min":1.7470000000000001},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":244.626,"5min":244.626,"15min":244.626},"last":0.41099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:10:24 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":2.5289999999999999,"15min":1.208},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.8049999999999997,"5min":58.615000000000002,"15min":58.615000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.095,"5min":3.6859999999999999,"15min":1.6319999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":3.484,"5min":99.372,"15min":99.372},"last":0.379}]},{"osd":3,"last update":"Mon Oct 9 04:10:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0139999999999998,"15min":1.534},"min":{"1min":0.34399999999999997,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":6.1150000000000002,"5min":99.134,"15min":99.134},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.329,"5min":3.1459999999999999,"15min":1.611},"min":{"1min":0.39100000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.1310000000000002,"5min":99.117999999999995,"15min":99.117999999999995},"last":1.2350000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:10:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.1160000000000001,"15min":0.69799999999999995},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.195,"5min":16.195,"15min":16.195},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.103,"15min":0.71599999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27500000000000002},"max":{"1min":4.0949999999999998,"5min":12.442,"15min":12.442},"last":0.34999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.79500000000000004,"15min":0.60199999999999998},"min":{"1min":0.316,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.6990000000000001,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.95799999999999996,"15min":0.68100000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.9359999999999999,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.95899999999999996,"15min":0.68000000000000005},"min":{"1min":0.35299999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.89900000000000002,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.0680000000000001,"15min":0.745},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":4.9089999999999998,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.497}]}]},{"osd":6,"up_from":47,"seq":201863463144,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2197116,"kb_used_data":1887540,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91548036,"statfs":{"total":95995035648,"available":93745188864,"internally_reserved":0,"allocated":1932840960,"data_stored":1932218399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7850000000000001,"5min":4.2990000000000004,"15min":1.8460000000000001},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":60.295000000000002,"5min":78.983999999999995,"15min":78.983999999999995},"last":4.4539999999999997},{"interface":"front","average":{"1min":6.8220000000000001,"5min":4.8659999999999997,"15min":2.0830000000000002},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":60.716000000000001,"5min":78.998000000000005,"15min":78.998000000000005},"last":3.0449999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.71,"15min":1.619},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":62.698,"5min":62.698,"15min":62.698},"last":0.308},{"interface":"front","average":{"1min":3.8420000000000001,"5min":3.895,"15min":1.7010000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.530999999999999,"5min":63.530999999999999,"15min":63.530999999999999},"last":1.7689999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":7.6840000000000002,"5min":3.8420000000000001,"15min":1.8420000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":77.953000000000003,"5min":77.953000000000003,"15min":77.953000000000003},"last":2.4129999999999998},{"interface":"front","average":{"1min":7.7320000000000002,"5min":3.875,"15min":1.8879999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":78.019000000000005,"5min":78.019000000000005,"15min":78.019000000000005},"last":2.4420000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0720000000000001,"5min":4.0350000000000001,"15min":1.911},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":53.497999999999998,"5min":57.509999999999998,"15min":57.509999999999998},"last":4.8399999999999999},{"interface":"front","average":{"1min":3.4910000000000001,"5min":4.3529999999999998,"15min":2.0219999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":53.460999999999999,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.51700000000000002,"15min":0.48799999999999999},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.264,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.434},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.84699999999999998,"15min":0.622},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.363,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.49099999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.224,"15min":0.75900000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.2849999999999999,"5min":54.773000000000003,"15min":54.773000000000003},"last":2.2549999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.3899999999999999,"15min":0.85199999999999998},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4159999999999999,"5min":54.853999999999999,"15min":54.853999999999999},"last":2.2669999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.69999999999999996,"15min":0.61499999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":2.903,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":2.1120000000000001},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.80900000000000005,"15min":0.68500000000000005},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.28699999999999998},"max":{"1min":2.835,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":2.1269999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089452,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2193708,"kb_used_data":1987428,"kb_used_omap":9,"kb_used_meta":206262,"kb_avail":91551444,"statfs":{"total":95995035648,"available":93748678656,"internally_reserved":0,"allocated":2035126272,"data_stored":2034283461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":211213289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":3.3450000000000002,"15min":1.4390000000000001},"min":{"1min":0.20899999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.833,"5min":85.977999999999994,"15min":85.977999999999994},"last":2.4350000000000001},{"interface":"front","average":{"1min":1.722,"5min":4.6120000000000001,"15min":1.8819999999999999},"min":{"1min":0.22500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.664,"5min":99.831999999999994,"15min":99.831999999999994},"last":2.3069999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":3.6469999999999998,"15min":1.5369999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.6,"5min":121.54600000000001,"15min":121.54600000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.3010000000000002,"5min":3.7919999999999998,"15min":1.702},"min":{"1min":0.23200000000000001,"5min":0.186,"15min":0.186},"max":{"1min":11.052,"5min":99.899000000000001,"15min":99.899000000000001},"last":10.460000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":4.8959999999999999,"15min":1.98},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":138.87299999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":10.010999999999999,"5min":6.2439999999999998,"15min":2.4169999999999998},"min":{"1min":0.30599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":138.49799999999999,"5min":138.49799999999999,"15min":138.49799999999999},"last":10.509}]},{"osd":4,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.5099999999999998,"15min":1.343},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.5219999999999998,"5min":99.989999999999995,"15min":99.989999999999995},"last":2.3500000000000001},{"interface":"front","average":{"1min":1.3220000000000001,"5min":2.5470000000000002,"15min":1.351},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3330000000000002,"5min":100.249,"15min":100.249},"last":1.341}]},{"osd":5,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.4350000000000001,"15min":1.3360000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4820000000000002,"5min":99.933999999999997,"15min":99.933999999999997},"last":2.3900000000000001},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.798,"15min":1.45},"min":{"1min":0.38300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.2789999999999999,"5min":100.34099999999999,"15min":100.34099999999999},"last":1.6659999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":3.1110000000000002,"15min":1.5549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":100.309,"15min":100.309},"last":1.3879999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.7450000000000001,"15min":1.4550000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4020000000000001,"5min":100.143,"15min":100.143},"last":0.433}]},{"osd":7,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":2.988,"15min":1.534},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.2210000000000001,"5min":100.214,"15min":100.214},"last":1.4199999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.7450000000000001,"15min":1.4750000000000001},"min":{"1min":0.374,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":3.335,"5min":100.113,"15min":100.113},"last":0.60499999999999998}]}]},{"osd":0,"up_from":84,"seq":360777252978,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2775232,"kb_used_data":2545272,"kb_used_omap":28,"kb_used_meta":229859,"kb_avail":90969920,"statfs":{"total":95995035648,"available":93153198080,"internally_reserved":0,"allocated":2606358528,"data_stored":2605494700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":235376464},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":4.5599999999999996,"15min":1.819},"min":{"1min":0.248,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":7.5949999999999998,"5min":118.48,"15min":118.48},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.5760000000000001,"5min":5.0810000000000004,"15min":1.9930000000000001},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":7.2599999999999998,"5min":118.384,"15min":118.384},"last":4.3049999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:11:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":4.5990000000000002,"15min":1.9419999999999999},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.8819999999999997,"5min":118.54000000000001,"15min":118.54000000000001},"last":4.0490000000000004},{"interface":"front","average":{"1min":1.736,"5min":5.0780000000000003,"15min":2.089},"min":{"1min":0.21199999999999999,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":8.2300000000000004,"5min":118.727,"15min":118.727},"last":0.57599999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":8.7230000000000008,"5min":8.109,"15min":3.3610000000000002},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":156.488,"5min":156.488,"15min":156.488},"last":5.6779999999999999},{"interface":"front","average":{"1min":9.0649999999999995,"5min":8.1379999999999999,"15min":3.4390000000000001},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":156.50700000000001,"5min":156.50700000000001,"15min":156.50700000000001},"last":4.0170000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:10:31 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":3.4540000000000002,"15min":1.5549999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3780000000000001,"5min":118.633,"15min":118.633},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.153,"5min":4.3639999999999999,"15min":1.861},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.0110000000000001,"5min":118.913,"15min":118.913},"last":0.497}]},{"osd":5,"last update":"Mon Oct 9 04:10:31 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":5.8689999999999998,"15min":2.3700000000000001},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.9340000000000002,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.145,"5min":5.3029999999999999,"15min":2.1930000000000001},"min":{"1min":0.33300000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.9950000000000001,"5min":182.66300000000001,"15min":182.66300000000001},"last":0.376}]},{"osd":6,"last update":"Mon Oct 9 04:10:31 2023","interfaces":[{"interface":"back","average":{"1min":3.758,"5min":3.8450000000000002,"15min":1.72},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":52.697000000000003,"5min":118.864,"15min":118.864},"last":0.40999999999999998},{"interface":"front","average":{"1min":4.0069999999999997,"5min":3.5379999999999998,"15min":1.6180000000000001},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":52.674999999999997,"5min":118.70399999999999,"15min":118.70399999999999},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:10:31 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":5.8920000000000003,"15min":2.399},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.508,"5min":185.024,"15min":185.024},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.3600000000000001,"5min":5.2830000000000004,"15min":2.2160000000000002},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.3460000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.63600000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925926,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700380,"kb_used_data":1425044,"kb_used_omap":81,"kb_used_meta":275246,"kb_avail":92044772,"statfs":{"total":95995035648,"available":94253846528,"internally_reserved":0,"allocated":1459245056,"data_stored":1458416327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":281852122},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.4550000000000001,"15min":1.1419999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.413,"5min":12.284000000000001,"15min":12.284000000000001},"last":2.7069999999999999},{"interface":"front","average":{"1min":2.5840000000000001,"5min":3.2650000000000001,"15min":1.5189999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":11.356999999999999,"5min":45.979999999999997,"15min":45.979999999999997},"last":0.23899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":3.6539999999999999,"15min":1.8160000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":14.99,"5min":53.695,"15min":68.009},"last":1.405},{"interface":"front","average":{"1min":2.5979999999999999,"5min":3.8730000000000002,"15min":1.919},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":16.597000000000001,"5min":53.439,"15min":66.975999999999999},"last":0.377}]},{"osd":3,"last update":"Mon Oct 9 04:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":2.2749999999999999,"15min":1.173},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2409999999999997,"5min":12.77,"15min":27.562000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.746,"5min":3.0459999999999998,"15min":1.405},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.952,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.313}]},{"osd":4,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":2.419,"15min":1.3280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9209999999999998,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.524,"5min":1.7450000000000001,"15min":1.0329999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.5809999999999995,"5min":14.305,"15min":14.305},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.601,"15min":1.373},"min":{"1min":0.373,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":26.667000000000002,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.7999999999999998,"5min":2.8620000000000001,"15min":1.5},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":26.654,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.40600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0199999999999996,"5min":2.9830000000000001,"15min":1.4690000000000001},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":114.14,"5min":114.14,"15min":114.14},"last":114.14},{"interface":"front","average":{"1min":7.8810000000000002,"5min":3.6789999999999998,"15min":1.7549999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":114.123,"5min":114.123,"15min":114.123},"last":114.123}]},{"osd":7,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.1690000000000005,"5min":3.4550000000000001,"15min":1.651},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":117.32599999999999,"5min":117.32599999999999,"15min":117.32599999999999},"last":117.32599999999999},{"interface":"front","average":{"1min":8.1639999999999997,"5min":3.9079999999999999,"15min":1.8540000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":117.28700000000001,"5min":117.28700000000001,"15min":117.28700000000001},"last":117.28700000000001}]}]},{"osd":3,"up_from":99,"seq":425201762399,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3241772,"kb_used_data":2853284,"kb_used_omap":7,"kb_used_meta":388472,"kb_avail":90503380,"statfs":{"total":95995035648,"available":92675461120,"internally_reserved":0,"allocated":2921762816,"data_stored":2920963147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":397795944},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":7.7279999999999998,"5min":5.4539999999999997,"15min":3.0920000000000001},"min":{"1min":0.25600000000000001,"5min":0.189,"15min":0.185},"max":{"1min":100.512,"5min":100.512,"15min":106.22},"last":3.0680000000000001},{"interface":"front","average":{"1min":8.4130000000000003,"5min":5.7060000000000004,"15min":3.1930000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":101.733,"5min":101.733,"15min":106.986},"last":6.8079999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.1470000000000002,"15min":2.069},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.19800000000000001},"max":{"1min":57.055999999999997,"5min":64.677000000000007,"15min":106.121},"last":5.1539999999999999},{"interface":"front","average":{"1min":7.2649999999999997,"5min":4.7859999999999996,"15min":1.8660000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":57.088999999999999,"5min":65.015000000000001,"15min":65.015000000000001},"last":0.23999999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":3.5609999999999999,"15min":1.905},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":4.593,"5min":85.974000000000004,"15min":106.105},"last":0.40100000000000002},{"interface":"front","average":{"1min":6.7130000000000001,"5min":4.79,"15min":2.3079999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":98.765000000000001,"5min":98.765000000000001,"15min":106.70399999999999},"last":3.6190000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":2.8090000000000002,"15min":1.7869999999999999},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.308},"max":{"1min":5.4740000000000002,"5min":37.652000000000001,"15min":106.85899999999999},"last":3.6949999999999998},{"interface":"front","average":{"1min":1.353,"5min":2.5350000000000001,"15min":1.266},"min":{"1min":0.32500000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.2409999999999997,"5min":37.442,"15min":37.442},"last":5.0309999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":2.2330000000000001,"15min":1.1559999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3150000000000004,"5min":12.819000000000001,"15min":12.819000000000001},"last":5.0979999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.1539999999999999,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.0789999999999997,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":2.004,"15min":1.8260000000000001},"min":{"1min":0.35299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1100000000000003,"5min":13.255000000000001,"15min":106.036},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.528,"5min":2.3759999999999999,"15min":1.6619999999999999},"min":{"1min":0.40400000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.4050000000000002,"5min":15.622,"15min":106.741},"last":3.7200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":2.5009999999999999,"15min":1.7090000000000001},"min":{"1min":0.38200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.5069999999999997,"5min":15.685,"15min":106.82599999999999},"last":3.6669999999999998},{"interface":"front","average":{"1min":1.224,"5min":2.2429999999999999,"15min":1.5980000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.266,"5min":13.228999999999999,"15min":106.084},"last":5.1289999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986416,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1642940,"kb_used_data":1337460,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92102212,"statfs":{"total":95995035648,"available":94312665088,"internally_reserved":0,"allocated":1369559040,"data_stored":1368936045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":2.8290000000000002,"15min":1.3500000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":1.726,"5min":63.128,"15min":63.128},"last":0.223},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.5110000000000001,"15min":1.3149999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1600000000000001,"5min":62.645000000000003,"15min":62.645000000000003},"last":3.1499999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":2.7330000000000001,"15min":1.284},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1189999999999998,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.321,"5min":2.714,"15min":1.276},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.9260000000000002,"5min":75.314999999999998,"15min":75.314999999999998},"last":4.1760000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":1.9890000000000001,"15min":1.0509999999999999},"min":{"1min":0.315,"5min":0.245,"15min":0.245},"max":{"1min":2.3730000000000002,"5min":19.356000000000002,"15min":19.356000000000002},"last":0.374},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.8859999999999999,"15min":1.004},"min":{"1min":0.30599999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.3820000000000001,"5min":19.324000000000002,"15min":19.324000000000002},"last":0.34999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.863,"15min":1.0169999999999999},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.5629999999999997,"5min":14.694000000000001,"15min":14.694000000000001},"last":2.4489999999999998},{"interface":"front","average":{"1min":1.927,"5min":2.1320000000000001,"15min":1.125},"min":{"1min":0.48999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":7.4909999999999997,"5min":10.476000000000001,"15min":10.476000000000001},"last":4.3979999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.77300000000000002,"15min":0.57899999999999996},"min":{"1min":0.27700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.72199999999999998,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.1279999999999999,"15min":0.71899999999999997},"min":{"1min":0.314,"5min":0.25,"15min":0.25},"max":{"1min":1.544,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.31}]},{"osd":6,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.0920000000000001,"15min":0.72699999999999998},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.6180000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.871,"15min":0.65400000000000003},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.669,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.38500000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.96999999999999997,"15min":0.69899999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.4890000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.004,"15min":0.76100000000000001},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.306,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":1.115}]}]},{"osd":5,"up_from":39,"seq":167503724779,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1395668,"kb_used_data":1094284,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92349484,"statfs":{"total":95995035648,"available":94565871616,"internally_reserved":0,"allocated":1120546816,"data_stored":1119943893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.4100000000000001,"15min":1.1899999999999999},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.6629999999999998,"5min":63.037999999999997,"15min":63.037999999999997},"last":0.245},{"interface":"front","average":{"1min":0.57999999999999996,"5min":2.552,"15min":1.2230000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.1680000000000001,"5min":62.927999999999997,"15min":62.927999999999997},"last":6.9109999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2040000000000002,"15min":1.446},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.4980000000000002,"5min":91.212999999999994,"15min":91.212999999999994},"last":6.8620000000000001},{"interface":"front","average":{"1min":1.252,"5min":3.75,"15min":1.6439999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.9460000000000002,"5min":91.254000000000005,"15min":91.254000000000005},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 04:10:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":3.3450000000000002,"15min":1.498},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.7809999999999997,"5min":61.310000000000002,"15min":61.310000000000002},"last":1.587},{"interface":"front","average":{"1min":0.88500000000000001,"5min":3.1890000000000001,"15min":1.423},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.6029999999999998,"5min":61.154000000000003,"15min":61.154000000000003},"last":7.6929999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:10:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.04,"15min":1.657},"min":{"1min":0.35099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.1699999999999999,"5min":91.834999999999994,"15min":91.834999999999994},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.4159999999999999,"5min":3.0760000000000001,"15min":1.6819999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.1100000000000003,"5min":91.802000000000007,"15min":91.802000000000007},"last":0.68200000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.96199999999999997,"15min":0.64400000000000002},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.253},"max":{"1min":2.1299999999999999,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.097,"15min":0.70999999999999996},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":1.6539999999999999,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.39400000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.91400000000000003,"15min":0.66100000000000003},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.4059999999999999,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.88500000000000001,"15min":0.67500000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":1.4630000000000001,"5min":5.7670000000000003,"15min":5.7670000000000003},"last":0.38}]},{"osd":7,"last update":"Mon Oct 9 04:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.1859999999999999,"15min":0.78700000000000003},"min":{"1min":0.35599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.7789999999999999,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.159,"15min":0.80300000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":1.5269999999999999,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":0.40500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2587824128,"data_stored":2587820056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2024280064,"data_stored":2024275992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1445920768,"data_stored":1445920768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2895118336,"data_stored":2895118336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1360592896,"data_stored":1360592896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1107099648,"data_stored":1107099648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1930170368,"data_stored":1930166296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1842872320,"data_stored":1842872320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:11:29.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:11:29.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:29.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:29.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:29.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:29.547 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:29.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:29.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:29.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:29.891 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:30.008 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:11:30.008 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:11:30.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:11:30.008 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:11:30.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118973 6.b 6:d98e6576:::benchmark_data_smithi163_287034_object10710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.595615+0000", 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 285.403749688, 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89643357099999998, 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:30.009 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118973 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595615+0000", 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595615+0000", 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595613+0000", 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595620+0000", 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3160000000000002e-06 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595634+0000", 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4226000000000001e-05 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595641+0000", 2023-10-09T04:11:30.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8870000000000003e-06 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595718+0000", 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7503999999999995e-05 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.595724+0000", 2023-10-09T04:11:30.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9780000000000002e-06 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.480295+0000", 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88457030800000003 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.482804+0000", 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025097090000000002 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.482878+0000", 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3585999999999996e-05 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491795+0000", 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089168819999999992 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491864+0000", 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9006999999999998e-05 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491958+0000", 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3683000000000004e-05 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492006+0000", 2023-10-09T04:11:30.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8560999999999998e-05 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492048+0000", 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1967e-05 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118988 6.b 6:d6a481af:::benchmark_data_smithi163_287034_object10725:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.615121+0000", 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 285.38424360300002, 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.877034545, 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:30.019 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118988 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615121+0000", 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615121+0000", 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615120+0000", 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615125+0000", 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4750000000000001e-06 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615137+0000", 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1885000000000001e-05 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615142+0000", 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7570000000000004e-06 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615228+0000", 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5951000000000004e-05 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.615234+0000", 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3389999999999998e-06 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.485807+0000", 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87057329500000002 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.485903+0000", 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5869e-05 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.485969+0000", 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6023000000000001e-05 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491830+0000", 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0058606880000000002 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491896+0000", 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5672000000000004e-05 2023-10-09T04:11:30.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492083+0000", 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018750900000000001 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492123+0000", 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9688999999999999e-05 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492155+0000", 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2382999999999998e-05 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36616 7.5 7:a003475e:::benchmark_data_smithi163_346082_object36615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.301619+0000", 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.697745693, 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98538146599999998, 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36616 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.301619+0000", 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.301619+0000", 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.301617+0000", 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.301673+0000", 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6054000000000002e-05 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.301693+0000", 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0136000000000002e-05 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.301703+0000", 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8889999999999997e-06 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.326701+0000", 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.02499852 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.326709+0000", 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4050000000000003e-06 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.274169+0000", 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94746079400000005 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.274914+0000", 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074465399999999998 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.286928+0000", 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012013660000000001 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.287000+0000", 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2309000000000002e-05 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36621 7.15 7:ae84a454:::benchmark_data_smithi163_346082_object36620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:11:30.036 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.379339+0000", 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.620025707, 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91469847000000004, 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36621 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.037 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379339+0000", 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379339+0000", 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379337+0000", 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379393+0000", 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6060999999999997e-05 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379412+0000", 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9521000000000002e-05 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379420+0000", 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.605e-06 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379464+0000", 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4453999999999998e-05 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.379474+0000", 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.022e-05 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.287634+0000", 2023-10-09T04:11:30.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90815953800000004 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.287764+0000", 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013037899999999999 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:11:30.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.287929+0000", 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016451700000000001 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.288763+0000", 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000834083 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.290271+0000", 2023-10-09T04:11:30.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001508162 2023-10-09T04:11:30.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.293928+0000", 2023-10-09T04:11:30.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003656374 2023-10-09T04:11:30.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.293987+0000", 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9781999999999998e-05 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.294037+0000", 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9636999999999999e-05 2023-10-09T04:11:30.047 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:58923 7.15 7:af81a40f:::benchmark_data_smithi163_346082_object58922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.281027+0000", 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.718336899999997, 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301018, 2023-10-09T04:11:30.048 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 58923 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281027+0000", 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281027+0000", 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281026+0000", 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281079+0000", 2023-10-09T04:11:30.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2969000000000001e-05 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281094+0000", 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4844999999999999e-05 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281101+0000", 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2219999999999996e-06 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281157+0000", 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5467999999999998e-05 2023-10-09T04:11:30.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281165+0000", 2023-10-09T04:11:30.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3739999999999998e-06 2023-10-09T04:11:30.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.308095+0000", 2023-10-09T04:11:30.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.026930361 2023-10-09T04:11:30.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.308204+0000", 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108232 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.308306+0000", 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010252299999999999 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310963+0000", 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026565629999999998 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310971+0000", 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3459999999999994e-06 2023-10-09T04:11:30.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311061+0000", 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9554000000000004e-05 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311091+0000", 2023-10-09T04:11:30.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0043e-05 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311129+0000", 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8433999999999998e-05 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:58929 7.15 7:abb9695c:::benchmark_data_smithi163_346082_object58928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.287449+0000", 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.711915116, 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.025253508, 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.060 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 58929 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287449+0000", 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287449+0000", 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287448+0000", 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287560+0000", 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112825 2023-10-09T04:11:30.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287575+0000", 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4942e-05 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287582+0000", 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3300000000000004e-06 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287706+0000", 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123996 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287715+0000", 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5380000000000008e-06 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.309383+0000", 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0216676739999999 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.309460+0000", 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7250999999999996e-05 2023-10-09T04:11:30.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.309569+0000", 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109252 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311856+0000", 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022868570000000002 2023-10-09T04:11:30.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312607+0000", 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00075095399999999997 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312640+0000", 2023-10-09T04:11:30.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2662e-05 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312668+0000", 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7776e-05 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312703+0000", 2023-10-09T04:11:30.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4949e-05 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:58966 7.15 7:addae021:::benchmark_data_smithi163_346082_object58965:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.312752+0000", 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.686611854999995, 2023-10-09T04:11:30.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0000743679999999, 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 58966 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312752+0000", 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312752+0000", 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312751+0000", 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312813+0000", 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2435999999999996e-05 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312826+0000", 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2473000000000001e-05 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312831+0000", 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1630000000000004e-06 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312889+0000", 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8561999999999999e-05 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.312899+0000", 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5429999999999999e-06 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310100+0000", 2023-10-09T04:11:30.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99720094000000004 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310217+0000", 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116659 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310311+0000", 2023-10-09T04:11:30.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4424000000000003e-05 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312613+0000", 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023018510000000002 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312737+0000", 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012388699999999999 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312768+0000", 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0902000000000003e-05 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312794+0000", 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.667e-05 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312827+0000", 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2387e-05 2023-10-09T04:11:30.080 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:58971 7.15 7:abb5104f:::benchmark_data_smithi163_346082_object58970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.316688+0000", 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.682675986999996, 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99626170000000003, 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.081 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 58971 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316688+0000", 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316688+0000", 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316687+0000", 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316742+0000", 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4737999999999997e-05 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316753+0000", 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1301000000000001e-05 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316759+0000", 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9139999999999996e-06 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316809+0000", 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0485000000000002e-05 2023-10-09T04:11:30.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.316821+0000", 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1469e-05 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311150+0000", 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99432907299999995 2023-10-09T04:11:30.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311251+0000", 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100783 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311335+0000", 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4283999999999995e-05 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312741+0000", 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014064119999999999 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312860+0000", 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011900099999999999 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312893+0000", 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2100000000000001e-05 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312919+0000", 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6706000000000001e-05 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312950+0000", 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0661999999999999e-05 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:59007 7.15 7:af8e08a0:::benchmark_data_smithi163_346082_object59006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.342150+0000", 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.657213756000004, 2023-10-09T04:11:30.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97269206200000002, 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59007 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342150+0000", 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342150+0000", 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342149+0000", 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342207+0000", 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8388999999999999e-05 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342220+0000", 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2328e-05 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342225+0000", 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9420000000000003e-06 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342276+0000", 2023-10-09T04:11:30.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1786999999999997e-05 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.342287+0000", 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0312000000000001e-05 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311891+0000", 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.969604511 2023-10-09T04:11:30.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311965+0000", 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3668e-05 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312043+0000", 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7936999999999996e-05 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312865+0000", 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00082229099999999995 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.314706+0000", 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018413049999999999 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.314758+0000", 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1239999999999997e-05 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.314806+0000", 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8350999999999998e-05 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.314843+0000", 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6434999999999999e-05 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:59063 7.15 7:adf2f67a:::benchmark_data_smithi163_346082_object59062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.385240+0000", 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.614123762999995, 2023-10-09T04:11:30.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93139765799999996, 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59063 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385240+0000", 2023-10-09T04:11:30.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385240+0000", 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385239+0000", 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385295+0000", 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5646000000000002e-05 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385310+0000", 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5441e-05 2023-10-09T04:11:30.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385317+0000", 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3609999999999997e-06 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385375+0000", 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.889e-05 2023-10-09T04:11:30.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.385387+0000", 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1447000000000001e-05 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312977+0000", 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92759008499999995 2023-10-09T04:11:30.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.313046+0000", 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9278999999999999e-05 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.313129+0000", 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2985000000000006e-05 2023-10-09T04:11:30.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316459+0000", 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033299660000000002 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316499+0000", 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0005000000000001e-05 2023-10-09T04:11:30.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316543+0000", 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3643999999999998e-05 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316594+0000", 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1557000000000001e-05 2023-10-09T04:11:30.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316638+0000", 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3668999999999998e-05 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:59090 7.15 7:aaf518a5:::benchmark_data_smithi163_346082_object59089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.405300+0000", 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.594064028999995, 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91564208700000005, 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59090 2023-10-09T04:11:30.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.405300+0000", 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.405300+0000", 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.405300+0000", 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.405361+0000", 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1420000000000005e-05 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.405371+0000", 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0178e-05 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.405377+0000", 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7509999999999997e-06 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.406541+0000", 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011644439999999999 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.406551+0000", 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0074999999999999e-05 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.313704+0000", 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90715236899999996 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.313804+0000", 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9807000000000002e-05 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.313934+0000", 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013054399999999999 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316652+0000", 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002717503 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316676+0000", 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4698e-05 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.320846+0000", 2023-10-09T04:11:30.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041698500000000001 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.320901+0000", 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4277000000000002e-05 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.320942+0000", 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.176e-05 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.117 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:59138 7.15 7:ab1a0d3e:::benchmark_data_smithi163_346082_object59137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.444828+0000", 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.554536518000006, 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87630884600000003, 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.118 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59138 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.444828+0000", 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.444828+0000", 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.444826+0000", 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.444912+0000", 2023-10-09T04:11:30.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5502999999999998e-05 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.444927+0000", 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5495e-05 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.444934+0000", 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8029999999999999e-06 2023-10-09T04:11:30.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.445002+0000", 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7825999999999997e-05 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.445015+0000", 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.296e-05 2023-10-09T04:11:30.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.314886+0000", 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86987091400000005 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.315007+0000", 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120869 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.315154+0000", 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000146957 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.316659+0000", 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015052049999999999 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.318266+0000", 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001607292 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.321054+0000", 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027875529999999999 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.321101+0000", 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7293999999999998e-05 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.321137+0000", 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5593999999999998e-05 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.126 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:59148 7.15 7:af03a957:::benchmark_data_smithi163_346082_object59147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.451157+0000", 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 75.548207400999999, 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87012110200000004, 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.127 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 59148 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451157+0000", 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451157+0000", 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451156+0000", 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451205+0000", 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9437000000000001e-05 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451217+0000", 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1503e-05 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451221+0000", 2023-10-09T04:11:30.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3209999999999998e-06 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451265+0000", 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4397000000000002e-05 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.451276+0000", 2023-10-09T04:11:30.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0631e-05 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.315818+0000", 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86454220100000001 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.315892+0000", 2023-10-09T04:11:30.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3787000000000004e-05 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.315977+0000", 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4697999999999994e-05 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.317513+0000", 2023-10-09T04:11:30.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015364210000000001 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.318291+0000", 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00077752500000000005 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.321191+0000", 2023-10-09T04:11:30.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002900803 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.321243+0000", 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1745e-05 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.321278+0000", 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4700999999999999e-05 2023-10-09T04:11:30.135 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80483 7.5 7:a6133132:::benchmark_data_smithi163_346082_object80482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.158958+0000", 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 52.840406019, 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2345513130000001, 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:11:30.136 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80483 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.158958+0000", 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.158958+0000", 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.158957+0000", 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159013+0000", 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6461e-05 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159025+0000", 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1671000000000001e-05 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159030+0000", 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7999999999999998e-06 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159069+0000", 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9007000000000001e-05 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159189+0000", 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011971 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159306+0000", 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011739900000000001 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.160632+0000", 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013256570000000001 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.160849+0000", 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000217123 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310638+0000", 2023-10-09T04:11:30.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14978875999999999 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310645+0000", 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7449999999999995e-06 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.393397+0000", 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.082751867 2023-10-09T04:11:30.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.393510+0000", 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112316 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80484 7.15 7:ade26b62:::benchmark_data_smithi163_346082_object80483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:11:30.144 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 52.834113068999997, 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.249257176, 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80484 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165250+0000", 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165302+0000", 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2011000000000001e-05 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165315+0000", 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2316e-05 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165321+0000", 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0920000000000003e-06 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165361+0000", 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9743000000000002e-05 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165448+0000", 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7162000000000003e-05 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:11:30.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165538+0000", 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0328999999999995e-05 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.166751+0000", 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012134100000000001 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.167144+0000", 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000392896 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310638+0000", 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14349324699999999 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310649+0000", 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1039999999999999e-05 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326699+0000", 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.016050111 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326711+0000", 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2063e-05 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414399+0000", 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0876884 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414508+0000", 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109128 2023-10-09T04:11:30.153 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80500 7.15 7:a9cb5e2a:::benchmark_data_smithi163_346082_object80499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.457432+0000", 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 52.541931927, 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9663766489999999, 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.154 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80500 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457432+0000", 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457432+0000", 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457431+0000", 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457464+0000", 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2923999999999999e-05 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457477+0000", 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3018e-05 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457490+0000", 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2914e-05 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457548+0000", 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8184999999999999e-05 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457557+0000", 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9069999999999996e-06 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326812+0000", 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86925475900000004 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326821+0000", 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.611e-06 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414770+0000", 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0879485680000001 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414935+0000", 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016542 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.415125+0000", 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018943599999999999 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420602+0000", 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054771819999999997 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420616+0000", 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3730000000000001e-05 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420656+0000", 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0658999999999998e-05 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423666+0000", 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030100000000000001 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423736+0000", 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9595e-05 2023-10-09T04:11:30.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423809+0000", 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2886000000000007e-05 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80501 7.5 7:a4fd8e51:::benchmark_data_smithi163_346082_object80500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:11:30.165 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.457508+0000", 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 52.541856551000002, 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95126671699999998, 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80501 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457508+0000", 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457508+0000", 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457507+0000", 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457542+0000", 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4523000000000001e-05 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457550+0000", 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9410000000000007e-06 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457555+0000", 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0699999999999997e-06 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457585+0000", 2023-10-09T04:11:30.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0366999999999999e-05 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457594+0000", 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0380000000000005e-06 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.394145+0000", 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93655127800000004 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.394257+0000", 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011124300000000001 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.394368+0000", 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111334 2023-10-09T04:11:30.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.395595+0000", 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001226897 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.397314+0000", 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017193499999999999 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.408104+0000", 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010790338999999999 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.408145+0000", 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1004000000000002e-05 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.408774+0000", 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00062891399999999995 2023-10-09T04:11:30.173 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80559 7.15 7:a8621579:::benchmark_data_smithi163_346082_object80558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:38.479446+0000", 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 51.519918302000001, 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94783312099999995, 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.174 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80559 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479446+0000", 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479446+0000", 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479445+0000", 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479511+0000", 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6091000000000001e-05 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479522+0000", 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0856000000000001e-05 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479526+0000", 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4059999999999998e-06 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479596+0000", 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9876999999999995e-05 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.479606+0000", 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5680000000000005e-06 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.416599+0000", 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93699311699999999 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.416712+0000", 2023-10-09T04:11:30.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113608 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.416849+0000", 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000136988 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420625+0000", 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037759180000000001 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420701+0000", 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5325000000000001e-05 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423861+0000", 2023-10-09T04:11:30.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003160323 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427084+0000", 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032232979999999999 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427154+0000", 2023-10-09T04:11:30.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9404000000000002e-05 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427279+0000", 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125301 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80592 7.15 7:a9ba0b44:::benchmark_data_smithi163_346082_object80591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:38.544399+0000", 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 51.454965489999999, 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.883150659, 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80592 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.184 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544399+0000", 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544399+0000", 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544394+0000", 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999948 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544422+0000", 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8021000000000002e-05 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544436+0000", 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.432e-05 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544459+0000", 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3090000000000001e-05 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544514+0000", 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4656e-05 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.544521+0000", 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4739999999999999e-06 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.418116+0000", 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87359504600000004 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.418264+0000", 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147513 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T04:11:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.418444+0000", 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018021899999999999 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420709+0000", 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00226487 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.424758+0000", 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040489499999999999 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.426616+0000", 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001858157 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427354+0000", 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00073812900000000002 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427422+0000", 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7563000000000003e-05 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427549+0000", 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127761 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80598 7.15 7:a85b2008:::benchmark_data_smithi163_346082_object80597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:38.569032+0000", 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 51.430332391999997, 2023-10-09T04:11:30.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85868839500000005, 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80598 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:11:30.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569032+0000", 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569032+0000", 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569031+0000", 2023-10-09T04:11:30.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569060+0000", 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9648000000000001e-05 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569074+0000", 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3713999999999999e-05 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569079+0000", 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2120000000000002e-06 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569118+0000", 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8414999999999997e-05 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.569126+0000", 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4519999999999997e-06 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.419368+0000", 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.850241892 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:11:30.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.419487+0000", 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118779 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.419596+0000", 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010911100000000001 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:11:30.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420719+0000", 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011230089999999999 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.424792+0000", 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040729490000000002 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.426648+0000", 2023-10-09T04:11:30.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001856629 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427615+0000", 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096652999999999995 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427656+0000", 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1047000000000002e-05 2023-10-09T04:11:30.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.427720+0000", 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4288000000000005e-05 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:11:30.202 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:11:30.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:30.203 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:30.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:30.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:30.354 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:30.389 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:30.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:30.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:30.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:30.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:30.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:30 smithi163 ceph-mon[213841]: pgmap v545: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 2197/216654 objects degraded (1.014%); 8851/216654 objects misplaced (4.085%); 701 KiB/s, 10 objects/s recovering 2023-10-09T04:11:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:30.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:30 smithi163 ceph-mon[221067]: pgmap v545: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 2197/216654 objects degraded (1.014%); 8851/216654 objects misplaced (4.085%); 701 KiB/s, 10 objects/s recovering 2023-10-09T04:11:30.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:30.637 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:30.638 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:30.795 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:30 smithi181 ceph-mon[196979]: pgmap v545: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 2197/216654 objects degraded (1.014%); 8851/216654 objects misplaced (4.085%); 701 KiB/s, 10 objects/s recovering 2023-10-09T04:11:30.795 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:30.795 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:30.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:11:30.881 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:30.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:30.908 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:11:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:11:30] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-10-09T04:11:30.914 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:11:31.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:31.138 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:31.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/816071935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:31.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:31.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[213841]: failed to encode map e213 with expected crc 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/816071935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:31.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:31.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:31.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:31.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:31.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T04:11:31.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:31 smithi163 ceph-mon[221067]: failed to encode map e213 with expected crc 2023-10-09T04:11:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/816071935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T04:11:32.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:31 smithi181 ceph-mon[196979]: failed to encode map e213 with expected crc 2023-10-09T04:11:32.585 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:32.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:32.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:32.797 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[213841]: failed to encode map e213 with expected crc 2023-10-09T04:11:32.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[213841]: failed to encode map e213 with expected crc 2023-10-09T04:11:32.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[213841]: pgmap v547: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2197/211488 objects degraded (1.039%); 8527/211488 objects misplaced (4.032%); 631 KiB/s, 9 objects/s recovering 2023-10-09T04:11:32.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/211488 objects degraded (1.039%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:32.798 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[221067]: failed to encode map e213 with expected crc 2023-10-09T04:11:32.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[221067]: failed to encode map e213 with expected crc 2023-10-09T04:11:32.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[221067]: pgmap v547: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2197/211488 objects degraded (1.039%); 8527/211488 objects misplaced (4.032%); 631 KiB/s, 9 objects/s recovering 2023-10-09T04:11:32.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/211488 objects degraded (1.039%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:32.819 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:33.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:32 smithi181 ceph-mon[196979]: failed to encode map e213 with expected crc 2023-10-09T04:11:33.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:32 smithi181 ceph-mon[196979]: failed to encode map e213 with expected crc 2023-10-09T04:11:33.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:32 smithi181 ceph-mon[196979]: pgmap v547: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2197/211488 objects degraded (1.039%); 8527/211488 objects misplaced (4.032%); 631 KiB/s, 9 objects/s recovering 2023-10-09T04:11:33.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/211488 objects degraded (1.039%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:33.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:33.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:33.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:33.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:33.641 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:33.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:33.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:33.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:33.869 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:33.939 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:34.048 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:34.104 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:34.154 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:34.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:34.204 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:34.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:34.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:34.261 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:34.306 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:34.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:34.339 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:34.379 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.407 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:11:34.425 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:34.603 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:34.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:34.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:34.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[221067]: pgmap v548: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2197/207747 objects degraded (1.058%); 8412/207747 objects misplaced (4.049%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:34.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/528255098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:34.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:34.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:34.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[213841]: pgmap v548: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2197/207747 objects degraded (1.058%); 8412/207747 objects misplaced (4.049%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:34.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/528255098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:34.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:34.707 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:34.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:34.814 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:35.009 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:11:35.010 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:11:35.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:34 smithi181 ceph-mon[196979]: pgmap v548: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2197/207747 objects degraded (1.058%); 8412/207747 objects misplaced (4.049%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:35.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/528255098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:35.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:11:35.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:35.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:11:36.361 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:36.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:36.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:36.499 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:36 smithi163 ceph-mon[221067]: from='client.35277 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:36.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:36 smithi163 ceph-mon[221067]: pgmap v549: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 2197/204021 objects degraded (1.077%); 8412/204021 objects misplaced (4.123%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:36 smithi163 ceph-mon[213841]: from='client.35277 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:36.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:36 smithi163 ceph-mon[213841]: pgmap v549: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 2197/204021 objects degraded (1.077%); 8412/204021 objects misplaced (4.123%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:36.715 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":548,"stamp":"2023-10-09T04:11:33.458301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4537259960,"num_objects":69249,"num_object_clones":0,"num_object_copies":207747,"num_objects_missing_on_primary":342,"num_objects_missing":342,"num_objects_degraded":2197,"num_objects_misplaced":8412,"num_objects_unfound":0,"num_objects_dirty":69249,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":111172,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10456,"num_bytes_recovered":686361268,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139122,"ondisk_log_size":139122,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16170248,"kb_used_data":13846408,"kb_used_omap":126,"kb_used_meta":2323393,"kb_avail":733790968,"statfs":{"total":767960285184,"available":751401951232,"internally_reserved":0,"allocated":14178721792,"data_stored":14172880102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2379155067},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,4,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-468713472,"num_objects":-7152,"num_object_clones":0,"num_object_copies":-21456,"num_objects_missing_on_primary":-110,"num_objects_missing":-110,"num_objects_degraded":0,"num_objects_misplaced":-886,"num_objects_unfound":0,"num_objects_dirty":-7152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7152,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":111,"num_bytes_recovered":7274496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003097"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":209,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649196+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:11:20.649196+0000","last_peered":"2023-10-09T04:11:20.649196+0000","last_clean":"2023-10-09T04:11:20.649196+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:11:20.649196+0000","last_undegraded":"2023-10-09T04:11:20.649196+0000","last_fullsized":"2023-10-09T04:11:20.649196+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":147,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644526+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:11:20.644526+0000","last_peered":"2023-10-09T04:11:20.644526+0000","last_clean":"2023-10-09T04:11:20.644526+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:11:20.644526+0000","last_undegraded":"2023-10-09T04:11:20.644526+0000","last_fullsized":"2023-10-09T04:11:20.644526+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":162,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644023+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:11:20.644023+0000","last_peered":"2023-10-09T04:11:20.644023+0000","last_clean":"2023-10-09T04:11:20.644023+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:20.644023+0000","last_undegraded":"2023-10-09T04:11:20.644023+0000","last_fullsized":"2023-10-09T04:11:20.644023+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.644057+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:11:21.644057+0000","last_peered":"2023-10-09T04:11:21.644057+0000","last_clean":"2023-10-09T04:11:21.644057+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:11:21.644057+0000","last_undegraded":"2023-10-09T04:11:21.644057+0000","last_fullsized":"2023-10-09T04:11:21.644057+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":166,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.639443+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:11:21.639443+0000","last_peered":"2023-10-09T04:11:21.639443+0000","last_clean":"2023-10-09T04:11:21.639443+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:11:21.639443+0000","last_undegraded":"2023-10-09T04:11:21.639443+0000","last_fullsized":"2023-10-09T04:11:21.639443+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684696+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:11:20.684696+0000","last_peered":"2023-10-09T04:11:20.684696+0000","last_clean":"2023-10-09T04:11:20.684696+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:11:20.684696+0000","last_undegraded":"2023-10-09T04:11:20.684696+0000","last_fullsized":"2023-10-09T04:11:20.684696+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"212'4772","reported_seq":7723,"reported_epoch":212,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:30.254348+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:11:30.254348+0000","last_peered":"2023-10-09T04:11:30.254348+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:11:30.254348+0000","last_undegraded":"2023-10-09T04:11:30.254348+0000","last_fullsized":"2023-10-09T04:11:30.254348+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4772,"log_dups_size":0,"ondisk_log_size":4772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2187,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3601,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2187}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":167,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.642607+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:11:21.642607+0000","last_peered":"2023-10-09T04:11:21.642607+0000","last_clean":"2023-10-09T04:11:21.642607+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:11:21.642607+0000","last_undegraded":"2023-10-09T04:11:21.642607+0000","last_fullsized":"2023-10-09T04:11:21.642607+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":190,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.639718+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:11:20.639718+0000","last_peered":"2023-10-09T04:11:20.639718+0000","last_clean":"2023-10-09T04:11:20.639718+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:11:20.639718+0000","last_undegraded":"2023-10-09T04:11:20.639718+0000","last_fullsized":"2023-10-09T04:11:20.639718+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"213'4947","reported_seq":9129,"reported_epoch":213,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:11:32.849561+0000","last_change":"2023-10-09T04:10:51.646576+0000","last_active":"2023-10-09T04:11:32.849561+0000","last_peered":"2023-10-09T04:11:32.849561+0000","last_clean":"2023-10-09T04:09:59.283093+0000","last_became_active":"2023-10-09T04:10:40.892310+0000","last_became_peered":"2023-10-09T04:10:40.892310+0000","last_unstale":"2023-10-09T04:11:32.849561+0000","last_undegraded":"2023-10-09T04:11:32.849561+0000","last_fullsized":"2023-10-09T04:10:40.864699+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4947,"log_dups_size":0,"ondisk_log_size":4947,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":342,"num_objects_missing":342,"num_objects_degraded":0,"num_objects_misplaced":1771,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3782,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1124,"num_bytes_recovered":73662464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"0,1,6,7","objects":1087},{"shards":"0,6","objects":743},{"shards":"1,2,3","objects":342}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684735+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:11:20.684735+0000","last_peered":"2023-10-09T04:11:20.684735+0000","last_clean":"2023-10-09T04:11:20.684735+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:11:20.684735+0000","last_undegraded":"2023-10-09T04:11:20.684735+0000","last_fullsized":"2023-10-09T04:11:20.684735+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":108,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599225+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:11:20.599225+0000","last_peered":"2023-10-09T04:11:20.599225+0000","last_clean":"2023-10-09T04:11:20.599225+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:11:20.599225+0000","last_undegraded":"2023-10-09T04:11:20.599225+0000","last_fullsized":"2023-10-09T04:11:20.599225+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"212'4765","reported_seq":7647,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.516464+0000","last_change":"2023-10-09T04:10:18.287893+0000","last_active":"2023-10-09T04:11:30.516464+0000","last_peered":"2023-10-09T04:11:30.516464+0000","last_clean":"2023-10-09T04:11:30.516464+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:11:30.516464+0000","last_undegraded":"2023-10-09T04:11:30.516464+0000","last_fullsized":"2023-10-09T04:11:30.516464+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902400,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3630,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":174,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599550+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:11:20.599550+0000","last_peered":"2023-10-09T04:11:20.599550+0000","last_clean":"2023-10-09T04:11:20.599550+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:11:20.599550+0000","last_undegraded":"2023-10-09T04:11:20.599550+0000","last_fullsized":"2023-10-09T04:11:20.599550+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644525+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:11:20.644525+0000","last_peered":"2023-10-09T04:11:20.644525+0000","last_clean":"2023-10-09T04:11:20.644525+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:11:20.644525+0000","last_undegraded":"2023-10-09T04:11:20.644525+0000","last_fullsized":"2023-10-09T04:11:20.644525+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"212'4882","reported_seq":7815,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:29.901716+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:11:29.901716+0000","last_peered":"2023-10-09T04:11:29.901716+0000","last_clean":"2023-10-09T04:11:29.901716+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:11:29.901716+0000","last_undegraded":"2023-10-09T04:11:29.901716+0000","last_fullsized":"2023-10-09T04:11:29.901716+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144375808,"num_objects":2203,"num_object_clones":0,"num_object_copies":6609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3693,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":176,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649227+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:11:20.649227+0000","last_peered":"2023-10-09T04:11:20.649227+0000","last_clean":"2023-10-09T04:11:20.649227+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:11:20.649227+0000","last_undegraded":"2023-10-09T04:11:20.649227+0000","last_fullsized":"2023-10-09T04:11:20.649227+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649370+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:11:20.649370+0000","last_peered":"2023-10-09T04:11:20.649370+0000","last_clean":"2023-10-09T04:11:20.649370+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:11:20.649370+0000","last_undegraded":"2023-10-09T04:11:20.649370+0000","last_fullsized":"2023-10-09T04:11:20.649370+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"213'4900","reported_seq":4965,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.832836+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:11:32.832836+0000","last_peered":"2023-10-09T04:11:32.832836+0000","last_clean":"2023-10-09T04:11:32.832836+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:11:32.832836+0000","last_undegraded":"2023-10-09T04:11:32.832836+0000","last_fullsized":"2023-10-09T04:11:32.832836+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4900,"log_dups_size":0,"ondisk_log_size":4900,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137494528,"num_objects":2098,"num_object_clones":0,"num_object_copies":6294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3746,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":173,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599243+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:11:20.599243+0000","last_peered":"2023-10-09T04:11:20.599243+0000","last_clean":"2023-10-09T04:11:20.599243+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:11:20.599243+0000","last_undegraded":"2023-10-09T04:11:20.599243+0000","last_fullsized":"2023-10-09T04:11:20.599243+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":148,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.654199+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:11:21.654199+0000","last_peered":"2023-10-09T04:11:21.654199+0000","last_clean":"2023-10-09T04:11:21.654199+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:11:21.654199+0000","last_undegraded":"2023-10-09T04:11:21.654199+0000","last_fullsized":"2023-10-09T04:11:21.654199+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"213'4753","reported_seq":4817,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.799379+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:11:32.799379+0000","last_peered":"2023-10-09T04:11:32.799379+0000","last_clean":"2023-10-09T04:11:32.799379+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:11:32.799379+0000","last_undegraded":"2023-10-09T04:11:32.799379+0000","last_fullsized":"2023-10-09T04:11:32.799379+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131694592,"num_objects":2010,"num_object_clones":0,"num_object_copies":6030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3636,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:22.829146+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:11:22.829146+0000","last_peered":"2023-10-09T04:11:22.829146+0000","last_clean":"2023-10-09T04:11:22.829146+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:11:22.829146+0000","last_undegraded":"2023-10-09T04:11:22.829146+0000","last_fullsized":"2023-10-09T04:11:22.829146+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684649+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:11:20.684649+0000","last_peered":"2023-10-09T04:11:20.684649+0000","last_clean":"2023-10-09T04:11:20.684649+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:11:20.684649+0000","last_undegraded":"2023-10-09T04:11:20.684649+0000","last_fullsized":"2023-10-09T04:11:20.684649+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"212'4778","reported_seq":7719,"reported_epoch":212,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:30.571696+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:11:30.571696+0000","last_peered":"2023-10-09T04:11:30.571696+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:11:30.571696+0000","last_undegraded":"2023-10-09T04:11:30.571696+0000","last_fullsized":"2023-10-09T04:11:30.571696+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146833408,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2241,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3663,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2241}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599166+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:11:20.599166+0000","last_peered":"2023-10-09T04:11:20.599166+0000","last_clean":"2023-10-09T04:11:20.599166+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:11:20.599166+0000","last_undegraded":"2023-10-09T04:11:20.599166+0000","last_fullsized":"2023-10-09T04:11:20.599166+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":167,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.644145+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:11:21.644145+0000","last_peered":"2023-10-09T04:11:21.644145+0000","last_clean":"2023-10-09T04:11:21.644145+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:11:21.644145+0000","last_undegraded":"2023-10-09T04:11:21.644145+0000","last_fullsized":"2023-10-09T04:11:21.644145+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"213'4906","reported_seq":4971,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.864525+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:11:32.864525+0000","last_peered":"2023-10-09T04:11:32.864525+0000","last_clean":"2023-10-09T04:11:32.864525+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:11:32.864525+0000","last_undegraded":"2023-10-09T04:11:32.864525+0000","last_fullsized":"2023-10-09T04:11:32.864525+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4906,"log_dups_size":0,"ondisk_log_size":4906,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141819904,"num_objects":2164,"num_object_clones":0,"num_object_copies":6492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3744,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":103,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.639304+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:11:21.639304+0000","last_peered":"2023-10-09T04:11:21.639304+0000","last_clean":"2023-10-09T04:11:21.639304+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:11:21.639304+0000","last_undegraded":"2023-10-09T04:11:21.639304+0000","last_fullsized":"2023-10-09T04:11:21.639304+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.640210+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:11:20.640210+0000","last_peered":"2023-10-09T04:11:20.640210+0000","last_clean":"2023-10-09T04:11:20.640210+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:11:20.640210+0000","last_undegraded":"2023-10-09T04:11:20.640210+0000","last_fullsized":"2023-10-09T04:11:20.640210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"212'7202","reported_seq":7256,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:28.614133+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:11:28.614133+0000","last_peered":"2023-10-09T04:11:28.614133+0000","last_clean":"2023-10-09T04:11:28.614133+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:11:28.614133+0000","last_undegraded":"2023-10-09T04:11:28.614133+0000","last_fullsized":"2023-10-09T04:11:28.614133+0000","mapping_epoch":167,"log_start":"171'300","ondisk_log_start":"171'300","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293208064,"num_objects":4474,"num_object_clones":0,"num_object_copies":13422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7202,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":172,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.760765+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:11:20.760765+0000","last_peered":"2023-10-09T04:11:20.760765+0000","last_clean":"2023-10-09T04:11:20.760765+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:11:20.760765+0000","last_undegraded":"2023-10-09T04:11:20.760765+0000","last_fullsized":"2023-10-09T04:11:20.760765+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":153,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684420+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:11:20.684420+0000","last_peered":"2023-10-09T04:11:20.684420+0000","last_clean":"2023-10-09T04:11:20.684420+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:11:20.684420+0000","last_undegraded":"2023-10-09T04:11:20.684420+0000","last_fullsized":"2023-10-09T04:11:20.684420+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"212'7260","reported_seq":7315,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:29.945302+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:11:29.945302+0000","last_peered":"2023-10-09T04:11:29.945302+0000","last_clean":"2023-10-09T04:11:29.945302+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:11:29.945302+0000","last_undegraded":"2023-10-09T04:11:29.945302+0000","last_fullsized":"2023-10-09T04:11:29.945302+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7260,"log_dups_size":0,"ondisk_log_size":7260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281935872,"num_objects":4302,"num_object_clones":0,"num_object_copies":12906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7260,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":173,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599456+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:11:20.599456+0000","last_peered":"2023-10-09T04:11:20.599456+0000","last_clean":"2023-10-09T04:11:20.599456+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:11:20.599456+0000","last_undegraded":"2023-10-09T04:11:20.599456+0000","last_fullsized":"2023-10-09T04:11:20.599456+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649720+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:11:20.649720+0000","last_peered":"2023-10-09T04:11:20.649720+0000","last_clean":"2023-10-09T04:11:20.649720+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:11:20.649720+0000","last_undegraded":"2023-10-09T04:11:20.649720+0000","last_fullsized":"2023-10-09T04:11:20.649720+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"213'4916","reported_seq":5088,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.811964+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:11:32.811964+0000","last_peered":"2023-10-09T04:11:32.811964+0000","last_clean":"2023-10-09T04:11:32.811964+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:11:32.811964+0000","last_undegraded":"2023-10-09T04:11:32.811964+0000","last_fullsized":"2023-10-09T04:11:32.811964+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":4916,"log_dups_size":0,"ondisk_log_size":4916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140378112,"num_objects":2142,"num_object_clones":0,"num_object_copies":6426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2530,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":155,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599446+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:11:20.599446+0000","last_peered":"2023-10-09T04:11:20.599446+0000","last_clean":"2023-10-09T04:11:20.599446+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:11:20.599446+0000","last_undegraded":"2023-10-09T04:11:20.599446+0000","last_fullsized":"2023-10-09T04:11:20.599446+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.643990+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:11:20.643990+0000","last_peered":"2023-10-09T04:11:20.643990+0000","last_clean":"2023-10-09T04:11:20.643990+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:11:20.643990+0000","last_undegraded":"2023-10-09T04:11:20.643990+0000","last_fullsized":"2023-10-09T04:11:20.643990+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"212'4705","reported_seq":4774,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:29.901482+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:11:29.901482+0000","last_peered":"2023-10-09T04:11:29.901482+0000","last_clean":"2023-10-09T04:11:29.901482+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:11:29.901482+0000","last_undegraded":"2023-10-09T04:11:29.901482+0000","last_fullsized":"2023-10-09T04:11:29.901482+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142016512,"num_objects":2167,"num_object_clones":0,"num_object_copies":6501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3565,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684594+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:11:20.684594+0000","last_peered":"2023-10-09T04:11:20.684594+0000","last_clean":"2023-10-09T04:11:20.684594+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:11:20.684594+0000","last_undegraded":"2023-10-09T04:11:20.684594+0000","last_fullsized":"2023-10-09T04:11:20.684594+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.641946+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:11:21.641946+0000","last_peered":"2023-10-09T04:11:21.641946+0000","last_clean":"2023-10-09T04:11:21.641946+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:11:21.641946+0000","last_undegraded":"2023-10-09T04:11:21.641946+0000","last_fullsized":"2023-10-09T04:11:21.641946+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"213'7473","reported_seq":7528,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.832932+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:11:32.832932+0000","last_peered":"2023-10-09T04:11:32.832932+0000","last_clean":"2023-10-09T04:11:32.832932+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:11:32.832932+0000","last_undegraded":"2023-10-09T04:11:32.832932+0000","last_fullsized":"2023-10-09T04:11:32.832932+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7473,"log_dups_size":0,"ondisk_log_size":7473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274530304,"num_objects":4189,"num_object_clones":0,"num_object_copies":12567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7473,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":117,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.642120+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:11:21.642120+0000","last_peered":"2023-10-09T04:11:21.642120+0000","last_clean":"2023-10-09T04:11:21.642120+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:11:21.642120+0000","last_undegraded":"2023-10-09T04:11:21.642120+0000","last_fullsized":"2023-10-09T04:11:21.642120+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649180+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:11:20.649180+0000","last_peered":"2023-10-09T04:11:20.649180+0000","last_clean":"2023-10-09T04:11:20.649180+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:11:20.649180+0000","last_undegraded":"2023-10-09T04:11:20.649180+0000","last_fullsized":"2023-10-09T04:11:20.649180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"212'7246","reported_seq":7317,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:11:30.546976+0000","last_change":"2023-10-09T04:10:38.444053+0000","last_active":"2023-10-09T04:11:30.546976+0000","last_peered":"2023-10-09T04:11:30.546976+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:11:30.546976+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7246,"log_dups_size":0,"ondisk_log_size":7246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286523392,"num_objects":4372,"num_object_clones":0,"num_object_copies":13116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7246,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":2197},{"shards":"0,5","objects":2175}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649285+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:11:20.649285+0000","last_peered":"2023-10-09T04:11:20.649285+0000","last_clean":"2023-10-09T04:11:20.649285+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:11:20.649285+0000","last_undegraded":"2023-10-09T04:11:20.649285+0000","last_fullsized":"2023-10-09T04:11:20.649285+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649434+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:11:20.649434+0000","last_peered":"2023-10-09T04:11:20.649434+0000","last_clean":"2023-10-09T04:11:20.649434+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:11:20.649434+0000","last_undegraded":"2023-10-09T04:11:20.649434+0000","last_fullsized":"2023-10-09T04:11:20.649434+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"212'7236","reported_seq":7291,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:28.645736+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:11:28.645736+0000","last_peered":"2023-10-09T04:11:28.645736+0000","last_clean":"2023-10-09T04:11:28.645736+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:11:28.645736+0000","last_undegraded":"2023-10-09T04:11:28.645736+0000","last_fullsized":"2023-10-09T04:11:28.645736+0000","mapping_epoch":167,"log_start":"171'400","ondisk_log_start":"171'400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287244312,"num_objects":4384,"num_object_clones":0,"num_object_copies":13152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4384,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7236,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649425+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:11:20.649425+0000","last_peered":"2023-10-09T04:11:20.649425+0000","last_clean":"2023-10-09T04:11:20.649425+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:11:20.649425+0000","last_undegraded":"2023-10-09T04:11:20.649425+0000","last_fullsized":"2023-10-09T04:11:20.649425+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649225+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:11:20.649225+0000","last_peered":"2023-10-09T04:11:20.649225+0000","last_clean":"2023-10-09T04:11:20.649225+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:11:20.649225+0000","last_undegraded":"2023-10-09T04:11:20.649225+0000","last_fullsized":"2023-10-09T04:11:20.649225+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"212'7099","reported_seq":7153,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.704953+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:11:30.704953+0000","last_peered":"2023-10-09T04:11:30.704953+0000","last_clean":"2023-10-09T04:11:30.704953+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:11:30.704953+0000","last_undegraded":"2023-10-09T04:11:30.704953+0000","last_fullsized":"2023-10-09T04:11:30.704953+0000","mapping_epoch":167,"log_start":"171'700","ondisk_log_start":"171'700","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6399,"log_dups_size":0,"ondisk_log_size":6399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283836416,"num_objects":4331,"num_object_clones":0,"num_object_copies":12993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7099,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":177,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.639549+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:11:20.639549+0000","last_peered":"2023-10-09T04:11:20.639549+0000","last_clean":"2023-10-09T04:11:20.639549+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:11:20.639549+0000","last_undegraded":"2023-10-09T04:11:20.639549+0000","last_fullsized":"2023-10-09T04:11:20.639549+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644619+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:11:20.644619+0000","last_peered":"2023-10-09T04:11:20.644619+0000","last_clean":"2023-10-09T04:11:20.644619+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:11:20.644619+0000","last_undegraded":"2023-10-09T04:11:20.644619+0000","last_fullsized":"2023-10-09T04:11:20.644619+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"212'4777","reported_seq":4964,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.146181+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:11:30.146181+0000","last_peered":"2023-10-09T04:11:30.146181+0000","last_clean":"2023-10-09T04:11:30.146181+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:11:30.146181+0000","last_undegraded":"2023-10-09T04:11:30.146181+0000","last_fullsized":"2023-10-09T04:11:30.146181+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":4777,"log_dups_size":0,"ondisk_log_size":4777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145555456,"num_objects":2221,"num_object_clones":0,"num_object_copies":6663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1124,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644585+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:11:20.644585+0000","last_peered":"2023-10-09T04:11:20.644585+0000","last_clean":"2023-10-09T04:11:20.644585+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:11:20.644585+0000","last_undegraded":"2023-10-09T04:11:20.644585+0000","last_fullsized":"2023-10-09T04:11:20.644585+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":188,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684670+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:11:20.684670+0000","last_peered":"2023-10-09T04:11:20.684670+0000","last_clean":"2023-10-09T04:11:20.684670+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:20.684670+0000","last_undegraded":"2023-10-09T04:11:20.684670+0000","last_fullsized":"2023-10-09T04:11:20.684670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":149,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644122+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:11:20.644122+0000","last_peered":"2023-10-09T04:11:20.644122+0000","last_clean":"2023-10-09T04:11:20.644122+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:11:20.644122+0000","last_undegraded":"2023-10-09T04:11:20.644122+0000","last_fullsized":"2023-10-09T04:11:20.644122+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"213'4882","reported_seq":4948,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.878943+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:11:32.878943+0000","last_peered":"2023-10-09T04:11:32.878943+0000","last_clean":"2023-10-09T04:11:32.878943+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:11:32.878943+0000","last_undegraded":"2023-10-09T04:11:32.878943+0000","last_fullsized":"2023-10-09T04:11:32.878943+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138805248,"num_objects":2118,"num_object_clones":0,"num_object_copies":6354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3728,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":175,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684559+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:11:20.684559+0000","last_peered":"2023-10-09T04:11:20.684559+0000","last_clean":"2023-10-09T04:11:20.684559+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:11:20.684559+0000","last_undegraded":"2023-10-09T04:11:20.684559+0000","last_fullsized":"2023-10-09T04:11:20.684559+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649434+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:11:20.649434+0000","last_peered":"2023-10-09T04:11:20.649434+0000","last_clean":"2023-10-09T04:11:20.649434+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:11:20.649434+0000","last_undegraded":"2023-10-09T04:11:20.649434+0000","last_fullsized":"2023-10-09T04:11:20.649434+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":197,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644390+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:11:20.644390+0000","last_peered":"2023-10-09T04:11:20.644390+0000","last_clean":"2023-10-09T04:11:20.644390+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:11:20.644390+0000","last_undegraded":"2023-10-09T04:11:20.644390+0000","last_fullsized":"2023-10-09T04:11:20.644390+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"213'4954","reported_seq":5020,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.842630+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:11:32.842630+0000","last_peered":"2023-10-09T04:11:32.842630+0000","last_clean":"2023-10-09T04:11:32.842630+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:11:32.842630+0000","last_undegraded":"2023-10-09T04:11:32.842630+0000","last_fullsized":"2023-10-09T04:11:32.842630+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4954,"log_dups_size":0,"ondisk_log_size":4954,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3792,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.641910+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:11:20.641910+0000","last_peered":"2023-10-09T04:11:20.641910+0000","last_clean":"2023-10-09T04:11:20.641910+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:11:20.641910+0000","last_undegraded":"2023-10-09T04:11:20.641910+0000","last_fullsized":"2023-10-09T04:11:20.641910+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":181,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.643764+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:11:20.643764+0000","last_peered":"2023-10-09T04:11:20.643764+0000","last_clean":"2023-10-09T04:11:20.643764+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:11:20.643764+0000","last_undegraded":"2023-10-09T04:11:20.643764+0000","last_fullsized":"2023-10-09T04:11:20.643764+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":182,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644693+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:11:20.644693+0000","last_peered":"2023-10-09T04:11:20.644693+0000","last_clean":"2023-10-09T04:11:20.644693+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:11:20.644693+0000","last_undegraded":"2023-10-09T04:11:20.644693+0000","last_fullsized":"2023-10-09T04:11:20.644693+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"213'4733","reported_seq":4798,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.878692+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:11:32.878692+0000","last_peered":"2023-10-09T04:11:32.878692+0000","last_clean":"2023-10-09T04:11:32.878692+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:11:32.878692+0000","last_undegraded":"2023-10-09T04:11:32.878692+0000","last_fullsized":"2023-10-09T04:11:32.878692+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4733,"log_dups_size":0,"ondisk_log_size":4733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138641408,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3615,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":195,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684537+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:11:20.684537+0000","last_peered":"2023-10-09T04:11:20.684537+0000","last_clean":"2023-10-09T04:11:20.684537+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:11:20.684537+0000","last_undegraded":"2023-10-09T04:11:20.684537+0000","last_fullsized":"2023-10-09T04:11:20.684537+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":552,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599239+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:11:20.599239+0000","last_peered":"2023-10-09T04:11:20.599239+0000","last_clean":"2023-10-09T04:11:20.599239+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:11:20.599239+0000","last_undegraded":"2023-10-09T04:11:20.599239+0000","last_fullsized":"2023-10-09T04:11:20.599239+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":108,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684606+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:11:20.684606+0000","last_peered":"2023-10-09T04:11:20.684606+0000","last_clean":"2023-10-09T04:11:20.684606+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:11:20.684606+0000","last_undegraded":"2023-10-09T04:11:20.684606+0000","last_fullsized":"2023-10-09T04:11:20.684606+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649620+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:11:20.649620+0000","last_peered":"2023-10-09T04:11:20.649620+0000","last_clean":"2023-10-09T04:11:20.649620+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:11:20.649620+0000","last_undegraded":"2023-10-09T04:11:20.649620+0000","last_fullsized":"2023-10-09T04:11:20.649620+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"212'4708","reported_seq":7054,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.355781+0000","last_change":"2023-10-09T04:10:53.276397+0000","last_active":"2023-10-09T04:11:30.355781+0000","last_peered":"2023-10-09T04:11:30.355781+0000","last_clean":"2023-10-09T04:11:30.355781+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:11:30.355781+0000","last_undegraded":"2023-10-09T04:11:30.355781+0000","last_fullsized":"2023-10-09T04:11:30.355781+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4708,"log_dups_size":0,"ondisk_log_size":4708,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143032320,"num_objects":2182,"num_object_clones":0,"num_object_copies":6546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3592,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":192,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599146+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:11:20.599146+0000","last_peered":"2023-10-09T04:11:20.599146+0000","last_clean":"2023-10-09T04:11:20.599146+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:11:20.599146+0000","last_undegraded":"2023-10-09T04:11:20.599146+0000","last_fullsized":"2023-10-09T04:11:20.599146+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.640448+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:11:21.640448+0000","last_peered":"2023-10-09T04:11:21.640448+0000","last_clean":"2023-10-09T04:11:21.640448+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:11:21.640448+0000","last_undegraded":"2023-10-09T04:11:21.640448+0000","last_fullsized":"2023-10-09T04:11:21.640448+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":201,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.643820+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:11:21.643820+0000","last_peered":"2023-10-09T04:11:21.643820+0000","last_clean":"2023-10-09T04:11:21.643820+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:11:21.643820+0000","last_undegraded":"2023-10-09T04:11:21.643820+0000","last_fullsized":"2023-10-09T04:11:21.643820+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"212'4942","reported_seq":5069,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.552486+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:11:30.552486+0000","last_peered":"2023-10-09T04:11:30.552486+0000","last_clean":"2023-10-09T04:11:30.552486+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:11:30.552486+0000","last_undegraded":"2023-10-09T04:11:30.552486+0000","last_fullsized":"2023-10-09T04:11:30.552486+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":4942,"log_dups_size":0,"ondisk_log_size":4942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145686528,"num_objects":2223,"num_object_clones":0,"num_object_copies":6669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2395,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":175,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599516+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:11:20.599516+0000","last_peered":"2023-10-09T04:11:20.599516+0000","last_clean":"2023-10-09T04:11:20.599516+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:11:20.599516+0000","last_undegraded":"2023-10-09T04:11:20.599516+0000","last_fullsized":"2023-10-09T04:11:20.599516+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":181,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644668+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:11:20.644668+0000","last_peered":"2023-10-09T04:11:20.644668+0000","last_clean":"2023-10-09T04:11:20.644668+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:11:20.644668+0000","last_undegraded":"2023-10-09T04:11:20.644668+0000","last_fullsized":"2023-10-09T04:11:20.644668+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.639199+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:11:21.639199+0000","last_peered":"2023-10-09T04:11:21.639199+0000","last_clean":"2023-10-09T04:11:21.639199+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:11:21.639199+0000","last_undegraded":"2023-10-09T04:11:21.639199+0000","last_fullsized":"2023-10-09T04:11:21.639199+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"212'4934","reported_seq":7992,"reported_epoch":212,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:30.544428+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:11:30.544428+0000","last_peered":"2023-10-09T04:11:30.544428+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:11:30.544428+0000","last_undegraded":"2023-10-09T04:11:30.544428+0000","last_fullsized":"2023-10-09T04:11:30.544428+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2213,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3763,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":2213}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":116,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599558+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:11:20.599558+0000","last_peered":"2023-10-09T04:11:20.599558+0000","last_clean":"2023-10-09T04:11:20.599558+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:11:20.599558+0000","last_undegraded":"2023-10-09T04:11:20.599558+0000","last_fullsized":"2023-10-09T04:11:20.599558+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":148,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.644948+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:11:20.644948+0000","last_peered":"2023-10-09T04:11:20.644948+0000","last_clean":"2023-10-09T04:11:20.644948+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:11:20.644948+0000","last_undegraded":"2023-10-09T04:11:20.644948+0000","last_fullsized":"2023-10-09T04:11:20.644948+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599401+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:11:20.599401+0000","last_peered":"2023-10-09T04:11:20.599401+0000","last_clean":"2023-10-09T04:11:20.599401+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:11:20.599401+0000","last_undegraded":"2023-10-09T04:11:20.599401+0000","last_fullsized":"2023-10-09T04:11:20.599401+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"212'4796","reported_seq":7004,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:28.579754+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:11:28.579754+0000","last_peered":"2023-10-09T04:11:28.579754+0000","last_clean":"2023-10-09T04:11:28.579754+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:11:28.579754+0000","last_undegraded":"2023-10-09T04:11:28.579754+0000","last_fullsized":"2023-10-09T04:11:28.579754+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":4796,"log_dups_size":0,"ondisk_log_size":4796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143589376,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2319,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":140,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.599608+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:11:20.599608+0000","last_peered":"2023-10-09T04:11:20.599608+0000","last_clean":"2023-10-09T04:11:20.599608+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:11:20.599608+0000","last_undegraded":"2023-10-09T04:11:20.599608+0000","last_fullsized":"2023-10-09T04:11:20.599608+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":172,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.643898+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:11:21.643898+0000","last_peered":"2023-10-09T04:11:21.643898+0000","last_clean":"2023-10-09T04:11:21.643898+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:11:21.643898+0000","last_undegraded":"2023-10-09T04:11:21.643898+0000","last_fullsized":"2023-10-09T04:11:21.643898+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649510+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:11:20.649510+0000","last_peered":"2023-10-09T04:11:20.649510+0000","last_clean":"2023-10-09T04:11:20.649510+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:11:20.649510+0000","last_undegraded":"2023-10-09T04:11:20.649510+0000","last_fullsized":"2023-10-09T04:11:20.649510+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":156,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.642035+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:11:21.642035+0000","last_peered":"2023-10-09T04:11:21.642035+0000","last_clean":"2023-10-09T04:11:21.642035+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:11:21.642035+0000","last_undegraded":"2023-10-09T04:11:21.642035+0000","last_fullsized":"2023-10-09T04:11:21.642035+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.642022+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:11:20.642022+0000","last_peered":"2023-10-09T04:11:20.642022+0000","last_clean":"2023-10-09T04:11:20.642022+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:11:20.642022+0000","last_undegraded":"2023-10-09T04:11:20.642022+0000","last_fullsized":"2023-10-09T04:11:20.642022+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"212'4815","reported_seq":4881,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:29.855696+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:11:29.855696+0000","last_peered":"2023-10-09T04:11:29.855696+0000","last_clean":"2023-10-09T04:11:29.855696+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:11:29.855696+0000","last_undegraded":"2023-10-09T04:11:29.855696+0000","last_fullsized":"2023-10-09T04:11:29.855696+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4815,"log_dups_size":0,"ondisk_log_size":4815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140443648,"num_objects":2143,"num_object_clones":0,"num_object_copies":6429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3675,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":173,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684682+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:11:20.684682+0000","last_peered":"2023-10-09T04:11:20.684682+0000","last_clean":"2023-10-09T04:11:20.684682+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:11:20.684682+0000","last_undegraded":"2023-10-09T04:11:20.684682+0000","last_fullsized":"2023-10-09T04:11:20.684682+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649466+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:11:20.649466+0000","last_peered":"2023-10-09T04:11:20.649466+0000","last_clean":"2023-10-09T04:11:20.649466+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:11:20.649466+0000","last_undegraded":"2023-10-09T04:11:20.649466+0000","last_fullsized":"2023-10-09T04:11:20.649466+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"213'4925","reported_seq":4987,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:32.845529+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:11:32.845529+0000","last_peered":"2023-10-09T04:11:32.845529+0000","last_clean":"2023-10-09T04:11:32.845529+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:11:32.845529+0000","last_undegraded":"2023-10-09T04:11:32.845529+0000","last_fullsized":"2023-10-09T04:11:32.845529+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":4925,"log_dups_size":0,"ondisk_log_size":4925,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140312576,"num_objects":2141,"num_object_clones":0,"num_object_copies":6423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3763,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":156,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.684736+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:11:20.684736+0000","last_peered":"2023-10-09T04:11:20.684736+0000","last_clean":"2023-10-09T04:11:20.684736+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:11:20.684736+0000","last_undegraded":"2023-10-09T04:11:20.684736+0000","last_fullsized":"2023-10-09T04:11:20.684736+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.649304+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:11:20.649304+0000","last_peered":"2023-10-09T04:11:20.649304+0000","last_clean":"2023-10-09T04:11:20.649304+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:11:20.649304+0000","last_undegraded":"2023-10-09T04:11:20.649304+0000","last_fullsized":"2023-10-09T04:11:20.649304+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.642381+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:11:20.642381+0000","last_peered":"2023-10-09T04:11:20.642381+0000","last_clean":"2023-10-09T04:11:20.642381+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:11:20.642381+0000","last_undegraded":"2023-10-09T04:11:20.642381+0000","last_fullsized":"2023-10-09T04:11:20.642381+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.641980+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:11:20.641980+0000","last_peered":"2023-10-09T04:11:20.641980+0000","last_clean":"2023-10-09T04:11:20.641980+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:11:20.641980+0000","last_undegraded":"2023-10-09T04:11:20.641980+0000","last_fullsized":"2023-10-09T04:11:20.641980+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":202,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:11:21.642681+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:11:21.642681+0000","last_peered":"2023-10-09T04:11:21.642681+0000","last_clean":"2023-10-09T04:11:21.642681+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:11:21.642681+0000","last_undegraded":"2023-10-09T04:11:21.642681+0000","last_fullsized":"2023-10-09T04:11:21.642681+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-10-09T04:11:20.648997+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:11:20.648997+0000","last_peered":"2023-10-09T04:11:20.648997+0000","last_clean":"2023-10-09T04:11:20.648997+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:11:20.648997+0000","last_undegraded":"2023-10-09T04:11:20.648997+0000","last_fullsized":"2023-10-09T04:11:20.648997+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4536664088,"num_objects":69225,"num_object_clones":0,"num_object_copies":207675,"num_objects_missing_on_primary":342,"num_objects_missing":342,"num_objects_degraded":2197,"num_objects_misplaced":8412,"num_objects_unfound":0,"num_objects_dirty":69225,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110872,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10446,"num_bytes_recovered":684589056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14093922304,"data_stored":14093910088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138906,"ondisk_log_size":138906,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201508,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1944584,"kb_used_data":1643200,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91800568,"statfs":{"total":95995035648,"available":94003781632,"internally_reserved":0,"allocated":1682636800,"data_stored":1682002396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":10.025,"5min":3.5390000000000001,"15min":1.583},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":126.752,"5min":126.752,"15min":126.752},"last":2.089},{"interface":"front","average":{"1min":10.521000000000001,"5min":3.609,"15min":1.6060000000000001},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":128.88399999999999,"5min":128.88399999999999,"15min":128.88399999999999},"last":7.8449999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:10:47 2023","interfaces":[{"interface":"back","average":{"1min":11.227,"5min":3.9630000000000001,"15min":1.736},"min":{"1min":0.28499999999999998,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":245.28700000000001,"5min":245.28700000000001,"15min":245.28700000000001},"last":8.2609999999999992},{"interface":"front","average":{"1min":11.473000000000001,"5min":3.8130000000000002,"15min":1.7470000000000001},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":244.626,"5min":244.626,"15min":244.626},"last":0.255}]},{"osd":2,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.8799999999999999,"15min":1.268},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4009999999999998,"5min":13.887,"15min":58.615000000000002},"last":1.02},{"interface":"front","average":{"1min":1.401,"5min":2.9279999999999999,"15min":1.6850000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.3109999999999999,"5min":99.372,"15min":99.372},"last":0.98399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:10:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0139999999999998,"15min":1.534},"min":{"1min":0.34399999999999997,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":6.1150000000000002,"5min":99.134,"15min":99.134},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.329,"5min":3.1459999999999999,"15min":1.611},"min":{"1min":0.39100000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.1310000000000002,"5min":99.117999999999995,"15min":99.117999999999995},"last":0.59299999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.079,"15min":0.69699999999999995},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9790000000000001,"5min":16.195,"15min":16.195},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.442,"5min":1.0109999999999999,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.052,"5min":12.442,"15min":12.442},"last":0.40400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.77800000000000002,"15min":0.59699999999999998},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81999999999999995,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.873,"15min":0.67800000000000005},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0550000000000002,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.38100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.91300000000000003,"15min":0.67600000000000005},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.99399999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.03,"15min":0.745},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":2.0819999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.437}]}]},{"osd":6,"up_from":47,"seq":201863463145,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2100412,"kb_used_data":1799028,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91644740,"statfs":{"total":95995035648,"available":93844213760,"internally_reserved":0,"allocated":1842204672,"data_stored":1841582111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7850000000000001,"5min":4.2990000000000004,"15min":1.8460000000000001},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":60.295000000000002,"5min":78.983999999999995,"15min":78.983999999999995},"last":1.788},{"interface":"front","average":{"1min":6.8220000000000001,"5min":4.8659999999999997,"15min":2.0830000000000002},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":60.716000000000001,"5min":78.998000000000005,"15min":78.998000000000005},"last":3.286}]},{"osd":1,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.71,"15min":1.619},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":62.698,"5min":62.698,"15min":62.698},"last":3.2549999999999999},{"interface":"front","average":{"1min":3.8420000000000001,"5min":3.895,"15min":1.7010000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.530999999999999,"5min":63.530999999999999,"15min":63.530999999999999},"last":6.4690000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":7.6840000000000002,"5min":3.8420000000000001,"15min":1.8420000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":77.953000000000003,"5min":77.953000000000003,"15min":77.953000000000003},"last":2.899},{"interface":"front","average":{"1min":7.7320000000000002,"5min":3.875,"15min":1.8879999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":78.019000000000005,"5min":78.019000000000005,"15min":78.019000000000005},"last":2.927}]},{"osd":3,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0720000000000001,"5min":4.0350000000000001,"15min":1.911},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":53.497999999999998,"5min":57.509999999999998,"15min":57.509999999999998},"last":4.0899999999999999},{"interface":"front","average":{"1min":3.4910000000000001,"5min":4.3529999999999998,"15min":2.0219999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":53.460999999999999,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.84999999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.51700000000000002,"15min":0.48799999999999999},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.264,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.877},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.84699999999999998,"15min":0.622},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.363,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.57699999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.224,"15min":0.75900000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.2849999999999999,"5min":54.773000000000003,"15min":54.773000000000003},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.3899999999999999,"15min":0.85199999999999998},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4159999999999999,"5min":54.853999999999999,"15min":54.853999999999999},"last":0.61499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.69999999999999996,"15min":0.61499999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":2.903,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.80900000000000005,"15min":0.68500000000000005},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.28699999999999998},"max":{"1min":2.835,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.80200000000000005}]}]},{"osd":1,"up_from":89,"seq":382252089453,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2080684,"kb_used_data":1886628,"kb_used_omap":9,"kb_used_meta":193974,"kb_avail":91664468,"statfs":{"total":95995035648,"available":93864415232,"internally_reserved":0,"allocated":1931907072,"data_stored":1931064261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":198630377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":3.3450000000000002,"15min":1.4390000000000001},"min":{"1min":0.20899999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.833,"5min":85.977999999999994,"15min":85.977999999999994},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.722,"5min":4.6120000000000001,"15min":1.8819999999999999},"min":{"1min":0.22500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.664,"5min":99.831999999999994,"15min":99.831999999999994},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":3.6469999999999998,"15min":1.5369999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.6,"5min":121.54600000000001,"15min":121.54600000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.3010000000000002,"5min":3.7919999999999998,"15min":1.702},"min":{"1min":0.23200000000000001,"5min":0.186,"15min":0.186},"max":{"1min":11.052,"5min":99.899000000000001,"15min":99.899000000000001},"last":0.5}]},{"osd":3,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":4.8959999999999999,"15min":1.98},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":138.87299999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.76400000000000001},{"interface":"front","average":{"1min":10.010999999999999,"5min":6.2439999999999998,"15min":2.4169999999999998},"min":{"1min":0.30599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":138.49799999999999,"5min":138.49799999999999,"15min":138.49799999999999},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.5099999999999998,"15min":1.343},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.5219999999999998,"5min":99.989999999999995,"15min":99.989999999999995},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.3220000000000001,"5min":2.5470000000000002,"15min":1.351},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3330000000000002,"5min":100.249,"15min":100.249},"last":0.88700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.4350000000000001,"15min":1.3360000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4820000000000002,"5min":99.933999999999997,"15min":99.933999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.798,"15min":1.45},"min":{"1min":0.38300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.2789999999999999,"5min":100.34099999999999,"15min":100.34099999999999},"last":0.78300000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":3.1110000000000002,"15min":1.5549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":100.309,"15min":100.309},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.7450000000000001,"15min":1.4550000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4020000000000001,"5min":100.143,"15min":100.143},"last":0.71699999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":2.988,"15min":1.534},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.2210000000000001,"5min":100.214,"15min":100.214},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.7450000000000001,"15min":1.4750000000000001},"min":{"1min":0.374,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":3.335,"5min":100.113,"15min":100.113},"last":0.74099999999999999}]}]},{"osd":0,"up_from":84,"seq":360777252980,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527760,"kb_used_data":2301960,"kb_used_omap":28,"kb_used_meta":225763,"kb_avail":91217392,"statfs":{"total":95995035648,"available":93406609408,"internally_reserved":0,"allocated":2357207040,"data_stored":2356335446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":231182160},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":4.5599999999999996,"15min":1.819},"min":{"1min":0.248,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":7.5949999999999998,"5min":118.48,"15min":118.48},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.5760000000000001,"5min":5.0810000000000004,"15min":1.9930000000000001},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":7.2599999999999998,"5min":118.384,"15min":118.384},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:11:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":4.5990000000000002,"15min":1.9419999999999999},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.8819999999999997,"5min":118.54000000000001,"15min":118.54000000000001},"last":0.38},{"interface":"front","average":{"1min":1.736,"5min":5.0780000000000003,"15min":2.089},"min":{"1min":0.21199999999999999,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":8.2300000000000004,"5min":118.727,"15min":118.727},"last":12.372}]},{"osd":3,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":8.7230000000000008,"5min":8.109,"15min":3.3610000000000002},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":156.488,"5min":156.488,"15min":156.488},"last":3.6880000000000002},{"interface":"front","average":{"1min":9.0649999999999995,"5min":8.1379999999999999,"15min":3.4390000000000001},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":156.50700000000001,"5min":156.50700000000001,"15min":156.50700000000001},"last":3.8580000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.3540000000000001,"15min":1.625},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8310000000000004,"5min":59.439999999999998,"15min":118.633},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.151,"5min":2.9889999999999999,"15min":1.897},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.593,"5min":59.453000000000003,"15min":118.913},"last":0.39200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":4.548,"15min":2.4169999999999998},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.6310000000000002,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.302,"5min":4.2830000000000004,"15min":2.3039999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.8919999999999995,"5min":182.66300000000001,"15min":182.66300000000001},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":8.0259999999999998,"5min":3.976,"15min":2.2109999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":137.36099999999999,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":7.3899999999999997,"5min":3.5800000000000001,"15min":2.0670000000000002},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":136.88999999999999,"5min":136.88999999999999,"15min":136.88999999999999},"last":0.40899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":7.2210000000000001,"5min":5.7480000000000002,"15min":2.8370000000000002},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":121.324,"5min":185.024,"15min":185.024},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.206,"5min":4.21,"15min":2.3180000000000001},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":6.6280000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.441}]}]},{"osd":2,"up_from":94,"seq":403726925927,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1652956,"kb_used_data":1351188,"kb_used_omap":81,"kb_used_meta":301486,"kb_avail":92092196,"statfs":{"total":95995035648,"available":94302408704,"internally_reserved":0,"allocated":1383616512,"data_stored":1382787783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":308721882},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.4550000000000001,"15min":1.1419999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.413,"5min":12.284000000000001,"15min":12.284000000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":2.5840000000000001,"5min":3.2650000000000001,"15min":1.5189999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":11.356999999999999,"5min":45.979999999999997,"15min":45.979999999999997},"last":7.0060000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":3.6539999999999999,"15min":1.8160000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":14.99,"5min":53.695,"15min":68.009},"last":0.90700000000000003},{"interface":"front","average":{"1min":2.5979999999999999,"5min":3.8730000000000002,"15min":1.919},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":16.597000000000001,"5min":53.439,"15min":66.975999999999999},"last":0.26900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":2.2749999999999999,"15min":1.173},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2409999999999997,"5min":12.77,"15min":27.562000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.746,"5min":3.0459999999999998,"15min":1.405},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.952,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.84399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":2.419,"15min":1.3280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9209999999999998,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.524,"5min":1.7450000000000001,"15min":1.0329999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.5809999999999995,"5min":14.305,"15min":14.305},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.601,"15min":1.373},"min":{"1min":0.373,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":26.667000000000002,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.86199999999999999},{"interface":"front","average":{"1min":2.7999999999999998,"5min":2.8620000000000001,"15min":1.5},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":26.654,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.65900000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0199999999999996,"5min":2.9830000000000001,"15min":1.4690000000000001},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":114.14,"5min":114.14,"15min":114.14},"last":0.73299999999999998},{"interface":"front","average":{"1min":7.8810000000000002,"5min":3.6789999999999998,"15min":1.7549999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":114.123,"5min":114.123,"15min":114.123},"last":0.89300000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.1690000000000005,"5min":3.4550000000000001,"15min":1.651},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":117.32599999999999,"5min":117.32599999999999,"15min":117.32599999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":8.1639999999999997,"5min":3.9079999999999999,"15min":1.8540000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":117.28700000000001,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.878}]}]},{"osd":3,"up_from":99,"seq":425201762401,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2993476,"kb_used_data":2609084,"kb_used_omap":7,"kb_used_meta":384376,"kb_avail":90751676,"statfs":{"total":95995035648,"available":92929716224,"internally_reserved":0,"allocated":2671702016,"data_stored":2670894345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":393601640},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":7.7279999999999998,"5min":5.4539999999999997,"15min":3.0920000000000001},"min":{"1min":0.25600000000000001,"5min":0.189,"15min":0.185},"max":{"1min":100.512,"5min":100.512,"15min":106.22},"last":1.907},{"interface":"front","average":{"1min":8.4130000000000003,"5min":5.7060000000000004,"15min":3.1930000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":101.733,"5min":101.733,"15min":106.986},"last":1.893}]},{"osd":1,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.1470000000000002,"15min":2.069},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.19800000000000001},"max":{"1min":57.055999999999997,"5min":64.677000000000007,"15min":106.121},"last":2.206},{"interface":"front","average":{"1min":7.2649999999999997,"5min":4.7859999999999996,"15min":1.8660000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":57.088999999999999,"5min":65.015000000000001,"15min":65.015000000000001},"last":1.946}]},{"osd":2,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":3.5609999999999999,"15min":1.905},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":4.593,"5min":85.974000000000004,"15min":106.105},"last":8.3689999999999998},{"interface":"front","average":{"1min":6.7130000000000001,"5min":4.79,"15min":2.3079999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":98.765000000000001,"5min":98.765000000000001,"15min":106.70399999999999},"last":8.8539999999999992}]},{"osd":4,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":2.8090000000000002,"15min":1.7869999999999999},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.308},"max":{"1min":5.4740000000000002,"5min":37.652000000000001,"15min":106.85899999999999},"last":8.8940000000000001},{"interface":"front","average":{"1min":1.353,"5min":2.5350000000000001,"15min":1.266},"min":{"1min":0.32500000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.2409999999999997,"5min":37.442,"15min":37.442},"last":2.2469999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":2.2330000000000001,"15min":1.1559999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3150000000000004,"5min":12.819000000000001,"15min":12.819000000000001},"last":2.2949999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.1539999999999999,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.0789999999999997,"5min":13.228999999999999,"15min":13.228999999999999},"last":1.9950000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":2.004,"15min":1.8260000000000001},"min":{"1min":0.35299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1100000000000003,"5min":13.255000000000001,"15min":106.036},"last":1.9710000000000001},{"interface":"front","average":{"1min":1.528,"5min":2.3759999999999999,"15min":1.6619999999999999},"min":{"1min":0.40400000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.4050000000000002,"5min":15.622,"15min":106.741},"last":2.1890000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":2.5009999999999999,"15min":1.7090000000000001},"min":{"1min":0.38200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.5069999999999997,"5min":15.685,"15min":106.82599999999999},"last":2.2240000000000002},{"interface":"front","average":{"1min":1.224,"5min":2.2429999999999999,"15min":1.5980000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.266,"5min":13.228999999999999,"15min":106.084},"last":2.2709999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986417,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1579196,"kb_used_data":1269620,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92165956,"statfs":{"total":95995035648,"available":94377938944,"internally_reserved":0,"allocated":1300090880,"data_stored":1299467885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":2.8290000000000002,"15min":1.3500000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":1.726,"5min":63.128,"15min":63.128},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.5110000000000001,"15min":1.3149999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1600000000000001,"5min":62.645000000000003,"15min":62.645000000000003},"last":0.33100000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":2.7330000000000001,"15min":1.284},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1189999999999998,"5min":79.075999999999993,"15min":79.075999999999993},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.321,"5min":2.714,"15min":1.276},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.9260000000000002,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8380000000000001,"15min":1.0900000000000001},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.698,"5min":19.356000000000002,"15min":19.356000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.125,"5min":1.845,"15min":1.042},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7109999999999999,"5min":19.324000000000002,"15min":19.324000000000002},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.863,"15min":1.0169999999999999},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.5629999999999997,"5min":14.694000000000001,"15min":14.694000000000001},"last":0.504},{"interface":"front","average":{"1min":1.927,"5min":2.1320000000000001,"15min":1.125},"min":{"1min":0.48999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":7.4909999999999997,"5min":10.476000000000001,"15min":10.476000000000001},"last":1.0329999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.77300000000000002,"15min":0.57899999999999996},"min":{"1min":0.27700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.72199999999999998,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.1279999999999999,"15min":0.71899999999999997},"min":{"1min":0.314,"5min":0.25,"15min":0.25},"max":{"1min":1.544,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.0920000000000001,"15min":0.72699999999999998},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.6180000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.871,"15min":0.65400000000000003},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.669,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.38200000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.96999999999999997,"15min":0.69899999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.4890000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.004,"15min":0.76100000000000001},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.306,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.46700000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724781,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1291180,"kb_used_data":985700,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92453972,"statfs":{"total":95995035648,"available":94672867328,"internally_reserved":0,"allocated":1009356800,"data_stored":1008745875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":6.391,"5min":3.4849999999999999,"15min":1.577},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":83.329999999999998,"5min":83.329999999999998,"15min":83.329999999999998},"last":6.3369999999999997},{"interface":"front","average":{"1min":7.7030000000000003,"5min":3.778,"15min":1.6990000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":85.451999999999998,"5min":85.451999999999998,"15min":85.451999999999998},"last":0.54000000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2040000000000002,"15min":1.446},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.4980000000000002,"5min":91.212999999999994,"15min":91.212999999999994},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.252,"5min":3.75,"15min":1.6439999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.9460000000000002,"5min":91.254000000000005,"15min":91.254000000000005},"last":0.56599999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":3.0880000000000001,"15min":1.524},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.2959999999999998,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.748},{"interface":"front","average":{"1min":1.968,"5min":3.2050000000000001,"15min":1.518},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.6929999999999996,"5min":61.154000000000003,"15min":61.154000000000003},"last":0.78900000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:10:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.04,"15min":1.657},"min":{"1min":0.35099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.1699999999999999,"5min":91.834999999999994,"15min":91.834999999999994},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.4159999999999999,"5min":3.0760000000000001,"15min":1.6819999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.1100000000000003,"5min":91.802000000000007,"15min":91.802000000000007},"last":0.86499999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.96199999999999997,"15min":0.64400000000000002},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.253},"max":{"1min":2.1299999999999999,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.505},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.097,"15min":0.70999999999999996},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.26600000000000001},"max":{"1min":1.6539999999999999,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.76500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.91400000000000003,"15min":0.66100000000000003},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.4059999999999999,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.88500000000000001,"15min":0.67500000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":1.4630000000000001,"5min":5.7670000000000003,"15min":5.7670000000000003},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.1859999999999999,"15min":0.78700000000000003},"min":{"1min":0.35599999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.7789999999999999,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.159,"15min":0.80300000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":1.5269999999999999,"5min":8.5329999999999995,"15min":8.5329999999999995},"last":0.46300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2337083392,"data_stored":2337079320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1924927488,"data_stored":1924923416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1372192768,"data_stored":1372192768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2652045312,"data_stored":2652045312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1292632064,"data_stored":1292632064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1001127936,"data_stored":1001127936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1838485504,"data_stored":1838481432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1675427840,"data_stored":1675427840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:11:36.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:11:36.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:36.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:36 smithi181 ceph-mon[196979]: from='client.35277 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:36 smithi181 ceph-mon[196979]: pgmap v549: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 2197/204021 objects degraded (1.077%); 8412/204021 objects misplaced (4.123%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:37.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:37.106 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:37.155 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:37.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:37.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:37.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:37.493 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:37.552 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:37.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:37.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:37.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/204021 objects degraded (1.077%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:37.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/204021 objects degraded (1.077%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:37.650 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:37.650 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:37.658 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:37.688 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:37.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:37.747 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/204021 objects degraded (1.077%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:11:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:37.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:37.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:37.883 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:37.900 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:37.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:37.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:38.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:38.096 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:38.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:11:38.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:38 smithi163 ceph-mon[221067]: pgmap v550: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2197/200229 objects degraded (1.097%); 8031/200229 objects misplaced (4.011%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:38.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:38 smithi163 ceph-mon[213841]: pgmap v550: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2197/200229 objects degraded (1.097%); 8031/200229 objects misplaced (4.011%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:38 smithi181 ceph-mon[196979]: pgmap v550: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2197/200229 objects degraded (1.097%); 8031/200229 objects misplaced (4.011%); 710 KiB/s, 11 objects/s recovering 2023-10-09T04:11:39.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:11:39.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:39.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:39.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:39.630 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:39.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:39.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:39.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:40.008 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:40.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:40.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:40.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:40.280 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:40.331 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:40.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:40.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:40.520 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:40.834 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:11:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:11:40] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:11:40.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:40 smithi163 ceph-mon[221067]: pgmap v551: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 2197/194682 objects degraded (1.129%); 7910/194682 objects misplaced (4.063%); 755 KiB/s, 11 objects/s recovering 2023-10-09T04:11:40.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:40 smithi163 ceph-mon[213841]: pgmap v551: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 2197/194682 objects degraded (1.129%); 7910/194682 objects misplaced (4.063%); 755 KiB/s, 11 objects/s recovering 2023-10-09T04:11:40.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:40.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:40.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:40.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:40 smithi181 ceph-mon[196979]: pgmap v551: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 2197/194682 objects degraded (1.129%); 7910/194682 objects misplaced (4.063%); 755 KiB/s, 11 objects/s recovering 2023-10-09T04:11:40.984 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:41.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:41.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:41.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:41.106 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:41.116 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:41.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:41.176 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:41.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:41.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:41.334 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:41.370 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:41.426 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:41.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:41.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:41.566 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:41.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:41.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:41.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:41.762 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:41.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:41.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:41.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:42.011 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:42.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:42.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:42.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:42.302 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:11:42.302 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2197/189243 objects degraded (1.161%), 1 pg degraded, 2 pgs undersized","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":666,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":213,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":8,"osd_in_since":1696823548,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":94},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":63081,"data_bytes":4133033912,"bytes_used":15298105344,"bytes_avail":752662179840,"bytes_total":767960285184,"degraded_objects":2197,"degraded_total":189243,"degraded_ratio":0.01160941223717654,"misplaced_objects":7541,"misplaced_total":189243,"misplaced_ratio":0.039848237451319206,"recovering_objects_per_sec":11,"recovering_bytes_per_sec":730809,"recovering_keys_per_sec":0,"num_objects_recovered":118,"num_bytes_recovered":7733248,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":700},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":21,"modified":"2023-10-09T04:04:35.266858+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"61f8f0b7-6515-43d1-9684-78e0e9193886":{"message":"Global Recovery Event (95s)\n [==========================..] (remaining: 5s)","progress":0.94949495792388916,"add_to_ceph_s":true}}} 2023-10-09T04:11:42.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:42.437 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:42.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:42.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:42.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:42.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:42.627 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:11:42.638 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:42.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:42.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:42 smithi181 ceph-mon[196979]: pgmap v552: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2197/189243 objects degraded (1.161%); 7541/189243 objects misplaced (3.985%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:11:42.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1729890880' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:42.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:42 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:42.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/189243 objects degraded (1.161%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:42.712 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2756361583' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:11:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:42.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:42.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:42.847 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:42.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:42.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:43.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:43.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[213841]: pgmap v552: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2197/189243 objects degraded (1.161%); 7541/189243 objects misplaced (3.985%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:11:43.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1729890880' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:43.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:43.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/189243 objects degraded (1.161%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:43.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2756361583' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:11:43.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[221067]: pgmap v552: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2197/189243 objects degraded (1.161%); 7541/189243 objects misplaced (3.985%); 714 KiB/s, 11 objects/s recovering 2023-10-09T04:11:43.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1729890880' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:43.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:11:43.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/189243 objects degraded (1.161%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:43.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2756361583' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:11:43.093 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:43.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:43.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:43.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:43.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:43.494 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:43.595 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:43.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:11:43.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[221067]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T04:11:43.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[221067]: failed to encode map e214 with expected crc 2023-10-09T04:11:43.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[221067]: failed to encode map e214 with expected crc 2023-10-09T04:11:43.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:43.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:43.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:11:43.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[213841]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T04:11:43.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[213841]: failed to encode map e214 with expected crc 2023-10-09T04:11:43.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:43 smithi163 ceph-mon[213841]: failed to encode map e214 with expected crc 2023-10-09T04:11:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:43.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:44.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:43 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:11:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:43 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:11:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:43 smithi181 ceph-mon[196979]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T04:11:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:43 smithi181 ceph-mon[196979]: failed to encode map e214 with expected crc 2023-10-09T04:11:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:43 smithi181 ceph-mon[196979]: failed to encode map e214 with expected crc 2023-10-09T04:11:44.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:44.122 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:44.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:44 smithi163 ceph-mon[213841]: failed to encode map e214 with expected crc 2023-10-09T04:11:44.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:44 smithi163 ceph-mon[213841]: pgmap v554: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2197/185604 objects degraded (1.184%); 7416/185604 objects misplaced (3.996%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:44.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:44 smithi163 ceph-mon[221067]: failed to encode map e214 with expected crc 2023-10-09T04:11:44.704 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:44 smithi163 ceph-mon[221067]: pgmap v554: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2197/185604 objects degraded (1.184%); 7416/185604 objects misplaced (3.996%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:44.872 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:44.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:44.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:44.980 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:11:44.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:11:44.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:45.017 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:44 smithi181 ceph-mon[196979]: failed to encode map e214 with expected crc 2023-10-09T04:11:45.017 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:44 smithi181 ceph-mon[196979]: pgmap v554: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 2197/185604 objects degraded (1.184%); 7416/185604 objects misplaced (3.996%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:45.020 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:45.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:45.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:45.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:45.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:45.284 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:45.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:45.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:45.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:45.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:45.663 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:45.696 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:45.696 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:45.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:45.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:45.815 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:45.847 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:45.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:45.884 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:45.946 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:45.972 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:45.992 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:46.028 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:46.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:46.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:46.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:46.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:46.100 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:46.101 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.101 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.101 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:11:46.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:46.262 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:46.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:46.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:46.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:46.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:46.516 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:46.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:46.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:46.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:46.715 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:46.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:46 smithi181 ceph-mon[196979]: pgmap v555: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 2197/181581 objects degraded (1.210%); 7416/181581 objects misplaced (4.084%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1246584599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:46.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:46.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:46.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:46.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:46.922 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:46.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:46 smithi163 ceph-mon[221067]: pgmap v555: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 2197/181581 objects degraded (1.210%); 7416/181581 objects misplaced (4.084%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:46.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1246584599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:46.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:46 smithi163 ceph-mon[213841]: pgmap v555: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 2197/181581 objects degraded (1.210%); 7416/181581 objects misplaced (4.084%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:46.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1246584599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:46.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:46.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:47.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:11:47.089 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:47.152 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:47.334 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:47.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:11:47.477 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:47.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:47.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:47.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:47.619 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:47.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/181581 objects degraded (1.210%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:47.658 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:47 smithi163 ceph-mon[213841]: pgmap v556: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 2197/178020 objects degraded (1.234%); 7049/178020 objects misplaced (3.960%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:47.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/181581 objects degraded (1.210%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:47.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:47 smithi163 ceph-mon[221067]: pgmap v556: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 2197/178020 objects degraded (1.234%); 7049/178020 objects misplaced (3.960%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:47.671 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:47.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:47.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:47.877 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/181581 objects degraded (1.210%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:47 smithi181 ceph-mon[196979]: pgmap v556: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 2197/178020 objects degraded (1.234%); 7049/178020 objects misplaced (3.960%); 787 KiB/s, 12 objects/s recovering 2023-10-09T04:11:48.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:48.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:48.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:48.221 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:48.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:48.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:48.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:48.411 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:48.453 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:48.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:48.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:48.583 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:48.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:48.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:48.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:48.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:48.789 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:48.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:48.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:48.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:48.993 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:48.995 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:49.049 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:49.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:49.083 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:49.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:49.181 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:49.230 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:49.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:49.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:49.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:11:49.429 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:49.571 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:49.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:11:49.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:11:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:49.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:11:49.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:49.899 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:50.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:50.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:50.235 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:11:50.240 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:11:50.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:50.361 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:50.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:50 smithi163 ceph-mon[213841]: pgmap v557: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 2197/172488 objects degraded (1.274%); 6927/172488 objects misplaced (4.016%); 793 KiB/s, 12 objects/s recovering 2023-10-09T04:11:50.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:50 smithi163 ceph-mon[221067]: pgmap v557: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 2197/172488 objects degraded (1.274%); 6927/172488 objects misplaced (4.016%); 793 KiB/s, 12 objects/s recovering 2023-10-09T04:11:50.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:50 smithi181 ceph-mon[196979]: pgmap v557: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 2197/172488 objects degraded (1.274%); 6927/172488 objects misplaced (4.016%); 793 KiB/s, 12 objects/s recovering 2023-10-09T04:11:51.038 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:11:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:11:50] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:11:51.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:51 smithi163 ceph-mon[213841]: from='client.54997 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:51.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:51.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:51.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:51.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:51 smithi163 ceph-mon[221067]: from='client.54997 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:51.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:51.627 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:51.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:51 smithi181 ceph-mon[196979]: from='client.54997 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:11:52.121 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":557,"stamp":"2023-10-09T04:11:49.465176+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3767015352,"num_objects":57496,"num_object_clones":0,"num_object_copies":172488,"num_objects_missing_on_primary":158,"num_objects_missing":158,"num_objects_degraded":2197,"num_objects_misplaced":6927,"num_objects_unfound":0,"num_objects_dirty":57496,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":122925,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10640,"num_bytes_recovered":698419892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148075,"ondisk_log_size":148075,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13986252,"kb_used_data":11638604,"kb_used_omap":126,"kb_used_meta":2347201,"kb_avail":735974964,"statfs":{"total":767960285184,"available":753638363136,"internally_reserved":0,"allocated":11917930496,"data_stored":11912013731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2403534459},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-484835328,"num_objects":-7398,"num_object_clones":0,"num_object_copies":-22194,"num_objects_missing_on_primary":-123,"num_objects_missing":-123,"num_objects_degraded":0,"num_objects_misplaced":-983,"num_objects_unfound":0,"num_objects_dirty":-7398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7398,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":8126464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002820"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":211,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901455+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:11:30.901455+0000","last_peered":"2023-10-09T04:11:30.901455+0000","last_clean":"2023-10-09T04:11:30.901455+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:11:30.901455+0000","last_undegraded":"2023-10-09T04:11:30.901455+0000","last_fullsized":"2023-10-09T04:11:30.901455+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":150,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.606428+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:11:42.606428+0000","last_peered":"2023-10-09T04:11:42.606428+0000","last_clean":"2023-10-09T04:11:42.606428+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:11:42.606428+0000","last_undegraded":"2023-10-09T04:11:42.606428+0000","last_fullsized":"2023-10-09T04:11:42.606428+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.887623+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:11:30.887623+0000","last_peered":"2023-10-09T04:11:30.887623+0000","last_clean":"2023-10-09T04:11:30.887623+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:30.887623+0000","last_undegraded":"2023-10-09T04:11:30.887623+0000","last_fullsized":"2023-10-09T04:11:30.887623+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":169,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921504+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:11:30.921504+0000","last_peered":"2023-10-09T04:11:30.921504+0000","last_clean":"2023-10-09T04:11:30.921504+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:11:30.921504+0000","last_undegraded":"2023-10-09T04:11:30.921504+0000","last_fullsized":"2023-10-09T04:11:30.921504+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":168,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:37.901429+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:11:37.901429+0000","last_peered":"2023-10-09T04:11:37.901429+0000","last_clean":"2023-10-09T04:11:37.901429+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:11:37.901429+0000","last_undegraded":"2023-10-09T04:11:37.901429+0000","last_fullsized":"2023-10-09T04:11:37.901429+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":147,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894606+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:11:30.894606+0000","last_peered":"2023-10-09T04:11:30.894606+0000","last_clean":"2023-10-09T04:11:30.894606+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:11:30.894606+0000","last_undegraded":"2023-10-09T04:11:30.894606+0000","last_fullsized":"2023-10-09T04:11:30.894606+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"214'5168","reported_seq":8121,"reported_epoch":214,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:45.670781+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:11:45.670781+0000","last_peered":"2023-10-09T04:11:45.670781+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:11:45.670781+0000","last_undegraded":"2023-10-09T04:11:45.670781+0000","last_fullsized":"2023-10-09T04:11:45.670781+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5168,"log_dups_size":0,"ondisk_log_size":5168,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117374976,"num_objects":1791,"num_object_clones":0,"num_object_copies":5373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1791,"num_objects_unfound":0,"num_objects_dirty":1791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3997,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1791}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":169,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616101+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:11:42.616101+0000","last_peered":"2023-10-09T04:11:42.616101+0000","last_clean":"2023-10-09T04:11:42.616101+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:11:42.616101+0000","last_undegraded":"2023-10-09T04:11:42.616101+0000","last_fullsized":"2023-10-09T04:11:42.616101+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.906836+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:11:30.906836+0000","last_peered":"2023-10-09T04:11:30.906836+0000","last_clean":"2023-10-09T04:11:30.906836+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:11:30.906836+0000","last_undegraded":"2023-10-09T04:11:30.906836+0000","last_fullsized":"2023-10-09T04:11:30.906836+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"214'5301","reported_seq":10036,"reported_epoch":214,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:11:47.961821+0000","last_change":"2023-10-09T04:10:51.646576+0000","last_active":"2023-10-09T04:11:47.961821+0000","last_peered":"2023-10-09T04:11:47.961821+0000","last_clean":"2023-10-09T04:09:59.283093+0000","last_became_active":"2023-10-09T04:10:40.892310+0000","last_became_peered":"2023-10-09T04:10:40.892310+0000","last_unstale":"2023-10-09T04:11:47.961821+0000","last_undegraded":"2023-10-09T04:11:47.961821+0000","last_fullsized":"2023-10-09T04:10:40.864699+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5301,"log_dups_size":0,"ondisk_log_size":5301,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119144448,"num_objects":1818,"num_object_clones":0,"num_object_copies":5454,"num_objects_missing_on_primary":158,"num_objects_missing":158,"num_objects_degraded":0,"num_objects_misplaced":1403,"num_objects_unfound":0,"num_objects_dirty":1818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4136,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1308,"num_bytes_recovered":85721088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"0,1,6,7","objects":1087},{"shards":"0,6","objects":573},{"shards":"1,2,3","objects":158}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894623+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:11:30.894623+0000","last_peered":"2023-10-09T04:11:30.894623+0000","last_clean":"2023-10-09T04:11:30.894623+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:11:30.894623+0000","last_undegraded":"2023-10-09T04:11:30.894623+0000","last_fullsized":"2023-10-09T04:11:30.894623+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":110,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886308+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:11:30.886308+0000","last_peered":"2023-10-09T04:11:30.886308+0000","last_clean":"2023-10-09T04:11:30.886308+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:11:30.886308+0000","last_undegraded":"2023-10-09T04:11:30.886308+0000","last_fullsized":"2023-10-09T04:11:30.886308+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"214'5117","reported_seq":8001,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:45.479838+0000","last_change":"2023-10-09T04:10:18.287893+0000","last_active":"2023-10-09T04:11:45.479838+0000","last_peered":"2023-10-09T04:11:45.479838+0000","last_clean":"2023-10-09T04:11:45.479838+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:11:45.479838+0000","last_undegraded":"2023-10-09T04:11:45.479838+0000","last_fullsized":"2023-10-09T04:11:45.479838+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117833728,"num_objects":1798,"num_object_clones":0,"num_object_copies":5394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3982,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":176,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886658+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:11:30.886658+0000","last_peered":"2023-10-09T04:11:30.886658+0000","last_clean":"2023-10-09T04:11:30.886658+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:11:30.886658+0000","last_undegraded":"2023-10-09T04:11:30.886658+0000","last_fullsized":"2023-10-09T04:11:30.886658+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.889285+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:11:30.889285+0000","last_peered":"2023-10-09T04:11:30.889285+0000","last_clean":"2023-10-09T04:11:30.889285+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:11:30.889285+0000","last_undegraded":"2023-10-09T04:11:30.889285+0000","last_fullsized":"2023-10-09T04:11:30.889285+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"214'5278","reported_seq":8213,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:44.909230+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:11:44.909230+0000","last_peered":"2023-10-09T04:11:44.909230+0000","last_clean":"2023-10-09T04:11:44.909230+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:11:44.909230+0000","last_undegraded":"2023-10-09T04:11:44.909230+0000","last_fullsized":"2023-10-09T04:11:44.909230+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5278,"log_dups_size":0,"ondisk_log_size":5278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118423552,"num_objects":1807,"num_object_clones":0,"num_object_copies":5421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4089,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901792+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:11:30.901792+0000","last_peered":"2023-10-09T04:11:30.901792+0000","last_clean":"2023-10-09T04:11:30.901792+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:11:30.901792+0000","last_undegraded":"2023-10-09T04:11:30.901792+0000","last_fullsized":"2023-10-09T04:11:30.901792+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901877+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:11:30.901877+0000","last_peered":"2023-10-09T04:11:30.901877+0000","last_clean":"2023-10-09T04:11:30.901877+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:11:30.901877+0000","last_undegraded":"2023-10-09T04:11:30.901877+0000","last_fullsized":"2023-10-09T04:11:30.901877+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"214'5214","reported_seq":5280,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.899256+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:11:47.899256+0000","last_peered":"2023-10-09T04:11:47.899256+0000","last_clean":"2023-10-09T04:11:47.899256+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:11:47.899256+0000","last_undegraded":"2023-10-09T04:11:47.899256+0000","last_fullsized":"2023-10-09T04:11:47.899256+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5214,"log_dups_size":0,"ondisk_log_size":5214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4060,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":175,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886347+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:11:30.886347+0000","last_peered":"2023-10-09T04:11:30.886347+0000","last_clean":"2023-10-09T04:11:30.886347+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:11:30.886347+0000","last_undegraded":"2023-10-09T04:11:30.886347+0000","last_fullsized":"2023-10-09T04:11:30.886347+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":150,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.887937+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:11:30.887937+0000","last_peered":"2023-10-09T04:11:30.887937+0000","last_clean":"2023-10-09T04:11:30.887937+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:11:30.887937+0000","last_undegraded":"2023-10-09T04:11:30.887937+0000","last_fullsized":"2023-10-09T04:11:30.887937+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"214'5072","reported_seq":5137,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.962315+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:11:47.962315+0000","last_peered":"2023-10-09T04:11:47.962315+0000","last_clean":"2023-10-09T04:11:47.962315+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:11:47.962315+0000","last_undegraded":"2023-10-09T04:11:47.962315+0000","last_fullsized":"2023-10-09T04:11:47.962315+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110788608,"num_objects":1691,"num_object_clones":0,"num_object_copies":5073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3955,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":169,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.932131+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:11:30.932131+0000","last_peered":"2023-10-09T04:11:30.932131+0000","last_clean":"2023-10-09T04:11:30.932131+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:11:30.932131+0000","last_undegraded":"2023-10-09T04:11:30.932131+0000","last_fullsized":"2023-10-09T04:11:30.932131+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894515+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:11:30.894515+0000","last_peered":"2023-10-09T04:11:30.894515+0000","last_clean":"2023-10-09T04:11:30.894515+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:11:30.894515+0000","last_undegraded":"2023-10-09T04:11:30.894515+0000","last_fullsized":"2023-10-09T04:11:30.894515+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"214'5133","reported_seq":8076,"reported_epoch":214,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:45.440568+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:11:45.440568+0000","last_peered":"2023-10-09T04:11:45.440568+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:11:45.440568+0000","last_undegraded":"2023-10-09T04:11:45.440568+0000","last_fullsized":"2023-10-09T04:11:45.440568+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5133,"log_dups_size":0,"ondisk_log_size":5133,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123568128,"num_objects":1886,"num_object_clones":0,"num_object_copies":5658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1886,"num_objects_unfound":0,"num_objects_dirty":1886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4018,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1886}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":156,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886113+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:11:30.886113+0000","last_peered":"2023-10-09T04:11:30.886113+0000","last_clean":"2023-10-09T04:11:30.886113+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:11:30.886113+0000","last_undegraded":"2023-10-09T04:11:30.886113+0000","last_fullsized":"2023-10-09T04:11:30.886113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":169,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615799+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:11:42.615799+0000","last_peered":"2023-10-09T04:11:42.615799+0000","last_clean":"2023-10-09T04:11:42.615799+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:11:42.615799+0000","last_undegraded":"2023-10-09T04:11:42.615799+0000","last_fullsized":"2023-10-09T04:11:42.615799+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"214'5254","reported_seq":5320,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.942089+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:11:47.942089+0000","last_peered":"2023-10-09T04:11:47.942089+0000","last_clean":"2023-10-09T04:11:47.942089+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:11:47.942089+0000","last_undegraded":"2023-10-09T04:11:47.942089+0000","last_fullsized":"2023-10-09T04:11:47.942089+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5254,"log_dups_size":0,"ondisk_log_size":5254,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119013376,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4092,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":105,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:37.901398+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:11:37.901398+0000","last_peered":"2023-10-09T04:11:37.901398+0000","last_clean":"2023-10-09T04:11:37.901398+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:11:37.901398+0000","last_undegraded":"2023-10-09T04:11:37.901398+0000","last_fullsized":"2023-10-09T04:11:37.901398+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.913278+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:11:30.913278+0000","last_peered":"2023-10-09T04:11:30.913278+0000","last_clean":"2023-10-09T04:11:30.913278+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:11:30.913278+0000","last_undegraded":"2023-10-09T04:11:30.913278+0000","last_fullsized":"2023-10-09T04:11:30.913278+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"214'8113","reported_seq":8170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:48.635635+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:11:48.635635+0000","last_peered":"2023-10-09T04:11:48.635635+0000","last_clean":"2023-10-09T04:11:48.635635+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:11:48.635635+0000","last_undegraded":"2023-10-09T04:11:48.635635+0000","last_fullsized":"2023-10-09T04:11:48.635635+0000","mapping_epoch":167,"log_start":"171'1100","ondisk_log_start":"171'1100","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233504768,"num_objects":3563,"num_object_clones":0,"num_object_copies":10689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8113,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":174,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886858+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:11:30.886858+0000","last_peered":"2023-10-09T04:11:30.886858+0000","last_clean":"2023-10-09T04:11:30.886858+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:11:30.886858+0000","last_undegraded":"2023-10-09T04:11:30.886858+0000","last_fullsized":"2023-10-09T04:11:30.886858+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894257+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:11:30.894257+0000","last_peered":"2023-10-09T04:11:30.894257+0000","last_clean":"2023-10-09T04:11:30.894257+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:11:30.894257+0000","last_undegraded":"2023-10-09T04:11:30.894257+0000","last_fullsized":"2023-10-09T04:11:30.894257+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"214'7951","reported_seq":8008,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:44.946657+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:11:44.946657+0000","last_peered":"2023-10-09T04:11:44.946657+0000","last_clean":"2023-10-09T04:11:44.946657+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:11:44.946657+0000","last_undegraded":"2023-10-09T04:11:44.946657+0000","last_fullsized":"2023-10-09T04:11:44.946657+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236650496,"num_objects":3611,"num_object_clones":0,"num_object_copies":10833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7951,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":175,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886652+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:11:30.886652+0000","last_peered":"2023-10-09T04:11:30.886652+0000","last_clean":"2023-10-09T04:11:30.886652+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:11:30.886652+0000","last_undegraded":"2023-10-09T04:11:30.886652+0000","last_fullsized":"2023-10-09T04:11:30.886652+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901959+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:11:30.901959+0000","last_peered":"2023-10-09T04:11:30.901959+0000","last_clean":"2023-10-09T04:11:30.901959+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:11:30.901959+0000","last_undegraded":"2023-10-09T04:11:30.901959+0000","last_fullsized":"2023-10-09T04:11:30.901959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"214'5266","reported_seq":5440,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.902540+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:11:47.902540+0000","last_peered":"2023-10-09T04:11:47.902540+0000","last_clean":"2023-10-09T04:11:47.902540+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:11:47.902540+0000","last_undegraded":"2023-10-09T04:11:47.902540+0000","last_fullsized":"2023-10-09T04:11:47.902540+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":5266,"log_dups_size":0,"ondisk_log_size":5266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117440512,"num_objects":1792,"num_object_clones":0,"num_object_copies":5376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2880,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":157,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886206+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:11:30.886206+0000","last_peered":"2023-10-09T04:11:30.886206+0000","last_clean":"2023-10-09T04:11:30.886206+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:11:30.886206+0000","last_undegraded":"2023-10-09T04:11:30.886206+0000","last_fullsized":"2023-10-09T04:11:30.886206+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":133,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.888946+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:11:30.888946+0000","last_peered":"2023-10-09T04:11:30.888946+0000","last_clean":"2023-10-09T04:11:30.888946+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:11:30.888946+0000","last_undegraded":"2023-10-09T04:11:30.888946+0000","last_fullsized":"2023-10-09T04:11:30.888946+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"214'5066","reported_seq":5137,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:44.952499+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:11:44.952499+0000","last_peered":"2023-10-09T04:11:44.952499+0000","last_clean":"2023-10-09T04:11:44.952499+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:11:44.952499+0000","last_undegraded":"2023-10-09T04:11:44.952499+0000","last_fullsized":"2023-10-09T04:11:44.952499+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118358016,"num_objects":1806,"num_object_clones":0,"num_object_copies":5418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3926,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894446+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:11:30.894446+0000","last_peered":"2023-10-09T04:11:30.894446+0000","last_clean":"2023-10-09T04:11:30.894446+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:11:30.894446+0000","last_undegraded":"2023-10-09T04:11:30.894446+0000","last_fullsized":"2023-10-09T04:11:30.894446+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":169,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921151+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:11:30.921151+0000","last_peered":"2023-10-09T04:11:30.921151+0000","last_clean":"2023-10-09T04:11:30.921151+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:11:30.921151+0000","last_undegraded":"2023-10-09T04:11:30.921151+0000","last_fullsized":"2023-10-09T04:11:30.921151+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"214'8161","reported_seq":8217,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.977814+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:11:47.977814+0000","last_peered":"2023-10-09T04:11:47.977814+0000","last_clean":"2023-10-09T04:11:47.977814+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:11:47.977814+0000","last_undegraded":"2023-10-09T04:11:47.977814+0000","last_fullsized":"2023-10-09T04:11:47.977814+0000","mapping_epoch":167,"log_start":"171'600","ondisk_log_start":"171'600","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7561,"log_dups_size":0,"ondisk_log_size":7561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229441536,"num_objects":3501,"num_object_clones":0,"num_object_copies":10503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8161,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":118,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921079+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:11:30.921079+0000","last_peered":"2023-10-09T04:11:30.921079+0000","last_clean":"2023-10-09T04:11:30.921079+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:11:30.921079+0000","last_undegraded":"2023-10-09T04:11:30.921079+0000","last_fullsized":"2023-10-09T04:11:30.921079+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901695+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:11:30.901695+0000","last_peered":"2023-10-09T04:11:30.901695+0000","last_clean":"2023-10-09T04:11:30.901695+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:11:30.901695+0000","last_undegraded":"2023-10-09T04:11:30.901695+0000","last_fullsized":"2023-10-09T04:11:30.901695+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"214'7949","reported_seq":8022,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:11:45.598968+0000","last_change":"2023-10-09T04:10:38.444053+0000","last_active":"2023-10-09T04:11:45.598968+0000","last_peered":"2023-10-09T04:11:45.598968+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:11:45.598968+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240451584,"num_objects":3669,"num_object_clones":0,"num_object_copies":11007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7949,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":2197},{"shards":"0,5","objects":1472}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901474+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:11:30.901474+0000","last_peered":"2023-10-09T04:11:30.901474+0000","last_clean":"2023-10-09T04:11:30.901474+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:11:30.901474+0000","last_undegraded":"2023-10-09T04:11:30.901474+0000","last_fullsized":"2023-10-09T04:11:30.901474+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":209,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901555+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:11:30.901555+0000","last_peered":"2023-10-09T04:11:30.901555+0000","last_clean":"2023-10-09T04:11:30.901555+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:11:30.901555+0000","last_undegraded":"2023-10-09T04:11:30.901555+0000","last_fullsized":"2023-10-09T04:11:30.901555+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"214'8116","reported_seq":8174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:48.608878+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:11:48.608878+0000","last_peered":"2023-10-09T04:11:48.608878+0000","last_clean":"2023-10-09T04:11:48.608878+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:11:48.608878+0000","last_undegraded":"2023-10-09T04:11:48.608878+0000","last_fullsized":"2023-10-09T04:11:48.608878+0000","mapping_epoch":167,"log_start":"171'1100","ondisk_log_start":"171'1100","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229572632,"num_objects":3504,"num_object_clones":0,"num_object_copies":10512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3504,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8116,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901792+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:11:30.901792+0000","last_peered":"2023-10-09T04:11:30.901792+0000","last_clean":"2023-10-09T04:11:30.901792+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:11:30.901792+0000","last_undegraded":"2023-10-09T04:11:30.901792+0000","last_fullsized":"2023-10-09T04:11:30.901792+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":165,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.902753+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:11:30.902753+0000","last_peered":"2023-10-09T04:11:30.902753+0000","last_clean":"2023-10-09T04:11:30.902753+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:11:30.902753+0000","last_undegraded":"2023-10-09T04:11:30.902753+0000","last_fullsized":"2023-10-09T04:11:30.902753+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"214'7800","reported_seq":7857,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:45.617617+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:11:45.617617+0000","last_peered":"2023-10-09T04:11:45.617617+0000","last_clean":"2023-10-09T04:11:45.617617+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:11:45.617617+0000","last_undegraded":"2023-10-09T04:11:45.617617+0000","last_fullsized":"2023-10-09T04:11:45.617617+0000","mapping_epoch":167,"log_start":"172'1400","ondisk_log_start":"172'1400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237895680,"num_objects":3630,"num_object_clones":0,"num_object_copies":10890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7800,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":179,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.906673+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:11:30.906673+0000","last_peered":"2023-10-09T04:11:30.906673+0000","last_clean":"2023-10-09T04:11:30.906673+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:11:30.906673+0000","last_undegraded":"2023-10-09T04:11:30.906673+0000","last_fullsized":"2023-10-09T04:11:30.906673+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:33.666727+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:11:33.666727+0000","last_peered":"2023-10-09T04:11:33.666727+0000","last_clean":"2023-10-09T04:11:33.666727+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:11:33.666727+0000","last_undegraded":"2023-10-09T04:11:33.666727+0000","last_fullsized":"2023-10-09T04:11:33.666727+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"214'5120","reported_seq":5309,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:45.248236+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:11:45.248236+0000","last_peered":"2023-10-09T04:11:45.248236+0000","last_clean":"2023-10-09T04:11:45.248236+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:11:45.248236+0000","last_undegraded":"2023-10-09T04:11:45.248236+0000","last_fullsized":"2023-10-09T04:11:45.248236+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1467,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.888074+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:11:30.888074+0000","last_peered":"2023-10-09T04:11:30.888074+0000","last_clean":"2023-10-09T04:11:30.888074+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:11:30.888074+0000","last_undegraded":"2023-10-09T04:11:30.888074+0000","last_fullsized":"2023-10-09T04:11:30.888074+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894607+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:11:30.894607+0000","last_peered":"2023-10-09T04:11:30.894607+0000","last_clean":"2023-10-09T04:11:30.894607+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:30.894607+0000","last_undegraded":"2023-10-09T04:11:30.894607+0000","last_fullsized":"2023-10-09T04:11:30.894607+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":151,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.889116+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:11:30.889116+0000","last_peered":"2023-10-09T04:11:30.889116+0000","last_clean":"2023-10-09T04:11:30.889116+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:11:30.889116+0000","last_undegraded":"2023-10-09T04:11:30.889116+0000","last_fullsized":"2023-10-09T04:11:30.889116+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"214'5249","reported_seq":5316,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.923248+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:11:47.923248+0000","last_peered":"2023-10-09T04:11:47.923248+0000","last_clean":"2023-10-09T04:11:47.923248+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:11:47.923248+0000","last_undegraded":"2023-10-09T04:11:47.923248+0000","last_fullsized":"2023-10-09T04:11:47.923248+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5249,"log_dups_size":0,"ondisk_log_size":5249,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114753536,"num_objects":1751,"num_object_clones":0,"num_object_copies":5253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4095,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894402+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:11:30.894402+0000","last_peered":"2023-10-09T04:11:30.894402+0000","last_clean":"2023-10-09T04:11:30.894402+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:11:30.894402+0000","last_undegraded":"2023-10-09T04:11:30.894402+0000","last_fullsized":"2023-10-09T04:11:30.894402+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901956+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:11:30.901956+0000","last_peered":"2023-10-09T04:11:30.901956+0000","last_clean":"2023-10-09T04:11:30.901956+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:11:30.901956+0000","last_undegraded":"2023-10-09T04:11:30.901956+0000","last_fullsized":"2023-10-09T04:11:30.901956+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":199,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.888007+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:11:30.888007+0000","last_peered":"2023-10-09T04:11:30.888007+0000","last_clean":"2023-10-09T04:11:30.888007+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:11:30.888007+0000","last_undegraded":"2023-10-09T04:11:30.888007+0000","last_fullsized":"2023-10-09T04:11:30.888007+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"214'5294","reported_seq":5361,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.963201+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:11:47.963201+0000","last_peered":"2023-10-09T04:11:47.963201+0000","last_clean":"2023-10-09T04:11:47.963201+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:11:47.963201+0000","last_undegraded":"2023-10-09T04:11:47.963201+0000","last_fullsized":"2023-10-09T04:11:47.963201+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5294,"log_dups_size":0,"ondisk_log_size":5294,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114819072,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4132,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":27,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.896574+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:11:30.896574+0000","last_peered":"2023-10-09T04:11:30.896574+0000","last_clean":"2023-10-09T04:11:30.896574+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:11:30.896574+0000","last_undegraded":"2023-10-09T04:11:30.896574+0000","last_fullsized":"2023-10-09T04:11:30.896574+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":183,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.891422+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:11:30.891422+0000","last_peered":"2023-10-09T04:11:30.891422+0000","last_clean":"2023-10-09T04:11:30.891422+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:11:30.891422+0000","last_undegraded":"2023-10-09T04:11:30.891422+0000","last_fullsized":"2023-10-09T04:11:30.891422+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":184,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.887296+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:11:30.887296+0000","last_peered":"2023-10-09T04:11:30.887296+0000","last_clean":"2023-10-09T04:11:30.887296+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:11:30.887296+0000","last_undegraded":"2023-10-09T04:11:30.887296+0000","last_fullsized":"2023-10-09T04:11:30.887296+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"214'5089","reported_seq":5155,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.854314+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:11:47.854314+0000","last_peered":"2023-10-09T04:11:47.854314+0000","last_clean":"2023-10-09T04:11:47.854314+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:11:47.854314+0000","last_undegraded":"2023-10-09T04:11:47.854314+0000","last_fullsized":"2023-10-09T04:11:47.854314+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5089,"log_dups_size":0,"ondisk_log_size":5089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115310592,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3971,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":197,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894387+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:11:30.894387+0000","last_peered":"2023-10-09T04:11:30.894387+0000","last_clean":"2023-10-09T04:11:30.894387+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:11:30.894387+0000","last_undegraded":"2023-10-09T04:11:30.894387+0000","last_fullsized":"2023-10-09T04:11:30.894387+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":554,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886065+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:11:30.886065+0000","last_peered":"2023-10-09T04:11:30.886065+0000","last_clean":"2023-10-09T04:11:30.886065+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:11:30.886065+0000","last_undegraded":"2023-10-09T04:11:30.886065+0000","last_fullsized":"2023-10-09T04:11:30.886065+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894389+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:11:30.894389+0000","last_peered":"2023-10-09T04:11:30.894389+0000","last_clean":"2023-10-09T04:11:30.894389+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:11:30.894389+0000","last_undegraded":"2023-10-09T04:11:30.894389+0000","last_fullsized":"2023-10-09T04:11:30.894389+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901763+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:11:30.901763+0000","last_peered":"2023-10-09T04:11:30.901763+0000","last_clean":"2023-10-09T04:11:30.901763+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:11:30.901763+0000","last_undegraded":"2023-10-09T04:11:30.901763+0000","last_fullsized":"2023-10-09T04:11:30.901763+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"214'5056","reported_seq":7404,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:45.568226+0000","last_change":"2023-10-09T04:10:53.276397+0000","last_active":"2023-10-09T04:11:45.568226+0000","last_peered":"2023-10-09T04:11:45.568226+0000","last_clean":"2023-10-09T04:11:45.568226+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:11:45.568226+0000","last_undegraded":"2023-10-09T04:11:45.568226+0000","last_fullsized":"2023-10-09T04:11:45.568226+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5056,"log_dups_size":0,"ondisk_log_size":5056,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120225792,"num_objects":1834,"num_object_clones":0,"num_object_copies":5502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3940,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":194,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886018+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:11:30.886018+0000","last_peered":"2023-10-09T04:11:30.886018+0000","last_clean":"2023-10-09T04:11:30.886018+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:11:30.886018+0000","last_undegraded":"2023-10-09T04:11:30.886018+0000","last_fullsized":"2023-10-09T04:11:30.886018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":169,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921034+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:11:30.921034+0000","last_peered":"2023-10-09T04:11:30.921034+0000","last_clean":"2023-10-09T04:11:30.921034+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:11:30.921034+0000","last_undegraded":"2023-10-09T04:11:30.921034+0000","last_fullsized":"2023-10-09T04:11:30.921034+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":202,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921385+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:11:30.921385+0000","last_peered":"2023-10-09T04:11:30.921385+0000","last_clean":"2023-10-09T04:11:30.921385+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:11:30.921385+0000","last_undegraded":"2023-10-09T04:11:30.921385+0000","last_fullsized":"2023-10-09T04:11:30.921385+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"214'5305","reported_seq":5434,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:45.427707+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:11:45.427707+0000","last_peered":"2023-10-09T04:11:45.427707+0000","last_clean":"2023-10-09T04:11:45.427707+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:11:45.427707+0000","last_undegraded":"2023-10-09T04:11:45.427707+0000","last_fullsized":"2023-10-09T04:11:45.427707+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2758,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886235+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:11:30.886235+0000","last_peered":"2023-10-09T04:11:30.886235+0000","last_clean":"2023-10-09T04:11:30.886235+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:11:30.886235+0000","last_undegraded":"2023-10-09T04:11:30.886235+0000","last_fullsized":"2023-10-09T04:11:30.886235+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":183,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.888184+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:11:30.888184+0000","last_peered":"2023-10-09T04:11:30.888184+0000","last_clean":"2023-10-09T04:11:30.888184+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:11:30.888184+0000","last_undegraded":"2023-10-09T04:11:30.888184+0000","last_fullsized":"2023-10-09T04:11:30.888184+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":154,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.920997+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:11:30.920997+0000","last_peered":"2023-10-09T04:11:30.920997+0000","last_clean":"2023-10-09T04:11:30.920997+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:11:30.920997+0000","last_undegraded":"2023-10-09T04:11:30.920997+0000","last_fullsized":"2023-10-09T04:11:30.920997+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"214'5300","reported_seq":8360,"reported_epoch":214,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:11:45.661785+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:11:45.661785+0000","last_peered":"2023-10-09T04:11:45.661785+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:11:45.661785+0000","last_undegraded":"2023-10-09T04:11:45.661785+0000","last_fullsized":"2023-10-09T04:11:45.661785+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121044992,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1847,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4129,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1847}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":118,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886733+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:11:30.886733+0000","last_peered":"2023-10-09T04:11:30.886733+0000","last_clean":"2023-10-09T04:11:30.886733+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:11:30.886733+0000","last_undegraded":"2023-10-09T04:11:30.886733+0000","last_fullsized":"2023-10-09T04:11:30.886733+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":150,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.890919+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:11:30.890919+0000","last_peered":"2023-10-09T04:11:30.890919+0000","last_clean":"2023-10-09T04:11:30.890919+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:11:30.890919+0000","last_undegraded":"2023-10-09T04:11:30.890919+0000","last_fullsized":"2023-10-09T04:11:30.890919+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":144,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886200+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:11:30.886200+0000","last_peered":"2023-10-09T04:11:30.886200+0000","last_clean":"2023-10-09T04:11:30.886200+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:11:30.886200+0000","last_undegraded":"2023-10-09T04:11:30.886200+0000","last_fullsized":"2023-10-09T04:11:30.886200+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"214'5263","reported_seq":7473,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:48.635274+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:11:48.635274+0000","last_peered":"2023-10-09T04:11:48.635274+0000","last_clean":"2023-10-09T04:11:48.635274+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:11:48.635274+0000","last_undegraded":"2023-10-09T04:11:48.635274+0000","last_fullsized":"2023-10-09T04:11:48.635274+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":5263,"log_dups_size":0,"ondisk_log_size":5263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112984064,"num_objects":1724,"num_object_clones":0,"num_object_copies":5172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2786,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":142,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.886841+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:11:30.886841+0000","last_peered":"2023-10-09T04:11:30.886841+0000","last_clean":"2023-10-09T04:11:30.886841+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:11:30.886841+0000","last_undegraded":"2023-10-09T04:11:30.886841+0000","last_fullsized":"2023-10-09T04:11:30.886841+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921401+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:11:30.921401+0000","last_peered":"2023-10-09T04:11:30.921401+0000","last_clean":"2023-10-09T04:11:30.921401+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:11:30.921401+0000","last_undegraded":"2023-10-09T04:11:30.921401+0000","last_fullsized":"2023-10-09T04:11:30.921401+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901843+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:11:30.901843+0000","last_peered":"2023-10-09T04:11:30.901843+0000","last_clean":"2023-10-09T04:11:30.901843+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:11:30.901843+0000","last_undegraded":"2023-10-09T04:11:30.901843+0000","last_fullsized":"2023-10-09T04:11:30.901843+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":157,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.921261+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:11:30.921261+0000","last_peered":"2023-10-09T04:11:30.921261+0000","last_clean":"2023-10-09T04:11:30.921261+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:11:30.921261+0000","last_undegraded":"2023-10-09T04:11:30.921261+0000","last_fullsized":"2023-10-09T04:11:30.921261+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":27,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.899147+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:11:30.899147+0000","last_peered":"2023-10-09T04:11:30.899147+0000","last_clean":"2023-10-09T04:11:30.899147+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:11:30.899147+0000","last_undegraded":"2023-10-09T04:11:30.899147+0000","last_fullsized":"2023-10-09T04:11:30.899147+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"214'5136","reported_seq":5204,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:44.864724+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:11:44.864724+0000","last_peered":"2023-10-09T04:11:44.864724+0000","last_clean":"2023-10-09T04:11:44.864724+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:11:44.864724+0000","last_undegraded":"2023-10-09T04:11:44.864724+0000","last_fullsized":"2023-10-09T04:11:44.864724+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5136,"log_dups_size":0,"ondisk_log_size":5136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119406592,"num_objects":1822,"num_object_clones":0,"num_object_copies":5466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3996,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":175,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894554+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:11:30.894554+0000","last_peered":"2023-10-09T04:11:30.894554+0000","last_clean":"2023-10-09T04:11:30.894554+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:11:30.894554+0000","last_undegraded":"2023-10-09T04:11:30.894554+0000","last_fullsized":"2023-10-09T04:11:30.894554+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901983+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:11:30.901983+0000","last_peered":"2023-10-09T04:11:30.901983+0000","last_clean":"2023-10-09T04:11:30.901983+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:11:30.901983+0000","last_undegraded":"2023-10-09T04:11:30.901983+0000","last_fullsized":"2023-10-09T04:11:30.901983+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"214'5288","reported_seq":5351,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:47.942083+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:11:47.942083+0000","last_peered":"2023-10-09T04:11:47.942083+0000","last_clean":"2023-10-09T04:11:47.942083+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:11:47.942083+0000","last_undegraded":"2023-10-09T04:11:47.942083+0000","last_fullsized":"2023-10-09T04:11:47.942083+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5288,"log_dups_size":0,"ondisk_log_size":5288,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116523008,"num_objects":1778,"num_object_clones":0,"num_object_copies":5334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4126,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.894688+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:11:30.894688+0000","last_peered":"2023-10-09T04:11:30.894688+0000","last_clean":"2023-10-09T04:11:30.894688+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:11:30.894688+0000","last_undegraded":"2023-10-09T04:11:30.894688+0000","last_fullsized":"2023-10-09T04:11:30.894688+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901565+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:11:30.901565+0000","last_peered":"2023-10-09T04:11:30.901565+0000","last_clean":"2023-10-09T04:11:30.901565+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:11:30.901565+0000","last_undegraded":"2023-10-09T04:11:30.901565+0000","last_fullsized":"2023-10-09T04:11:30.901565+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":27,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.897643+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:11:30.897643+0000","last_peered":"2023-10-09T04:11:30.897643+0000","last_clean":"2023-10-09T04:11:30.897643+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:11:30.897643+0000","last_undegraded":"2023-10-09T04:11:30.897643+0000","last_fullsized":"2023-10-09T04:11:30.897643+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":27,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.896644+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:11:30.896644+0000","last_peered":"2023-10-09T04:11:30.896644+0000","last_clean":"2023-10-09T04:11:30.896644+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:11:30.896644+0000","last_undegraded":"2023-10-09T04:11:30.896644+0000","last_fullsized":"2023-10-09T04:11:30.896644+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":203,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.922801+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:11:30.922801+0000","last_peered":"2023-10-09T04:11:30.922801+0000","last_clean":"2023-10-09T04:11:30.922801+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:11:30.922801+0000","last_undegraded":"2023-10-09T04:11:30.922801+0000","last_fullsized":"2023-10-09T04:11:30.922801+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:11:30.901366+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:11:30.901366+0000","last_peered":"2023-10-09T04:11:30.901366+0000","last_clean":"2023-10-09T04:11:30.901366+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:11:30.901366+0000","last_undegraded":"2023-10-09T04:11:30.901366+0000","last_fullsized":"2023-10-09T04:11:30.901366+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":3766419480,"num_objects":57472,"num_object_clones":0,"num_object_copies":172416,"num_objects_missing_on_primary":158,"num_objects_missing":158,"num_objects_degraded":2197,"num_objects_misplaced":6927,"num_objects_unfound":0,"num_objects_dirty":57472,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122625,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10630,"num_bytes_recovered":696647680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11829129216,"data_stored":11829117000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147859,"ondisk_log_size":147859,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201511,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700000,"kb_used_data":1402712,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92045152,"statfs":{"total":95995035648,"available":94254235648,"internally_reserved":0,"allocated":1436377088,"data_stored":1435734682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":10.025,"5min":3.5390000000000001,"15min":1.583},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":126.752,"5min":126.752,"15min":126.752},"last":3.238},{"interface":"front","average":{"1min":10.521000000000001,"5min":3.609,"15min":1.6060000000000001},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":128.88399999999999,"5min":128.88399999999999,"15min":128.88399999999999},"last":0.373}]},{"osd":1,"last update":"Mon Oct 9 04:10:47 2023","interfaces":[{"interface":"back","average":{"1min":11.227,"5min":3.9630000000000001,"15min":1.736},"min":{"1min":0.28499999999999998,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":245.28700000000001,"5min":245.28700000000001,"15min":245.28700000000001},"last":4.0039999999999996},{"interface":"front","average":{"1min":11.473000000000001,"5min":3.8130000000000002,"15min":1.7470000000000001},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":244.626,"5min":244.626,"15min":244.626},"last":7.0919999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.8799999999999999,"15min":1.268},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4009999999999998,"5min":13.887,"15min":58.615000000000002},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.401,"5min":2.9279999999999999,"15min":1.6850000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.3109999999999999,"5min":99.372,"15min":99.372},"last":0.79700000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:10:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0139999999999998,"15min":1.534},"min":{"1min":0.34399999999999997,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":6.1150000000000002,"5min":99.134,"15min":99.134},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.329,"5min":3.1459999999999999,"15min":1.611},"min":{"1min":0.39100000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.1310000000000002,"5min":99.117999999999995,"15min":99.117999999999995},"last":0.96299999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.079,"15min":0.69699999999999995},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9790000000000001,"5min":16.195,"15min":16.195},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.442,"5min":1.0109999999999999,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.052,"5min":12.442,"15min":12.442},"last":0.29499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.77800000000000002,"15min":0.59699999999999998},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81999999999999995,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.311},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.873,"15min":0.67800000000000005},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0550000000000002,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.35699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.91300000000000003,"15min":0.67600000000000005},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.99399999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.03,"15min":0.745},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":2.0819999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.38300000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463148,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1822164,"kb_used_data":1520716,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91922988,"statfs":{"total":95995035648,"available":94129139712,"internally_reserved":0,"allocated":1557213184,"data_stored":1556582621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7850000000000001,"5min":4.2990000000000004,"15min":1.8460000000000001},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":60.295000000000002,"5min":78.983999999999995,"15min":78.983999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":6.8220000000000001,"5min":4.8659999999999997,"15min":2.0830000000000002},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":60.716000000000001,"5min":78.998000000000005,"15min":78.998000000000005},"last":0.32100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.71,"15min":1.619},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":62.698,"5min":62.698,"15min":62.698},"last":1.575},{"interface":"front","average":{"1min":3.8420000000000001,"5min":3.895,"15min":1.7010000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.530999999999999,"5min":63.530999999999999,"15min":63.530999999999999},"last":1.4950000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":7.6840000000000002,"5min":3.8420000000000001,"15min":1.8420000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":77.953000000000003,"5min":77.953000000000003,"15min":77.953000000000003},"last":1.0149999999999999},{"interface":"front","average":{"1min":7.7320000000000002,"5min":3.875,"15min":1.8879999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":78.019000000000005,"5min":78.019000000000005,"15min":78.019000000000005},"last":1.0329999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":3.0720000000000001,"5min":4.0350000000000001,"15min":1.911},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":53.497999999999998,"5min":57.509999999999998,"15min":57.509999999999998},"last":1.5900000000000001},{"interface":"front","average":{"1min":3.4910000000000001,"5min":4.3529999999999998,"15min":2.0219999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":53.460999999999999,"5min":57.453000000000003,"15min":57.453000000000003},"last":1.8939999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.51700000000000002,"15min":0.48799999999999999},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.264,"5min":2.6909999999999998,"15min":2.6909999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.84699999999999998,"15min":0.622},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.363,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.224,"15min":0.75900000000000001},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.2849999999999999,"5min":54.773000000000003,"15min":54.773000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.3899999999999999,"15min":0.85199999999999998},"min":{"1min":0.32500000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4159999999999999,"5min":54.853999999999999,"15min":54.853999999999999},"last":0.38200000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.69999999999999996,"15min":0.61499999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":2.903,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.80900000000000005,"15min":0.68500000000000005},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.28699999999999998},"max":{"1min":2.835,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.41299999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089457,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1739596,"kb_used_data":1527556,"kb_used_omap":9,"kb_used_meta":212022,"kb_avail":92005556,"statfs":{"total":95995035648,"available":94213689344,"internally_reserved":0,"allocated":1564217344,"data_stored":1563359001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":217111529},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":3.3450000000000002,"15min":1.4390000000000001},"min":{"1min":0.20899999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":14.833,"5min":85.977999999999994,"15min":85.977999999999994},"last":5.5510000000000002},{"interface":"front","average":{"1min":1.722,"5min":4.6120000000000001,"15min":1.8819999999999999},"min":{"1min":0.22500000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.664,"5min":99.831999999999994,"15min":99.831999999999994},"last":5.774}]},{"osd":2,"last update":"Mon Oct 9 04:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":3.6469999999999998,"15min":1.5369999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.6,"5min":121.54600000000001,"15min":121.54600000000001},"last":5.6470000000000002},{"interface":"front","average":{"1min":2.3010000000000002,"5min":3.7919999999999998,"15min":1.702},"min":{"1min":0.23200000000000001,"5min":0.186,"15min":0.186},"max":{"1min":11.052,"5min":99.899000000000001,"15min":99.899000000000001},"last":5.8040000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":4.8959999999999999,"15min":1.98},"min":{"1min":0.29999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":138.87299999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":5.7060000000000004},{"interface":"front","average":{"1min":10.010999999999999,"5min":6.2439999999999998,"15min":2.4169999999999998},"min":{"1min":0.30599999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":138.49799999999999,"5min":138.49799999999999,"15min":138.49799999999999},"last":12.821999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.5099999999999998,"15min":1.343},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.5219999999999998,"5min":99.989999999999995,"15min":99.989999999999995},"last":12.747},{"interface":"front","average":{"1min":1.3220000000000001,"5min":2.5470000000000002,"15min":1.351},"min":{"1min":0.28799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3330000000000002,"5min":100.249,"15min":100.249},"last":1.0609999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":2.4350000000000001,"15min":1.3360000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4820000000000002,"5min":99.933999999999997,"15min":99.933999999999997},"last":12.696999999999999},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.798,"15min":1.45},"min":{"1min":0.38300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.2789999999999999,"5min":100.34099999999999,"15min":100.34099999999999},"last":0.97899999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":3.1110000000000002,"15min":1.5549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4509999999999996,"5min":100.309,"15min":100.309},"last":1.038},{"interface":"front","average":{"1min":0.96799999999999997,"5min":2.7450000000000001,"15min":1.4550000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.4020000000000001,"5min":100.143,"15min":100.143},"last":5.6230000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":2.988,"15min":1.534},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.2210000000000001,"5min":100.214,"15min":100.214},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.7450000000000001,"15min":1.4750000000000001},"min":{"1min":0.374,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":3.335,"5min":100.113,"15min":100.113},"last":5.5919999999999996}]}]},{"osd":0,"up_from":84,"seq":360777252983,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2161568,"kb_used_data":1935576,"kb_used_omap":28,"kb_used_meta":225763,"kb_avail":91583584,"statfs":{"total":95995035648,"available":93781590016,"internally_reserved":0,"allocated":1982029824,"data_stored":1981150464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":231182160},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":3.6059999999999999,"15min":1.982},"min":{"1min":0.184,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":12.698,"5min":103.996,"15min":118.48},"last":9.9619999999999997},{"interface":"front","average":{"1min":3.5910000000000002,"5min":4.1959999999999997,"15min":2.2080000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":12.632999999999999,"5min":106.34399999999999,"15min":118.384},"last":4.0190000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:11:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":4.5990000000000002,"15min":1.9419999999999999},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.8819999999999997,"5min":118.54000000000001,"15min":118.54000000000001},"last":4.7800000000000002},{"interface":"front","average":{"1min":1.736,"5min":5.0780000000000003,"15min":2.089},"min":{"1min":0.21199999999999999,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":8.2300000000000004,"5min":118.727,"15min":118.727},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":5.9320000000000004,"15min":3.601},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.161,"5min":156.488,"15min":156.488},"last":9.9169999999999998},{"interface":"front","average":{"1min":4.1909999999999998,"5min":5.8979999999999997,"15min":3.6890000000000001},"min":{"1min":0.40699999999999997,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":9.6010000000000009,"5min":156.50700000000001,"15min":156.50700000000001},"last":9.6010000000000009}]},{"osd":4,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.3540000000000001,"15min":1.625},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8310000000000004,"5min":59.439999999999998,"15min":118.633},"last":3.9590000000000001},{"interface":"front","average":{"1min":1.151,"5min":2.9889999999999999,"15min":1.897},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.593,"5min":59.453000000000003,"15min":118.913},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":4.548,"15min":2.4169999999999998},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.6310000000000002,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.302,"5min":4.2830000000000004,"15min":2.3039999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.8919999999999995,"5min":182.66300000000001,"15min":182.66300000000001},"last":2.9980000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":8.0259999999999998,"5min":3.976,"15min":2.2109999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":137.36099999999999,"5min":137.36099999999999,"15min":137.36099999999999},"last":2.972},{"interface":"front","average":{"1min":7.3899999999999997,"5min":3.5800000000000001,"15min":2.0670000000000002},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":136.88999999999999,"5min":136.88999999999999,"15min":136.88999999999999},"last":3.9870000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":7.2210000000000001,"5min":5.7480000000000002,"15min":2.8370000000000002},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":121.324,"5min":185.024,"15min":185.024},"last":2.9399999999999999},{"interface":"front","average":{"1min":2.206,"5min":4.21,"15min":2.3180000000000001},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":6.6280000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":4.0030000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925930,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1445048,"kb_used_data":1129520,"kb_used_omap":81,"kb_used_meta":315438,"kb_avail":92300104,"statfs":{"total":95995035648,"available":94515306496,"internally_reserved":0,"allocated":1156628480,"data_stored":1155787984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":323008730},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.4550000000000001,"15min":1.1419999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.413,"5min":12.284000000000001,"15min":12.284000000000001},"last":12.298},{"interface":"front","average":{"1min":2.5840000000000001,"5min":3.2650000000000001,"15min":1.5189999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":11.356999999999999,"5min":45.979999999999997,"15min":45.979999999999997},"last":1.621}]},{"osd":1,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":3.6539999999999999,"15min":1.8160000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":14.99,"5min":53.695,"15min":68.009},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.5979999999999999,"5min":3.8730000000000002,"15min":1.919},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":16.597000000000001,"5min":53.439,"15min":66.975999999999999},"last":1.8320000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":2.2749999999999999,"15min":1.173},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2409999999999997,"5min":12.77,"15min":27.562000000000001},"last":12.212999999999999},{"interface":"front","average":{"1min":1.746,"5min":3.0459999999999998,"15min":1.405},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.952,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.26000000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":2.419,"15min":1.3280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9209999999999998,"5min":46.075000000000003,"15min":46.075000000000003},"last":1.7789999999999999},{"interface":"front","average":{"1min":1.524,"5min":1.7450000000000001,"15min":1.0329999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.5809999999999995,"5min":14.305,"15min":14.305},"last":6.96}]},{"osd":5,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.601,"15min":1.373},"min":{"1min":0.373,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":26.667000000000002,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.7999999999999998,"5min":2.8620000000000001,"15min":1.5},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":26.654,"5min":46.106000000000002,"15min":46.106000000000002},"last":1.804}]},{"osd":6,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0199999999999996,"5min":2.9830000000000001,"15min":1.4690000000000001},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":114.14,"5min":114.14,"15min":114.14},"last":7.024},{"interface":"front","average":{"1min":7.8810000000000002,"5min":3.6789999999999998,"15min":1.7549999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":114.123,"5min":114.123,"15min":114.123},"last":0.43099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.1690000000000005,"5min":3.4550000000000001,"15min":1.651},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":117.32599999999999,"5min":117.32599999999999,"15min":117.32599999999999},"last":6.9980000000000002},{"interface":"front","average":{"1min":8.1639999999999997,"5min":3.9079999999999999,"15min":1.8540000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":117.28700000000001,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.45900000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762404,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2612508,"kb_used_data":2224020,"kb_used_omap":7,"kb_used_meta":388472,"kb_avail":91132644,"statfs":{"total":95995035648,"available":93319827456,"internally_reserved":0,"allocated":2277396480,"data_stored":2276580807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":397795944},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":7.7279999999999998,"5min":5.4539999999999997,"15min":3.0920000000000001},"min":{"1min":0.25600000000000001,"5min":0.189,"15min":0.185},"max":{"1min":100.512,"5min":100.512,"15min":106.22},"last":11.944000000000001},{"interface":"front","average":{"1min":8.4130000000000003,"5min":5.7060000000000004,"15min":3.1930000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":101.733,"5min":101.733,"15min":106.986},"last":7.7519999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.1470000000000002,"15min":2.069},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.19800000000000001},"max":{"1min":57.055999999999997,"5min":64.677000000000007,"15min":106.121},"last":0.253},{"interface":"front","average":{"1min":7.2649999999999997,"5min":4.7859999999999996,"15min":1.8660000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":57.088999999999999,"5min":65.015000000000001,"15min":65.015000000000001},"last":1.0049999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":3.5609999999999999,"15min":1.905},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":4.593,"5min":85.974000000000004,"15min":106.105},"last":0.48799999999999999},{"interface":"front","average":{"1min":6.7130000000000001,"5min":4.79,"15min":2.3079999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":98.765000000000001,"5min":98.765000000000001,"15min":106.70399999999999},"last":0.91800000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":2.8090000000000002,"15min":1.7869999999999999},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.308},"max":{"1min":5.4740000000000002,"5min":37.652000000000001,"15min":106.85899999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.353,"5min":2.5350000000000001,"15min":1.266},"min":{"1min":0.32500000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.2409999999999997,"5min":37.442,"15min":37.442},"last":0.36299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":2.2330000000000001,"15min":1.1559999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3150000000000004,"5min":12.819000000000001,"15min":12.819000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.1539999999999999,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.0789999999999997,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.53000000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":2.004,"15min":1.8260000000000001},"min":{"1min":0.35299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1100000000000003,"5min":13.255000000000001,"15min":106.036},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.528,"5min":2.3759999999999999,"15min":1.6619999999999999},"min":{"1min":0.40400000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.4050000000000002,"5min":15.622,"15min":106.741},"last":0.55400000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":2.5009999999999999,"15min":1.7090000000000001},"min":{"1min":0.38200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.5069999999999997,"5min":15.685,"15min":106.82599999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.224,"5min":2.2429999999999999,"15min":1.5980000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.266,"5min":13.228999999999999,"15min":106.084},"last":0.39700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986420,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1373076,"kb_used_data":1067596,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92372076,"statfs":{"total":95995035648,"available":94589005824,"internally_reserved":0,"allocated":1093218304,"data_stored":1092587307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":2.8290000000000002,"15min":1.3500000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":1.726,"5min":63.128,"15min":63.128},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.5110000000000001,"15min":1.3149999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1600000000000001,"5min":62.645000000000003,"15min":62.645000000000003},"last":1.147}]},{"osd":1,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":2.7330000000000001,"15min":1.284},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.1189999999999998,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.372},{"interface":"front","average":{"1min":1.321,"5min":2.714,"15min":1.276},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.9260000000000002,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.499}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8380000000000001,"15min":1.0900000000000001},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.698,"5min":19.356000000000002,"15min":19.356000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.125,"5min":1.845,"15min":1.042},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7109999999999999,"5min":19.324000000000002,"15min":19.324000000000002},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.863,"15min":1.0169999999999999},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.5629999999999997,"5min":14.694000000000001,"15min":14.694000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.927,"5min":2.1320000000000001,"15min":1.125},"min":{"1min":0.48999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":7.4909999999999997,"5min":10.476000000000001,"15min":10.476000000000001},"last":0.66900000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.70999999999999996,"15min":0.57599999999999996},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3100000000000001,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92600000000000005,"15min":0.71599999999999997},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.7290000000000001,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.46300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.0920000000000001,"15min":0.72699999999999998},"min":{"1min":0.29399999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.6180000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.871,"15min":0.65400000000000003},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.669,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.96999999999999997,"15min":0.69899999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.4890000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.004,"15min":0.76100000000000001},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.306,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.40000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724784,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1132292,"kb_used_data":830908,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92612860,"statfs":{"total":95995035648,"available":94835568640,"internally_reserved":0,"allocated":850849792,"data_stored":850230865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":6.391,"5min":3.4849999999999999,"15min":1.577},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":83.329999999999998,"5min":83.329999999999998,"15min":83.329999999999998},"last":2.621},{"interface":"front","average":{"1min":7.7030000000000003,"5min":3.778,"15min":1.6990000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":85.451999999999998,"5min":85.451999999999998,"15min":85.451999999999998},"last":2.7949999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":3.2040000000000002,"15min":1.446},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.4980000000000002,"5min":91.212999999999994,"15min":91.212999999999994},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.252,"5min":3.75,"15min":1.6439999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.9460000000000002,"5min":91.254000000000005,"15min":91.254000000000005},"last":2.5550000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":3.0880000000000001,"15min":1.524},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.2959999999999998,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.968,"5min":3.2050000000000001,"15min":1.518},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.6929999999999996,"5min":61.154000000000003,"15min":61.154000000000003},"last":2.444}]},{"osd":3,"last update":"Mon Oct 9 04:10:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":3.04,"15min":1.657},"min":{"1min":0.35099999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.1699999999999999,"5min":91.834999999999994,"15min":91.834999999999994},"last":2.4809999999999999},{"interface":"front","average":{"1min":1.4159999999999999,"5min":3.0760000000000001,"15min":1.6819999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.1100000000000003,"5min":91.802000000000007,"15min":91.802000000000007},"last":2.468}]},{"osd":4,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.88900000000000001,"15min":0.65700000000000003},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.253},"max":{"1min":3.2440000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.251},{"interface":"front","average":{"1min":0.625,"5min":0.95599999999999996,"15min":0.71799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26600000000000001},"max":{"1min":3.4420000000000002,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.27700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.91400000000000003,"15min":0.66100000000000003},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.4059999999999999,"5min":6.8369999999999997,"15min":6.8369999999999997},"last":0.372},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.88500000000000001,"15min":0.67500000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32400000000000001},"max":{"1min":1.4630000000000001,"5min":5.7670000000000003,"15min":5.7670000000000003},"last":0.38900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.98999999999999999,"15min":0.78300000000000003},"min":{"1min":0.307,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2450000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90300000000000002,"15min":0.80300000000000005},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":1.8220000000000001,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.35599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1965035520,"data_stored":1965031448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1559171072,"data_stored":1559167000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1144127488,"data_stored":1144127488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2259156992,"data_stored":2259156992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1082916864,"data_stored":1082916864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":839909376,"data_stored":839909376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1552814080,"data_stored":1552810008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1425997824,"data_stored":1425997824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:11:52.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:52.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:52.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:52.353 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:52.385 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:52 smithi163 ceph-mon[213841]: pgmap v558: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 2197/167214 objects degraded (1.314%); 6590/167214 objects misplaced (3.941%); 793 KiB/s, 12 objects/s recovering 2023-10-09T04:11:52.386 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/172488 objects degraded (1.274%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:52.386 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:52 smithi163 ceph-mon[221067]: pgmap v558: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 2197/167214 objects degraded (1.314%); 6590/167214 objects misplaced (3.941%); 793 KiB/s, 12 objects/s recovering 2023-10-09T04:11:52.386 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/172488 objects degraded (1.274%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:52.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:52.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:52.512 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:52.607 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:52.624 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:52 smithi181 ceph-mon[196979]: pgmap v558: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 2197/167214 objects degraded (1.314%); 6590/167214 objects misplaced (3.941%); 793 KiB/s, 12 objects/s recovering 2023-10-09T04:11:52.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/172488 objects degraded (1.274%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:52.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:52.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:52.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118445 6.10 6:0cc09dea:::benchmark_data_smithi163_287034_object10182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:11:52.980 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.59999841899997, 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1585794680000001, 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118445 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:11:52.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375077+0000", 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.993e-06 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375083+0000", 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.409e-06 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375088+0000", 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6820000000000002e-06 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375142+0000", 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3881000000000001e-05 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:52.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375152+0000", 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0315e-05 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328144+0000", 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95299157199999995 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328159+0000", 2023-10-09T04:11:52.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5143999999999999e-05 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.470403+0000", 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1422438539999999 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.471213+0000", 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00081007099999999999 2023-10-09T04:11:52.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.524358+0000", 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.053145539999999998 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.524485+0000", 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012661700000000001 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.524664+0000", 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017896200000000001 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528975+0000", 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043109400000000001 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:52.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.530404+0000", 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014287880000000001 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531875+0000", 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014716200000000001 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531932+0000", 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7028999999999999e-05 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.533654+0000", 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017214610000000001 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:52.990 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118476 6.10 6:09db8b21:::benchmark_data_smithi163_287034_object10213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.57193733600002, 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1736697060000001, 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:52.991 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118476 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403138+0000", 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0920000000000002e-06 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403145+0000", 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0759999999999997e-06 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403150+0000", 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3800000000000002e-06 2023-10-09T04:11:52.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.404136+0000", 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00098518299999999994 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.404146+0000", 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0443e-05 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328185+0000", 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.92403893599999998 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328196+0000", 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0682999999999999e-05 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.526057+0000", 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.197861617 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.526773+0000", 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00071598699999999996 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.538943+0000", 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012170006000000001 2023-10-09T04:11:52.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.539024+0000", 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1154000000000003e-05 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.539117+0000", 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2702999999999997e-05 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.544254+0000", 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051368059999999998 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.575431+0000", 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031177375 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:52.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.575473+0000", 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1443000000000003e-05 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.575512+0000", 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9471000000000002e-05 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576805+0000", 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00129284 2023-10-09T04:11:53.000 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118507 6.10 6:0ec16c38:::benchmark_data_smithi163_287034_object10244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.433804+0000", 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.54126909199999, 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1442500999999998, 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.001 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118507 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433804+0000", 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433804+0000", 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433803+0000", 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433806+0000", 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6350000000000002e-06 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433812+0000", 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9490000000000001e-06 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433817+0000", 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4620000000000004e-06 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.435949+0000", 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002131853 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.435958+0000", 2023-10-09T04:11:53.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8629999999999997e-06 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328221+0000", 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.89226339099999996 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328233+0000", 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1858000000000001e-05 2023-10-09T04:11:53.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.526823+0000", 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.198589455 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.527446+0000", 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062304899999999998 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.542998+0000", 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015552237 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.543076+0000", 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8610999999999997e-05 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.543171+0000", 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4475999999999995e-05 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552461+0000", 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092900210000000007 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.009 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576876+0000", 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024414754 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576913+0000", 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6848999999999999e-05 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576947+0000", 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4910000000000003e-05 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578054+0000", 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001106201 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118571 6.10 6:0880eece:::benchmark_data_smithi163_287034_object10308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.494993+0000", 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.48007988699999, 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0842064360000001, 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118571 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.012 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494993+0000", 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494993+0000", 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494991+0000", 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494998+0000", 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7440000000000001e-06 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495013+0000", 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4965e-05 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495021+0000", 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8159999999999994e-06 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495069+0000", 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8427999999999998e-05 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495077+0000", 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8250000000000005e-06 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328259+0000", 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.83318210100000001 2023-10-09T04:11:53.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328270+0000", 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0679000000000001e-05 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.527503+0000", 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992328809999999 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528252+0000", 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00074927999999999995 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.547348+0000", 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019096418 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.547427+0000", 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8205000000000001e-05 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.547521+0000", 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4179999999999996e-05 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.556222+0000", 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087014689999999999 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578120+0000", 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021897371999999998 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578158+0000", 2023-10-09T04:11:53.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8798999999999999e-05 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578193+0000", 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4233999999999998e-05 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579199+0000", 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010065149999999999 2023-10-09T04:11:53.021 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118621 6.10 6:0b7ca3a2:::benchmark_data_smithi163_287034_object10358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.684903+0000", 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.290169805, 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.895705559, 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.022 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118621 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684903+0000", 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684903+0000", 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684901+0000", 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684909+0000", 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5310000000000004e-06 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684921+0000", 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2441e-05 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684929+0000", 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1200000000000002e-06 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684983+0000", 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4024999999999999e-05 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.026 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684994+0000", 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1004e-05 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328294+0000", 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.64329977900000002 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328305+0000", 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0750999999999999e-05 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528308+0000", 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2000031630000001 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528949+0000", 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00064121999999999996 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552514+0000", 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023565136 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552642+0000", 2023-10-09T04:11:53.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012748800000000001 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552829+0000", 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000187472 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.559228+0000", 2023-10-09T04:11:53.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0063989119999999997 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579251+0000", 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020022604999999999 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579295+0000", 2023-10-09T04:11:53.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4267999999999997e-05 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579360+0000", 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4722999999999997e-05 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580608+0000", 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001248664 2023-10-09T04:11:53.032 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118638 6.10 6:0fa2b777:::benchmark_data_smithi163_287034_object10375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.701481+0000", 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.27359130000002, 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8804589630000001, 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.033 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118638 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701481+0000", 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701481+0000", 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701480+0000", 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701487+0000", 2023-10-09T04:11:53.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0879999999999997e-06 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701503+0000", 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6024000000000001e-05 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701511+0000", 2023-10-09T04:11:53.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.106e-06 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701592+0000", 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.072e-05 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701602+0000", 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0186999999999999e-05 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328348+0000", 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.626745777 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328356+0000", 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1200000000000002e-06 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.529034+0000", 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.200677835 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.529558+0000", 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00052381699999999999 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.558015+0000", 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028457805999999999 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.558099+0000", 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3707000000000003e-05 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.558198+0000", 2023-10-09T04:11:53.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9327999999999995e-05 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.563233+0000", 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050347070000000002 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580680+0000", 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017446990999999999 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580738+0000", 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8322000000000002e-05 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580798+0000", 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9917000000000002e-05 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.042 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.581940+0000", 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011419970000000001 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118657 6.10 6:0f9ab685:::benchmark_data_smithi163_287034_object10394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:11:53.043 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.25559229100003, 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8638215659999999, 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118657 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719482+0000", 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4830000000000002e-06 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719488+0000", 2023-10-09T04:11:53.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.066e-06 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719493+0000", 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7650000000000001e-06 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.721632+0000", 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021384300000000002 2023-10-09T04:11:53.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.721641+0000", 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6739999999999992e-06 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328375+0000", 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.60673376999999995 2023-10-09T04:11:53.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328382+0000", 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.153e-06 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.529613+0000", 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2012305969999999 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.530319+0000", 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070645599999999995 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.562059+0000", 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031739648000000002 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.050 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.562132+0000", 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2917000000000007e-05 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.562242+0000", 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011035300000000001 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.567349+0000", 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005106836 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.582003+0000", 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014653875 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.582060+0000", 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6601999999999998e-05 2023-10-09T04:11:53.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.582113+0000", 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3189999999999997e-05 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583302+0000", 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011892000000000001 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.053 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118725 6.10 6:09136b91:::benchmark_data_smithi163_287034_object10462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.789819+0000", 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.18525372800002, 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.794865508, 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.054 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118725 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789819+0000", 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789819+0000", 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789817+0000", 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789825+0000", 2023-10-09T04:11:53.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1850000000000004e-06 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789837+0000", 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1620999999999999e-05 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789852+0000", 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5384999999999999e-05 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789921+0000", 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8412999999999997e-05 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789930+0000", 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1169999999999993e-06 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328400+0000", 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.53847029599999996 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328408+0000", 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.525e-06 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.530450+0000", 2023-10-09T04:11:53.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2020422900000001 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531212+0000", 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00076219199999999997 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.566185+0000", 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034972681999999998 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.566265+0000", 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0339000000000004e-05 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.566377+0000", 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111296 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.571433+0000", 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050563200000000004 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583380+0000", 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011947157999999999 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583434+0000", 2023-10-09T04:11:53.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3634000000000002e-05 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583464+0000", 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0400999999999999e-05 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584684+0000", 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001220308 2023-10-09T04:11:53.063 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118727 6.10 6:0e812dc0:::benchmark_data_smithi163_287034_object10464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.795748+0000", 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.17932483999999, 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7904910080000001, 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.064 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118727 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795748+0000", 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795748+0000", 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795746+0000", 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795756+0000", 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9990000000000003e-06 2023-10-09T04:11:53.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795768+0000", 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2173e-05 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795776+0000", 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2729999999999997e-06 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795834+0000", 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8752999999999998e-05 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795847+0000", 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2395000000000001e-05 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328426+0000", 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.53257936699999997 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328433+0000", 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8689999999999998e-06 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531246+0000", 2023-10-09T04:11:53.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.202813353 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531805+0000", 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055812899999999998 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.570272+0000", 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.038467012000000002 2023-10-09T04:11:53.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.570363+0000", 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0966999999999998e-05 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.570457+0000", 2023-10-09T04:11:53.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4271999999999995e-05 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576817+0000", 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0063599110000000002 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584737+0000", 2023-10-09T04:11:53.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079197989999999999 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584794+0000", 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7494999999999998e-05 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584847+0000", 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2562000000000003e-05 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586239+0000", 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013921890000000001 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.074 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118738 6.10 6:0d6f2f89:::benchmark_data_smithi163_287034_object10475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.803706+0000", 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 310.17136685499997, 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7839791599999999, 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.075 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118738 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803706+0000", 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803706+0000", 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803705+0000", 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:11:53.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803711+0000", 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7169999999999996e-06 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803722+0000", 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1712000000000001e-05 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803730+0000", 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7000000000000008e-06 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803790+0000", 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9763999999999999e-05 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803800+0000", 2023-10-09T04:11:53.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9850000000000001e-06 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328453+0000", 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.52465375999999997 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328460+0000", 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9609999999999999e-06 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.533713+0000", 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2052529590000001 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.534961+0000", 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012478809999999999 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.574203+0000", 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039241693000000001 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.574281+0000", 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7752999999999999e-05 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.574387+0000", 2023-10-09T04:11:53.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106183 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578065+0000", 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036780369999999999 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586286+0000", 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082206889999999998 2023-10-09T04:11:53.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586342+0000", 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6122999999999998e-05 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586390+0000", 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8096999999999997e-05 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.587685+0000", 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012951200000000001 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80549 7.12 7:4a26220b:::benchmark_data_smithi163_346082_object80548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 74.504657323999993, 2023-10-09T04:11:53.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5311827440000001, 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80549 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:53.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470446+0000", 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0809000000000003e-05 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470453+0000", 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6939999999999995e-06 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470459+0000", 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8830000000000003e-06 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470525+0000", 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5624999999999996e-05 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470537+0000", 2023-10-09T04:11:53.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1918e-05 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.968283+0000", 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4977461619999999 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.972146+0000", 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038629369999999999 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.978159+0000", 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060134689999999996 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.994408+0000", 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016248729999999999 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.995092+0000", 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068414799999999998 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.995313+0000", 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000220638 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.996444+0000", 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011307999999999999 2023-10-09T04:11:53.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.001598+0000", 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051543429999999996 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80579 7.12 7:4e07a344:::benchmark_data_smithi163_346082_object80578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 74.454828065000001, 2023-10-09T04:11:53.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4902949560000001, 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80579 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.094 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520243+0000", 2023-10-09T04:11:53.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520271+0000", 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8158999999999999e-05 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520288+0000", 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7629999999999999e-05 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523866+0000", 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035770730000000001 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524028+0000", 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016236000000000001 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524050+0000", 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1882000000000002e-05 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.982571+0000", 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.458521492 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.985387+0000", 2023-10-09T04:11:53.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028160400000000001 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.989941+0000", 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045534929999999996 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.001841+0000", 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011900571 2023-10-09T04:11:53.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.002750+0000", 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090909300000000003 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.003526+0000", 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077512299999999996 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.004995+0000", 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001469143 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.010540+0000", 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055447580000000003 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80582 7.12 7:4a340bf1:::benchmark_data_smithi163_346082_object80581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 74.451154556000006, 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5356164219999999, 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.102 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80582 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523917+0000", 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523945+0000", 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7841999999999999e-05 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523957+0000", 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2449000000000001e-05 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523974+0000", 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7101000000000001e-05 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524082+0000", 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107395 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524090+0000", 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8189999999999998e-06 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.010809+0000", 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4867180810000002 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.014231+0000", 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003422043 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.019765+0000", 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055343019999999996 2023-10-09T04:11:53.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.037504+0000", 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017739296000000002 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.038374+0000", 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086940900000000002 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.052524+0000", 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014150563999999999 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.053699+0000", 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011743890000000001 2023-10-09T04:11:53.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.059534+0000", 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058359709999999997 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80583 7.12 7:4f777d92:::benchmark_data_smithi163_346082_object80582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:11:53.110 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 74.450764028999998, 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5426957880000001, 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80583 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524308+0000", 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524363+0000", 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5349999999999997e-05 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524377+0000", 2023-10-09T04:11:53.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4075e-05 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524384+0000", 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1230000000000002e-06 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524440+0000", 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5284999999999997e-05 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524453+0000", 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3614999999999999e-05 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.024073+0000", 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.499619434 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.027624+0000", 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003551102 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.032819+0000", 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051955509999999996 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.039280+0000", 2023-10-09T04:11:53.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064610379999999997 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.052274+0000", 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012993754 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.059828+0000", 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075536780000000003 2023-10-09T04:11:53.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.061217+0000", 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001389033 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.067004+0000", 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005787665 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.118 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80604 7.12 7:4ae32c24:::benchmark_data_smithi163_346082_object80603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.545411482999995, 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.646247169, 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.119 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80604 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:11:53.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429660+0000", 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429684+0000", 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4735e-05 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429700+0000", 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6118999999999999e-05 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429710+0000", 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4110000000000002e-06 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.039654+0000", 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6099443359999999 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.042687+0000", 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030327050000000001 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.048100+0000", 2023-10-09T04:11:53.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054131359999999998 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.067276+0000", 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019175580000000001 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.068101+0000", 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082490899999999997 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.068965+0000", 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086460499999999995 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.070415+0000", 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001449547 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.075908+0000", 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054935929999999997 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80621 7.12 7:4a7d6f10:::benchmark_data_smithi163_346082_object80620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.522519396000007, 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6714043699999999, 2023-10-09T04:11:53.126 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80621 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452552+0000", 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452577+0000", 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5196000000000001e-05 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452588+0000", 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1482e-05 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452595+0000", 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8430000000000004e-06 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.076258+0000", 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6236622060000001 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.079917+0000", 2023-10-09T04:11:53.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003659425 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.085706+0000", 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057886170000000002 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.115020+0000", 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.029313949999999998 2023-10-09T04:11:53.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.115996+0000", 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097686999999999991 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.116902+0000", 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090546300000000001 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.118150+0000", 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001248377 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.123958+0000", 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058073669999999999 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.133 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80659 7.12 7:4da2b418:::benchmark_data_smithi163_346082_object80658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.479621713, 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.638010832, 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.134 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80659 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495449+0000", 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495472+0000", 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2648999999999999e-05 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495487+0000", 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4714e-05 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495494+0000", 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6179999999999997e-06 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.089728+0000", 2023-10-09T04:11:53.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5942332260000001 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.092886+0000", 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031585609999999998 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.098139+0000", 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052528790000000002 2023-10-09T04:11:53.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.124279+0000", 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.026139367 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.125465+0000", 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011861739999999999 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.126393+0000", 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092876999999999999 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.127602+0000", 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012081450000000001 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.133462+0000", 2023-10-09T04:11:53.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058601410000000001 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80699 7.12 7:49a7773c:::benchmark_data_smithi163_346082_object80698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.433010803000002, 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.601076436, 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.141 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80699 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542061+0000", 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542082+0000", 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0676999999999999e-05 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542089+0000", 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1969999999999999e-06 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542095+0000", 2023-10-09T04:11:53.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8889999999999999e-06 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.101752+0000", 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.559657235 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.105743+0000", 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039902699999999998 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.111032+0000", 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052895010000000003 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.124581+0000", 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013548701999999999 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.134215+0000", 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096345360000000008 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.135040+0000", 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082435799999999997 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.136244+0000", 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00120403 2023-10-09T04:11:53.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.143138+0000", 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006894608 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80726 7.12 7:4d882541:::benchmark_data_smithi163_346082_object80725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:11:53.148 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.396247603999996, 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6025676880000002, 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80726 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.149 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578824+0000", 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578846+0000", 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2328e-05 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578858+0000", 2023-10-09T04:11:53.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1749999999999999e-05 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578866+0000", 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.542e-06 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.143524+0000", 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.564657902 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.146910+0000", 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033861049999999999 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.152321+0000", 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054116600000000004 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.172638+0000", 2023-10-09T04:11:53.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020316186999999999 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.172924+0000", 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028628699999999998 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.173994+0000", 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010702839999999999 2023-10-09T04:11:53.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.175304+0000", 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013095439999999999 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.181393+0000", 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060889680000000002 2023-10-09T04:11:53.155 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80738 7.12 7:4f007b37:::benchmark_data_smithi163_346082_object80737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.373565716000002, 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5902670159999999, 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:11:53.156 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80738 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:11:53.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601505+0000", 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601561+0000", 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6643e-05 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601585+0000", 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3541e-05 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601597+0000", 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2396e-05 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.156776+0000", 2023-10-09T04:11:53.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5551789820000002 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.160320+0000", 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035439960000000002 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.165689+0000", 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053691839999999999 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.173228+0000", 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075388089999999996 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.182616+0000", 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093880849999999991 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:11:53.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.183537+0000", 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092063400000000004 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.185023+0000", 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014860819999999999 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.191774+0000", 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067509279999999998 2023-10-09T04:11:53.162 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.163 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.163 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.163 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:11:53.163 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:11:53.163 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:11:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:53.164 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:53.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:53.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:53.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:53.292 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:53.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:53.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:53.363 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:11:53.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:53.485 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:53.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:53.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:53.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:53.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:53.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3273332765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:53.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:53 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:53.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3273332765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:53.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:53 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:53.736 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3273332765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:53 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:11:54.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:54.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:54.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:54.305 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:54.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:54.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:54.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:54.462 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:54.498 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:54.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:54.560 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:54 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:11:54.560 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:54.560 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:54 smithi181 ceph-mon[196979]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T04:11:54.560 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:54 smithi181 ceph-mon[196979]: failed to encode map e215 with expected crc 2023-10-09T04:11:54.560 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:54 smithi181 ceph-mon[196979]: failed to encode map e215 with expected crc 2023-10-09T04:11:54.561 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:54 smithi181 ceph-mon[196979]: pgmap v560: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2197/163500 objects degraded (1.344%); 6456/163500 objects misplaced (3.949%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:54.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:11:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[213841]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T04:11:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[213841]: failed to encode map e215 with expected crc 2023-10-09T04:11:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[213841]: failed to encode map e215 with expected crc 2023-10-09T04:11:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[213841]: pgmap v560: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2197/163500 objects degraded (1.344%); 6456/163500 objects misplaced (3.949%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:11:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:11:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[221067]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T04:11:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[221067]: failed to encode map e215 with expected crc 2023-10-09T04:11:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[221067]: failed to encode map e215 with expected crc 2023-10-09T04:11:54.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:54 smithi163 ceph-mon[221067]: pgmap v560: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2197/163500 objects degraded (1.344%); 6456/163500 objects misplaced (3.949%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:54.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:54.699 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:54.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:54.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:54.760 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:11:54.855 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:54.886 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:54.937 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:54.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:54.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:55.042 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:55.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:11:55.136 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:55.379 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:55.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:55.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:55.589 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:55.663 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:55.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:55.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:55.871 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:55.930 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:56.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:56.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:56.197 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:11:56.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:56.228 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:11:56.269 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:11:56.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:11:56.330 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:56.362 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:11:56.404 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:11:56.452 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:11:56.501 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:11:56.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:11:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:56.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:11:56.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.541 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:11:56.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:56.665 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:56.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:56.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:56.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:56.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:11:56.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:11:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:56 smithi163 ceph-mon[213841]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 2197/159555 objects degraded (1.377%); 6456/159555 objects misplaced (4.046%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:56.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3665758244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:56.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:56 smithi163 ceph-mon[221067]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 2197/159555 objects degraded (1.377%); 6456/159555 objects misplaced (4.046%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:56.837 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3665758244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:56.902 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:57.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:56 smithi181 ceph-mon[196979]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 2197/159555 objects degraded (1.377%); 6456/159555 objects misplaced (4.046%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:57.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3665758244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:11:57.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:11:57.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:57.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:11:57.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:57.712 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:57.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:57.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:57.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/159555 objects degraded (1.377%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:57.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/159555 objects degraded (1.377%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:57.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:11:57.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/159555 objects degraded (1.377%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:11:57.895 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:57.930 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:57.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:11:57.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:57.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:11:58.050 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:11:58.192 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:58.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:11:58.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:58.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:11:58.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:58.506 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:58.565 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:58.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:58.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:58.680 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:11:58.701 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:58.758 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:58.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:11:58 smithi181 ceph-mon[196979]: pgmap v562: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 2197/156147 objects degraded (1.407%); 6110/156147 objects misplaced (3.913%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:58.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:11:58 smithi163 ceph-mon[213841]: pgmap v562: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 2197/156147 objects degraded (1.407%); 6110/156147 objects misplaced (3.913%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:58.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:11:58 smithi163 ceph-mon[221067]: pgmap v562: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 2197/156147 objects degraded (1.407%); 6110/156147 objects misplaced (3.913%); 819 KiB/s, 12 objects/s recovering 2023-10-09T04:11:58.867 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:58.897 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:58.937 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:11:58.937 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:11:58.959 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:58.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:58.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:59.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:59.096 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:59.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:59.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:59.263 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:11:59.292 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:59.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:11:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:59.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:11:59.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:11:59.485 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:59.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:11:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:59.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:11:59.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:11:59.682 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:11:59 smithi181 conmon[175350]: logger=sqlstore.transactions t=2023-10-09T04:11:59.52251111Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:11:59.688 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:11:59.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:11:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:11:59.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:11:59.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:11:59.896 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:00.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:12:00.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:12:00.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:00.559 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:00 smithi163 ceph-mon[213841]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/150831 objects degraded (1.457%); 5969/150831 objects misplaced (3.957%); 876 KiB/s, 13 objects/s recovering 2023-10-09T04:12:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:00.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:00.672 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:00.787 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:00.817 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:12:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:12:00] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:12:00.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:00 smithi163 ceph-mon[221067]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/150831 objects degraded (1.457%); 5969/150831 objects misplaced (3.957%); 876 KiB/s, 13 objects/s recovering 2023-10-09T04:12:00.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:00.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:00.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:01.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:00 smithi181 ceph-mon[196979]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/150831 objects degraded (1.457%); 5969/150831 objects misplaced (3.957%); 876 KiB/s, 13 objects/s recovering 2023-10-09T04:12:01.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:01.133 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:01.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:01.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:01.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:01.430 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:01.513 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:01.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:01.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:01.523 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:01.528 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:12:01.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:01.659 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:01.717 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:01.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:01.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:01.878 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:02.200 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:02.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:02.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:02.376 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:02.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:02 smithi163 ceph-mon[213841]: pgmap v564: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 2197/145449 objects degraded (1.510%); 5619/145449 objects misplaced (3.863%); 876 KiB/s, 13 objects/s recovering 2023-10-09T04:12:02.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:02 smithi163 ceph-mon[213841]: from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:02.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/145449 objects degraded (1.510%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:02 smithi163 ceph-mon[221067]: pgmap v564: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 2197/145449 objects degraded (1.510%); 5619/145449 objects misplaced (3.863%); 876 KiB/s, 13 objects/s recovering 2023-10-09T04:12:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:02 smithi163 ceph-mon[221067]: from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/145449 objects degraded (1.510%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:02.848 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:02.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:02.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:02.888 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:02.906 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":564,"stamp":"2023-10-09T04:12:01.472560+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3176339384,"num_objects":48483,"num_object_clones":0,"num_object_copies":145449,"num_objects_missing_on_primary":20,"num_objects_missing":20,"num_objects_degraded":2197,"num_objects_misplaced":5619,"num_objects_unfound":0,"num_objects_dirty":48483,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":131938,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10777,"num_bytes_recovered":707398324,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":154988,"ondisk_log_size":154988,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12200756,"kb_used_data":9784436,"kb_used_omap":126,"kb_used_meta":2416065,"kb_avail":737760460,"statfs":{"total":767960285184,"available":755466711040,"internally_reserved":0,"allocated":10019262464,"data_stored":10013306161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2474051195},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,0,0,8,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-475463680,"num_objects":-7255,"num_object_clones":0,"num_object_copies":-21765,"num_objects_missing_on_primary":-138,"num_objects_missing":-138,"num_objects_degraded":0,"num_objects_misplaced":-971,"num_objects_unfound":0,"num_objects_dirty":-7255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7255,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005839"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":212,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615581+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:11:42.615581+0000","last_peered":"2023-10-09T04:11:42.615581+0000","last_clean":"2023-10-09T04:11:42.615581+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:11:42.615581+0000","last_undegraded":"2023-10-09T04:11:42.615581+0000","last_fullsized":"2023-10-09T04:11:42.615581+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:48.675922+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:11:48.675922+0000","last_peered":"2023-10-09T04:11:48.675922+0000","last_clean":"2023-10-09T04:11:48.675922+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:11:48.675922+0000","last_undegraded":"2023-10-09T04:11:48.675922+0000","last_fullsized":"2023-10-09T04:11:48.675922+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":165,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.600570+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:11:42.600570+0000","last_peered":"2023-10-09T04:11:42.600570+0000","last_clean":"2023-10-09T04:11:42.600570+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:42.600570+0000","last_undegraded":"2023-10-09T04:11:42.600570+0000","last_fullsized":"2023-10-09T04:11:42.600570+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615729+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:11:42.615729+0000","last_peered":"2023-10-09T04:11:42.615729+0000","last_clean":"2023-10-09T04:11:42.615729+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:11:42.615729+0000","last_undegraded":"2023-10-09T04:11:42.615729+0000","last_fullsized":"2023-10-09T04:11:42.615729+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":169,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610130+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:11:42.610130+0000","last_peered":"2023-10-09T04:11:42.610130+0000","last_clean":"2023-10-09T04:11:42.610130+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:11:42.610130+0000","last_undegraded":"2023-10-09T04:11:42.610130+0000","last_fullsized":"2023-10-09T04:11:42.610130+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":148,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605548+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:11:42.605548+0000","last_peered":"2023-10-09T04:11:42.605548+0000","last_clean":"2023-10-09T04:11:42.605548+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:11:42.605548+0000","last_undegraded":"2023-10-09T04:11:42.605548+0000","last_fullsized":"2023-10-09T04:11:42.605548+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"215'5498","reported_seq":8452,"reported_epoch":215,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:00.652602+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:12:00.652602+0000","last_peered":"2023-10-09T04:12:00.652602+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:12:00.652602+0000","last_undegraded":"2023-10-09T04:12:00.652602+0000","last_fullsized":"2023-10-09T04:12:00.652602+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95748096,"num_objects":1461,"num_object_clones":0,"num_object_copies":4383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1461,"num_objects_unfound":0,"num_objects_dirty":1461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4327,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1461}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.968936+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:11:42.968936+0000","last_peered":"2023-10-09T04:11:42.968936+0000","last_clean":"2023-10-09T04:11:42.968936+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:11:42.968936+0000","last_undegraded":"2023-10-09T04:11:42.968936+0000","last_fullsized":"2023-10-09T04:11:42.968936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":193,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.609660+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:11:42.609660+0000","last_peered":"2023-10-09T04:11:42.609660+0000","last_clean":"2023-10-09T04:11:42.609660+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:11:42.609660+0000","last_undegraded":"2023-10-09T04:11:42.609660+0000","last_fullsized":"2023-10-09T04:11:42.609660+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"215'5547","reported_seq":10695,"reported_epoch":215,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:11:58.005466+0000","last_change":"2023-10-09T04:10:51.646576+0000","last_active":"2023-10-09T04:11:58.005466+0000","last_peered":"2023-10-09T04:11:58.005466+0000","last_clean":"2023-10-09T04:09:59.283093+0000","last_became_active":"2023-10-09T04:10:40.892310+0000","last_became_peered":"2023-10-09T04:10:40.892310+0000","last_unstale":"2023-10-09T04:11:58.005466+0000","last_undegraded":"2023-10-09T04:11:58.005466+0000","last_fullsized":"2023-10-09T04:10:40.864699+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5547,"log_dups_size":0,"ondisk_log_size":5547,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103022592,"num_objects":1572,"num_object_clones":0,"num_object_copies":4716,"num_objects_missing_on_primary":20,"num_objects_missing":20,"num_objects_degraded":0,"num_objects_misplaced":1128,"num_objects_unfound":0,"num_objects_dirty":1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4382,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"0,1,2,3","objects":1},{"shards":"0,1,6,7","objects":1087},{"shards":"0,6","objects":464},{"shards":"1,2,3","objects":20}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605959+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:11:42.605959+0000","last_peered":"2023-10-09T04:11:42.605959+0000","last_clean":"2023-10-09T04:11:42.605959+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:11:42.605959+0000","last_undegraded":"2023-10-09T04:11:42.605959+0000","last_fullsized":"2023-10-09T04:11:42.605959+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":111,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.618259+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:11:42.618259+0000","last_peered":"2023-10-09T04:11:42.618259+0000","last_clean":"2023-10-09T04:11:42.618259+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:11:42.618259+0000","last_undegraded":"2023-10-09T04:11:42.618259+0000","last_fullsized":"2023-10-09T04:11:42.618259+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"215'5444","reported_seq":8329,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:12:00.554555+0000","last_change":"2023-10-09T04:10:18.287893+0000","last_active":"2023-10-09T04:12:00.554555+0000","last_peered":"2023-10-09T04:12:00.554555+0000","last_clean":"2023-10-09T04:12:00.554555+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:12:00.554555+0000","last_undegraded":"2023-10-09T04:12:00.554555+0000","last_fullsized":"2023-10-09T04:12:00.554555+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96403456,"num_objects":1471,"num_object_clones":0,"num_object_copies":4413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4309,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":177,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.618040+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:11:42.618040+0000","last_peered":"2023-10-09T04:11:42.618040+0000","last_clean":"2023-10-09T04:11:42.618040+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:11:42.618040+0000","last_undegraded":"2023-10-09T04:11:42.618040+0000","last_fullsized":"2023-10-09T04:11:42.618040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:53.365531+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:11:53.365531+0000","last_peered":"2023-10-09T04:11:53.365531+0000","last_clean":"2023-10-09T04:11:53.365531+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:11:53.365531+0000","last_undegraded":"2023-10-09T04:11:53.365531+0000","last_fullsized":"2023-10-09T04:11:53.365531+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"215'5576","reported_seq":8512,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:59.916771+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:11:59.916771+0000","last_peered":"2023-10-09T04:11:59.916771+0000","last_clean":"2023-10-09T04:11:59.916771+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:11:59.916771+0000","last_undegraded":"2023-10-09T04:11:59.916771+0000","last_fullsized":"2023-10-09T04:11:59.916771+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98893824,"num_objects":1509,"num_object_clones":0,"num_object_copies":4527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4387,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":179,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615888+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:11:42.615888+0000","last_peered":"2023-10-09T04:11:42.615888+0000","last_clean":"2023-10-09T04:11:42.615888+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:11:42.615888+0000","last_undegraded":"2023-10-09T04:11:42.615888+0000","last_fullsized":"2023-10-09T04:11:42.615888+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":173,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616656+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:11:42.616656+0000","last_peered":"2023-10-09T04:11:42.616656+0000","last_clean":"2023-10-09T04:11:42.616656+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:11:42.616656+0000","last_undegraded":"2023-10-09T04:11:42.616656+0000","last_fullsized":"2023-10-09T04:11:42.616656+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"215'5428","reported_seq":5495,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.962973+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:11:57.962973+0000","last_peered":"2023-10-09T04:11:57.962973+0000","last_clean":"2023-10-09T04:11:57.962973+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:11:57.962973+0000","last_undegraded":"2023-10-09T04:11:57.962973+0000","last_fullsized":"2023-10-09T04:11:57.962973+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5428,"log_dups_size":0,"ondisk_log_size":5428,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102891520,"num_objects":1570,"num_object_clones":0,"num_object_copies":4710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4274,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":176,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617783+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:11:42.617783+0000","last_peered":"2023-10-09T04:11:42.617783+0000","last_clean":"2023-10-09T04:11:42.617783+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:11:42.617783+0000","last_undegraded":"2023-10-09T04:11:42.617783+0000","last_fullsized":"2023-10-09T04:11:42.617783+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":151,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.600770+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:11:42.600770+0000","last_peered":"2023-10-09T04:11:42.600770+0000","last_clean":"2023-10-09T04:11:42.600770+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:11:42.600770+0000","last_undegraded":"2023-10-09T04:11:42.600770+0000","last_fullsized":"2023-10-09T04:11:42.600770+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"215'5289","reported_seq":5355,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.962664+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:11:57.962664+0000","last_peered":"2023-10-09T04:11:57.962664+0000","last_clean":"2023-10-09T04:11:57.962664+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:11:57.962664+0000","last_undegraded":"2023-10-09T04:11:57.962664+0000","last_fullsized":"2023-10-09T04:11:57.962664+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96567296,"num_objects":1474,"num_object_clones":0,"num_object_copies":4422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4172,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617739+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:11:42.617739+0000","last_peered":"2023-10-09T04:11:42.617739+0000","last_clean":"2023-10-09T04:11:42.617739+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:11:42.617739+0000","last_undegraded":"2023-10-09T04:11:42.617739+0000","last_fullsized":"2023-10-09T04:11:42.617739+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":173,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605506+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:11:42.605506+0000","last_peered":"2023-10-09T04:11:42.605506+0000","last_clean":"2023-10-09T04:11:42.605506+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:11:42.605506+0000","last_undegraded":"2023-10-09T04:11:42.605506+0000","last_fullsized":"2023-10-09T04:11:42.605506+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"215'5491","reported_seq":8435,"reported_epoch":215,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:00.554604+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:12:00.554604+0000","last_peered":"2023-10-09T04:12:00.554604+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:12:00.554604+0000","last_undegraded":"2023-10-09T04:12:00.554604+0000","last_fullsized":"2023-10-09T04:12:00.554604+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5491,"log_dups_size":0,"ondisk_log_size":5491,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100106240,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1528,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4376,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1528}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.618193+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:11:42.618193+0000","last_peered":"2023-10-09T04:11:42.618193+0000","last_clean":"2023-10-09T04:11:42.618193+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:11:42.618193+0000","last_undegraded":"2023-10-09T04:11:42.618193+0000","last_fullsized":"2023-10-09T04:11:42.618193+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.956486+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:11:42.956486+0000","last_peered":"2023-10-09T04:11:42.956486+0000","last_clean":"2023-10-09T04:11:42.956486+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:11:42.956486+0000","last_undegraded":"2023-10-09T04:11:42.956486+0000","last_fullsized":"2023-10-09T04:11:42.956486+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"215'5472","reported_seq":5539,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.929479+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:11:57.929479+0000","last_peered":"2023-10-09T04:11:57.929479+0000","last_clean":"2023-10-09T04:11:57.929479+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:11:57.929479+0000","last_undegraded":"2023-10-09T04:11:57.929479+0000","last_fullsized":"2023-10-09T04:11:57.929479+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4310,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":106,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610037+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:11:42.610037+0000","last_peered":"2023-10-09T04:11:42.610037+0000","last_clean":"2023-10-09T04:11:42.610037+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:11:42.610037+0000","last_undegraded":"2023-10-09T04:11:42.610037+0000","last_fullsized":"2023-10-09T04:11:42.610037+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":179,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.609240+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:11:42.609240+0000","last_peered":"2023-10-09T04:11:42.609240+0000","last_clean":"2023-10-09T04:11:42.609240+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:11:42.609240+0000","last_undegraded":"2023-10-09T04:11:42.609240+0000","last_fullsized":"2023-10-09T04:11:42.609240+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"215'8570","reported_seq":8628,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:58.671913+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:11:58.671913+0000","last_peered":"2023-10-09T04:11:58.671913+0000","last_clean":"2023-10-09T04:11:58.671913+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:11:58.671913+0000","last_undegraded":"2023-10-09T04:11:58.671913+0000","last_fullsized":"2023-10-09T04:11:58.671913+0000","mapping_epoch":167,"log_start":"172'1500","ondisk_log_start":"172'1500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203554816,"num_objects":3106,"num_object_clones":0,"num_object_copies":9318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8570,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":175,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617940+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:11:42.617940+0000","last_peered":"2023-10-09T04:11:42.617940+0000","last_clean":"2023-10-09T04:11:42.617940+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:11:42.617940+0000","last_undegraded":"2023-10-09T04:11:42.617940+0000","last_fullsized":"2023-10-09T04:11:42.617940+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":156,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.604897+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:11:42.604897+0000","last_peered":"2023-10-09T04:11:42.604897+0000","last_clean":"2023-10-09T04:11:42.604897+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:11:42.604897+0000","last_undegraded":"2023-10-09T04:11:42.604897+0000","last_fullsized":"2023-10-09T04:11:42.604897+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"215'8597","reported_seq":8655,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:59.991240+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:11:59.991240+0000","last_peered":"2023-10-09T04:11:59.991240+0000","last_clean":"2023-10-09T04:11:59.991240+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:11:59.991240+0000","last_undegraded":"2023-10-09T04:11:59.991240+0000","last_fullsized":"2023-10-09T04:11:59.991240+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194314240,"num_objects":2965,"num_object_clones":0,"num_object_copies":8895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8597,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":176,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.618041+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:11:42.618041+0000","last_peered":"2023-10-09T04:11:42.618041+0000","last_clean":"2023-10-09T04:11:42.618041+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:11:42.618041+0000","last_undegraded":"2023-10-09T04:11:42.618041+0000","last_fullsized":"2023-10-09T04:11:42.618041+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616815+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:11:42.616815+0000","last_peered":"2023-10-09T04:11:42.616815+0000","last_clean":"2023-10-09T04:11:42.616815+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:11:42.616815+0000","last_undegraded":"2023-10-09T04:11:42.616815+0000","last_fullsized":"2023-10-09T04:11:42.616815+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"215'5497","reported_seq":5672,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.991163+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:11:57.991163+0000","last_peered":"2023-10-09T04:11:57.991163+0000","last_clean":"2023-10-09T04:11:57.991163+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:11:57.991163+0000","last_undegraded":"2023-10-09T04:11:57.991163+0000","last_fullsized":"2023-10-09T04:11:57.991163+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3111,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":158,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617910+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:11:42.617910+0000","last_peered":"2023-10-09T04:11:42.617910+0000","last_clean":"2023-10-09T04:11:42.617910+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:11:42.617910+0000","last_undegraded":"2023-10-09T04:11:42.617910+0000","last_fullsized":"2023-10-09T04:11:42.617910+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.601053+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:11:42.601053+0000","last_peered":"2023-10-09T04:11:42.601053+0000","last_clean":"2023-10-09T04:11:42.601053+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:11:42.601053+0000","last_undegraded":"2023-10-09T04:11:42.601053+0000","last_fullsized":"2023-10-09T04:11:42.601053+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"215'5390","reported_seq":5462,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:59.937304+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:11:59.937304+0000","last_peered":"2023-10-09T04:11:59.937304+0000","last_clean":"2023-10-09T04:11:59.937304+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:11:59.937304+0000","last_undegraded":"2023-10-09T04:11:59.937304+0000","last_fullsized":"2023-10-09T04:11:59.937304+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5390,"log_dups_size":0,"ondisk_log_size":5390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97124352,"num_objects":1482,"num_object_clones":0,"num_object_copies":4446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4250,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605313+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:11:42.605313+0000","last_peered":"2023-10-09T04:11:42.605313+0000","last_clean":"2023-10-09T04:11:42.605313+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:11:42.605313+0000","last_undegraded":"2023-10-09T04:11:42.605313+0000","last_fullsized":"2023-10-09T04:11:42.605313+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610289+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:11:42.610289+0000","last_peered":"2023-10-09T04:11:42.610289+0000","last_clean":"2023-10-09T04:11:42.610289+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:11:42.610289+0000","last_undegraded":"2023-10-09T04:11:42.610289+0000","last_fullsized":"2023-10-09T04:11:42.610289+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"215'8611","reported_seq":8669,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:58.019551+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:11:58.019551+0000","last_peered":"2023-10-09T04:11:58.019551+0000","last_clean":"2023-10-09T04:11:58.019551+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:11:58.019551+0000","last_undegraded":"2023-10-09T04:11:58.019551+0000","last_fullsized":"2023-10-09T04:11:58.019551+0000","mapping_epoch":167,"log_start":"171'1100","ondisk_log_start":"171'1100","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199950336,"num_objects":3051,"num_object_clones":0,"num_object_copies":9153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8611,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":119,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610256+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:11:42.610256+0000","last_peered":"2023-10-09T04:11:42.610256+0000","last_clean":"2023-10-09T04:11:42.610256+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:11:42.610256+0000","last_undegraded":"2023-10-09T04:11:42.610256+0000","last_fullsized":"2023-10-09T04:11:42.610256+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615809+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:11:42.615809+0000","last_peered":"2023-10-09T04:11:42.615809+0000","last_clean":"2023-10-09T04:11:42.615809+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:11:42.615809+0000","last_undegraded":"2023-10-09T04:11:42.615809+0000","last_fullsized":"2023-10-09T04:11:42.615809+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"215'8612","reported_seq":8686,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:12:00.599405+0000","last_change":"2023-10-09T04:10:38.444053+0000","last_active":"2023-10-09T04:12:00.599405+0000","last_peered":"2023-10-09T04:12:00.599405+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:12:00.599405+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197001216,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8612,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":2194},{"shards":"0,5","objects":809}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":134,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615655+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:11:42.615655+0000","last_peered":"2023-10-09T04:11:42.615655+0000","last_clean":"2023-10-09T04:11:42.615655+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:11:42.615655+0000","last_undegraded":"2023-10-09T04:11:42.615655+0000","last_fullsized":"2023-10-09T04:11:42.615655+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":210,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615654+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:11:42.615654+0000","last_peered":"2023-10-09T04:11:42.615654+0000","last_clean":"2023-10-09T04:11:42.615654+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:11:42.615654+0000","last_undegraded":"2023-10-09T04:11:42.615654+0000","last_fullsized":"2023-10-09T04:11:42.615654+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"215'8584","reported_seq":8643,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:58.579573+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:11:58.579573+0000","last_peered":"2023-10-09T04:11:58.579573+0000","last_clean":"2023-10-09T04:11:58.579573+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:11:58.579573+0000","last_undegraded":"2023-10-09T04:11:58.579573+0000","last_fullsized":"2023-10-09T04:11:58.579573+0000","mapping_epoch":167,"log_start":"172'1600","ondisk_log_start":"172'1600","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198901784,"num_objects":3036,"num_object_clones":0,"num_object_copies":9108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3036,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8584,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616658+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:11:42.616658+0000","last_peered":"2023-10-09T04:11:42.616658+0000","last_clean":"2023-10-09T04:11:42.616658+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:11:42.616658+0000","last_undegraded":"2023-10-09T04:11:42.616658+0000","last_fullsized":"2023-10-09T04:11:42.616658+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":166,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616063+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:11:42.616063+0000","last_peered":"2023-10-09T04:11:42.616063+0000","last_clean":"2023-10-09T04:11:42.616063+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:11:42.616063+0000","last_undegraded":"2023-10-09T04:11:42.616063+0000","last_fullsized":"2023-10-09T04:11:42.616063+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"215'8488","reported_seq":8546,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:12:00.703504+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:12:00.703504+0000","last_peered":"2023-10-09T04:12:00.703504+0000","last_clean":"2023-10-09T04:12:00.703504+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:12:00.703504+0000","last_undegraded":"2023-10-09T04:12:00.703504+0000","last_fullsized":"2023-10-09T04:12:00.703504+0000","mapping_epoch":167,"log_start":"173'2100","ondisk_log_start":"173'2100","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8488,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":180,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.609487+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:11:42.609487+0000","last_peered":"2023-10-09T04:11:42.609487+0000","last_clean":"2023-10-09T04:11:42.609487+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:11:42.609487+0000","last_undegraded":"2023-10-09T04:11:42.609487+0000","last_fullsized":"2023-10-09T04:11:42.609487+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":167,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.604770+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:11:42.604770+0000","last_peered":"2023-10-09T04:11:42.604770+0000","last_clean":"2023-10-09T04:11:42.604770+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:11:42.604770+0000","last_undegraded":"2023-10-09T04:11:42.604770+0000","last_fullsized":"2023-10-09T04:11:42.604770+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"215'5475","reported_seq":5665,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:12:00.217585+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:12:00.217585+0000","last_peered":"2023-10-09T04:12:00.217585+0000","last_clean":"2023-10-09T04:12:00.217585+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:12:00.217585+0000","last_undegraded":"2023-10-09T04:12:00.217585+0000","last_fullsized":"2023-10-09T04:12:00.217585+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99811328,"num_objects":1523,"num_object_clones":0,"num_object_copies":4569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1822,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":190,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.600919+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:11:42.600919+0000","last_peered":"2023-10-09T04:11:42.600919+0000","last_clean":"2023-10-09T04:11:42.600919+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:11:42.600919+0000","last_undegraded":"2023-10-09T04:11:42.600919+0000","last_fullsized":"2023-10-09T04:11:42.600919+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":191,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605590+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:11:42.605590+0000","last_peered":"2023-10-09T04:11:42.605590+0000","last_clean":"2023-10-09T04:11:42.605590+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:11:42.605590+0000","last_undegraded":"2023-10-09T04:11:42.605590+0000","last_fullsized":"2023-10-09T04:11:42.605590+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":152,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.601236+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:11:42.601236+0000","last_peered":"2023-10-09T04:11:42.601236+0000","last_clean":"2023-10-09T04:11:42.601236+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:11:42.601236+0000","last_undegraded":"2023-10-09T04:11:42.601236+0000","last_fullsized":"2023-10-09T04:11:42.601236+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"215'5475","reported_seq":5543,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.851887+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:11:57.851887+0000","last_peered":"2023-10-09T04:11:57.851887+0000","last_clean":"2023-10-09T04:11:57.851887+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:11:57.851887+0000","last_undegraded":"2023-10-09T04:11:57.851887+0000","last_fullsized":"2023-10-09T04:11:57.851887+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99942400,"num_objects":1525,"num_object_clones":0,"num_object_copies":4575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4321,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":178,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605148+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:11:42.605148+0000","last_peered":"2023-10-09T04:11:42.605148+0000","last_clean":"2023-10-09T04:11:42.605148+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:11:42.605148+0000","last_undegraded":"2023-10-09T04:11:42.605148+0000","last_fullsized":"2023-10-09T04:11:42.605148+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616719+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:11:42.616719+0000","last_peered":"2023-10-09T04:11:42.616719+0000","last_clean":"2023-10-09T04:11:42.616719+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:11:42.616719+0000","last_undegraded":"2023-10-09T04:11:42.616719+0000","last_fullsized":"2023-10-09T04:11:42.616719+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":200,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.600846+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:11:42.600846+0000","last_peered":"2023-10-09T04:11:42.600846+0000","last_clean":"2023-10-09T04:11:42.600846+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:11:42.600846+0000","last_undegraded":"2023-10-09T04:11:42.600846+0000","last_fullsized":"2023-10-09T04:11:42.600846+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"215'5507","reported_seq":5575,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.914462+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:11:57.914462+0000","last_peered":"2023-10-09T04:11:57.914462+0000","last_clean":"2023-10-09T04:11:57.914462+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:11:57.914462+0000","last_undegraded":"2023-10-09T04:11:57.914462+0000","last_fullsized":"2023-10-09T04:11:57.914462+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5507,"log_dups_size":0,"ondisk_log_size":5507,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100859904,"num_objects":1539,"num_object_clones":0,"num_object_copies":4617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4345,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.614356+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:11:42.614356+0000","last_peered":"2023-10-09T04:11:42.614356+0000","last_clean":"2023-10-09T04:11:42.614356+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:11:42.614356+0000","last_undegraded":"2023-10-09T04:11:42.614356+0000","last_fullsized":"2023-10-09T04:11:42.614356+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":184,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.600021+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:11:42.600021+0000","last_peered":"2023-10-09T04:11:42.600021+0000","last_clean":"2023-10-09T04:11:42.600021+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:11:42.600021+0000","last_undegraded":"2023-10-09T04:11:42.600021+0000","last_fullsized":"2023-10-09T04:11:42.600021+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":185,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.604849+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:11:42.604849+0000","last_peered":"2023-10-09T04:11:42.604849+0000","last_clean":"2023-10-09T04:11:42.604849+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:11:42.604849+0000","last_undegraded":"2023-10-09T04:11:42.604849+0000","last_fullsized":"2023-10-09T04:11:42.604849+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"215'5310","reported_seq":5377,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.967521+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:11:57.967521+0000","last_peered":"2023-10-09T04:11:57.967521+0000","last_clean":"2023-10-09T04:11:57.967521+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:11:57.967521+0000","last_undegraded":"2023-10-09T04:11:57.967521+0000","last_fullsized":"2023-10-09T04:11:57.967521+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100827136,"num_objects":1538,"num_object_clones":0,"num_object_copies":4614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4192,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":198,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605270+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:11:42.605270+0000","last_peered":"2023-10-09T04:11:42.605270+0000","last_clean":"2023-10-09T04:11:42.605270+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:11:42.605270+0000","last_undegraded":"2023-10-09T04:11:42.605270+0000","last_fullsized":"2023-10-09T04:11:42.605270+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":555,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617586+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:11:42.617586+0000","last_peered":"2023-10-09T04:11:42.617586+0000","last_clean":"2023-10-09T04:11:42.617586+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:11:42.617586+0000","last_undegraded":"2023-10-09T04:11:42.617586+0000","last_fullsized":"2023-10-09T04:11:42.617586+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":111,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605081+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:11:42.605081+0000","last_peered":"2023-10-09T04:11:42.605081+0000","last_clean":"2023-10-09T04:11:42.605081+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:11:42.605081+0000","last_undegraded":"2023-10-09T04:11:42.605081+0000","last_fullsized":"2023-10-09T04:11:42.605081+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":94,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615867+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:11:42.615867+0000","last_peered":"2023-10-09T04:11:42.615867+0000","last_clean":"2023-10-09T04:11:42.615867+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:11:42.615867+0000","last_undegraded":"2023-10-09T04:11:42.615867+0000","last_fullsized":"2023-10-09T04:11:42.615867+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"215'5409","reported_seq":7758,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:12:00.448111+0000","last_change":"2023-10-09T04:10:53.276397+0000","last_active":"2023-10-09T04:12:00.448111+0000","last_peered":"2023-10-09T04:12:00.448111+0000","last_clean":"2023-10-09T04:12:00.448111+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:12:00.448111+0000","last_undegraded":"2023-10-09T04:12:00.448111+0000","last_fullsized":"2023-10-09T04:12:00.448111+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97091584,"num_objects":1481,"num_object_clones":0,"num_object_copies":4443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4293,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":195,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617576+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:11:42.617576+0000","last_peered":"2023-10-09T04:11:42.617576+0000","last_clean":"2023-10-09T04:11:42.617576+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:11:42.617576+0000","last_undegraded":"2023-10-09T04:11:42.617576+0000","last_fullsized":"2023-10-09T04:11:42.617576+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":170,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610901+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:11:42.610901+0000","last_peered":"2023-10-09T04:11:42.610901+0000","last_clean":"2023-10-09T04:11:42.610901+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:11:42.610901+0000","last_undegraded":"2023-10-09T04:11:42.610901+0000","last_fullsized":"2023-10-09T04:11:42.610901+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":203,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615504+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:11:42.615504+0000","last_peered":"2023-10-09T04:11:42.615504+0000","last_clean":"2023-10-09T04:11:42.615504+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:11:42.615504+0000","last_undegraded":"2023-10-09T04:11:42.615504+0000","last_fullsized":"2023-10-09T04:11:42.615504+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"215'5651","reported_seq":5781,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:12:00.503896+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:12:00.503896+0000","last_peered":"2023-10-09T04:12:00.503896+0000","last_clean":"2023-10-09T04:12:00.503896+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:12:00.503896+0000","last_undegraded":"2023-10-09T04:12:00.503896+0000","last_fullsized":"2023-10-09T04:12:00.503896+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99221504,"num_objects":1514,"num_object_clones":0,"num_object_copies":4542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3104,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":178,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617969+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:11:42.617969+0000","last_peered":"2023-10-09T04:11:42.617969+0000","last_clean":"2023-10-09T04:11:42.617969+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:11:42.617969+0000","last_undegraded":"2023-10-09T04:11:42.617969+0000","last_fullsized":"2023-10-09T04:11:42.617969+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":184,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.601000+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:11:42.601000+0000","last_peered":"2023-10-09T04:11:42.601000+0000","last_clean":"2023-10-09T04:11:42.601000+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:11:42.601000+0000","last_undegraded":"2023-10-09T04:11:42.601000+0000","last_fullsized":"2023-10-09T04:11:42.601000+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610217+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:11:42.610217+0000","last_peered":"2023-10-09T04:11:42.610217+0000","last_clean":"2023-10-09T04:11:42.610217+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:11:42.610217+0000","last_undegraded":"2023-10-09T04:11:42.610217+0000","last_fullsized":"2023-10-09T04:11:42.610217+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"215'5645","reported_seq":8706,"reported_epoch":215,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:00.603337+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:12:00.603337+0000","last_peered":"2023-10-09T04:12:00.603337+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:12:00.603337+0000","last_undegraded":"2023-10-09T04:12:00.603337+0000","last_fullsized":"2023-10-09T04:12:00.603337+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98435072,"num_objects":1502,"num_object_clones":0,"num_object_copies":4506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1502,"num_objects_unfound":0,"num_objects_dirty":1502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4474,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1502}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":119,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617834+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:11:42.617834+0000","last_peered":"2023-10-09T04:11:42.617834+0000","last_clean":"2023-10-09T04:11:42.617834+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:11:42.617834+0000","last_undegraded":"2023-10-09T04:11:42.617834+0000","last_fullsized":"2023-10-09T04:11:42.617834+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":151,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.604874+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:11:42.604874+0000","last_peered":"2023-10-09T04:11:42.604874+0000","last_clean":"2023-10-09T04:11:42.604874+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:11:42.604874+0000","last_undegraded":"2023-10-09T04:11:42.604874+0000","last_fullsized":"2023-10-09T04:11:42.604874+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":145,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617721+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:11:42.617721+0000","last_peered":"2023-10-09T04:11:42.617721+0000","last_clean":"2023-10-09T04:11:42.617721+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:11:42.617721+0000","last_undegraded":"2023-10-09T04:11:42.617721+0000","last_fullsized":"2023-10-09T04:11:42.617721+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"215'5521","reported_seq":7732,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:58.638579+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:11:58.638579+0000","last_peered":"2023-10-09T04:11:58.638579+0000","last_clean":"2023-10-09T04:11:58.638579+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:11:58.638579+0000","last_undegraded":"2023-10-09T04:11:58.638579+0000","last_fullsized":"2023-10-09T04:11:58.638579+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96075776,"num_objects":1466,"num_object_clones":0,"num_object_copies":4398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3044,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":143,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.617909+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:11:42.617909+0000","last_peered":"2023-10-09T04:11:42.617909+0000","last_clean":"2023-10-09T04:11:42.617909+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:11:42.617909+0000","last_undegraded":"2023-10-09T04:11:42.617909+0000","last_fullsized":"2023-10-09T04:11:42.617909+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615590+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:11:42.615590+0000","last_peered":"2023-10-09T04:11:42.615590+0000","last_clean":"2023-10-09T04:11:42.615590+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:11:42.615590+0000","last_undegraded":"2023-10-09T04:11:42.615590+0000","last_fullsized":"2023-10-09T04:11:42.615590+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616703+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:11:42.616703+0000","last_peered":"2023-10-09T04:11:42.616703+0000","last_clean":"2023-10-09T04:11:42.616703+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:11:42.616703+0000","last_undegraded":"2023-10-09T04:11:42.616703+0000","last_fullsized":"2023-10-09T04:11:42.616703+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":158,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.610358+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:11:42.610358+0000","last_peered":"2023-10-09T04:11:42.610358+0000","last_clean":"2023-10-09T04:11:42.610358+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:11:42.610358+0000","last_undegraded":"2023-10-09T04:11:42.610358+0000","last_fullsized":"2023-10-09T04:11:42.610358+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":28,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.613707+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:11:42.613707+0000","last_peered":"2023-10-09T04:11:42.613707+0000","last_clean":"2023-10-09T04:11:42.613707+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:11:42.613707+0000","last_undegraded":"2023-10-09T04:11:42.613707+0000","last_fullsized":"2023-10-09T04:11:42.613707+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"215'5469","reported_seq":5538,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:59.916857+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:11:59.916857+0000","last_peered":"2023-10-09T04:11:59.916857+0000","last_clean":"2023-10-09T04:11:59.916857+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:11:59.916857+0000","last_undegraded":"2023-10-09T04:11:59.916857+0000","last_fullsized":"2023-10-09T04:11:59.916857+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97583104,"num_objects":1489,"num_object_clones":0,"num_object_copies":4467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4329,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":176,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605423+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:11:42.605423+0000","last_peered":"2023-10-09T04:11:42.605423+0000","last_clean":"2023-10-09T04:11:42.605423+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:11:42.605423+0000","last_undegraded":"2023-10-09T04:11:42.605423+0000","last_fullsized":"2023-10-09T04:11:42.605423+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616702+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:11:42.616702+0000","last_peered":"2023-10-09T04:11:42.616702+0000","last_clean":"2023-10-09T04:11:42.616702+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:11:42.616702+0000","last_undegraded":"2023-10-09T04:11:42.616702+0000","last_fullsized":"2023-10-09T04:11:42.616702+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"215'5516","reported_seq":5580,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-09T04:11:57.885470+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:11:57.885470+0000","last_peered":"2023-10-09T04:11:57.885470+0000","last_clean":"2023-10-09T04:11:57.885470+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:11:57.885470+0000","last_undegraded":"2023-10-09T04:11:57.885470+0000","last_fullsized":"2023-10-09T04:11:57.885470+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101580800,"num_objects":1550,"num_object_clones":0,"num_object_copies":4650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4354,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":159,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.605683+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:11:42.605683+0000","last_peered":"2023-10-09T04:11:42.605683+0000","last_clean":"2023-10-09T04:11:42.605683+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:11:42.605683+0000","last_undegraded":"2023-10-09T04:11:42.605683+0000","last_fullsized":"2023-10-09T04:11:42.605683+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":190,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615723+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:11:42.615723+0000","last_peered":"2023-10-09T04:11:42.615723+0000","last_clean":"2023-10-09T04:11:42.615723+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:11:42.615723+0000","last_undegraded":"2023-10-09T04:11:42.615723+0000","last_fullsized":"2023-10-09T04:11:42.615723+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":28,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.614001+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:11:42.614001+0000","last_peered":"2023-10-09T04:11:42.614001+0000","last_clean":"2023-10-09T04:11:42.614001+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:11:42.614001+0000","last_undegraded":"2023-10-09T04:11:42.614001+0000","last_fullsized":"2023-10-09T04:11:42.614001+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":28,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.614432+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:11:42.614432+0000","last_peered":"2023-10-09T04:11:42.614432+0000","last_clean":"2023-10-09T04:11:42.614432+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:11:42.614432+0000","last_undegraded":"2023-10-09T04:11:42.614432+0000","last_fullsized":"2023-10-09T04:11:42.614432+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":204,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.616163+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:11:42.616163+0000","last_peered":"2023-10-09T04:11:42.616163+0000","last_clean":"2023-10-09T04:11:42.616163+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:11:42.616163+0000","last_undegraded":"2023-10-09T04:11:42.616163+0000","last_fullsized":"2023-10-09T04:11:42.616163+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-10-09T04:11:42.615565+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:11:42.615565+0000","last_peered":"2023-10-09T04:11:42.615565+0000","last_clean":"2023-10-09T04:11:42.615565+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:11:42.615565+0000","last_undegraded":"2023-10-09T04:11:42.615565+0000","last_fullsized":"2023-10-09T04:11:42.615565+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":3175743512,"num_objects":48459,"num_object_clones":0,"num_object_copies":145377,"num_objects_missing_on_primary":20,"num_objects_missing":20,"num_objects_degraded":2197,"num_objects_misplaced":5619,"num_objects_unfound":0,"num_objects_dirty":48459,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131638,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10767,"num_bytes_recovered":705626112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9932124160,"data_stored":9932111944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":154772,"ondisk_log_size":154772,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201514,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1462444,"kb_used_data":1165156,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92282708,"statfs":{"total":95995035648,"available":94497492992,"internally_reserved":0,"allocated":1193119744,"data_stored":1192473337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":10.025,"5min":3.5390000000000001,"15min":1.583},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":126.752,"5min":126.752,"15min":126.752},"last":0.48999999999999999},{"interface":"front","average":{"1min":10.521000000000001,"5min":3.609,"15min":1.6060000000000001},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":128.88399999999999,"5min":128.88399999999999,"15min":128.88399999999999},"last":3.9540000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":10.563000000000001,"5min":5.8120000000000003,"15min":2.3980000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":103.014,"5min":245.28700000000001,"15min":245.28700000000001},"last":1.7509999999999999},{"interface":"front","average":{"1min":10.186999999999999,"5min":5.3700000000000001,"15min":2.379},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":103.083,"5min":244.626,"15min":244.626},"last":0.28100000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.8799999999999999,"15min":1.268},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4009999999999998,"5min":13.887,"15min":58.615000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.401,"5min":2.9279999999999999,"15min":1.6850000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.3109999999999999,"5min":99.372,"15min":99.372},"last":2.0739999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.5469999999999999,"15min":1.5589999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":3.0430000000000001,"5min":13.124000000000001,"15min":99.134},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.6599999999999999,"15min":1.6399999999999999},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.0230000000000001,"5min":13.140000000000001,"15min":99.117999999999995},"last":0.621}]},{"osd":4,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.079,"15min":0.69699999999999995},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9790000000000001,"5min":16.195,"15min":16.195},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.442,"5min":1.0109999999999999,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.052,"5min":12.442,"15min":12.442},"last":0.35799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.77800000000000002,"15min":0.59699999999999998},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81999999999999995,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.873,"15min":0.67800000000000005},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0550000000000002,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.91300000000000003,"15min":0.67600000000000005},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.99399999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.03,"15min":0.745},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":2.0819999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.54500000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463151,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1553056,"kb_used_data":1247576,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92192096,"statfs":{"total":95995035648,"available":94404706304,"internally_reserved":0,"allocated":1277517824,"data_stored":1276883260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7850000000000001,"5min":4.2990000000000004,"15min":1.8460000000000001},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":60.295000000000002,"5min":78.983999999999995,"15min":78.983999999999995},"last":0.85999999999999999},{"interface":"front","average":{"1min":6.8220000000000001,"5min":4.8659999999999997,"15min":2.0830000000000002},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":60.716000000000001,"5min":78.998000000000005,"15min":78.998000000000005},"last":1.506}]},{"osd":1,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.657,"15min":1.6950000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":4.6840000000000002,"5min":62.698,"15min":62.698},"last":2.1179999999999999},{"interface":"front","average":{"1min":1.909,"5min":3.875,"15min":1.794},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":8.3529999999999998,"5min":63.530999999999999,"15min":63.530999999999999},"last":2.2360000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":7.6840000000000002,"5min":3.8420000000000001,"15min":1.8420000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":77.953000000000003,"5min":77.953000000000003,"15min":77.953000000000003},"last":0.53900000000000003},{"interface":"front","average":{"1min":7.7320000000000002,"5min":3.875,"15min":1.8879999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":78.019000000000005,"5min":78.019000000000005,"15min":78.019000000000005},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":3.516,"15min":2.0339999999999998},"min":{"1min":0.35799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.336,"5min":57.509999999999998,"15min":57.509999999999998},"last":1.351},{"interface":"front","average":{"1min":1.1120000000000001,"5min":3.7450000000000001,"15min":2.052},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1840000000000002,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.32600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.3720000000000001,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.72199999999999998,"15min":0.64400000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2970000000000002,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.34300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":0.84599999999999997,"15min":0.84899999999999998},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.081,"5min":0.97399999999999998,"15min":0.95499999999999996},"min":{"1min":0.34399999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":23.006,"5min":23.006,"15min":54.853999999999999},"last":0.38700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.69999999999999996,"15min":0.61499999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":2.903,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.376},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.80900000000000005,"15min":0.68500000000000005},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.28699999999999998},"max":{"1min":2.835,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.41399999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089459,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1574488,"kb_used_data":1354256,"kb_used_omap":9,"kb_used_meta":220214,"kb_avail":92170664,"statfs":{"total":95995035648,"available":94382759936,"internally_reserved":0,"allocated":1386758144,"data_stored":1385895800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":225500137},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,0,0,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":6.4269999999999996,"5min":3.7930000000000001,"15min":1.843},"min":{"1min":0.247,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":61.689999999999998,"5min":85.977999999999994,"15min":85.977999999999994},"last":0.318},{"interface":"front","average":{"1min":7.4409999999999998,"5min":4.2519999999999998,"15min":2.3519999999999999},"min":{"1min":0.31900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":61.670000000000002,"5min":85.998999999999995,"15min":99.831999999999994},"last":8.9139999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":3.6469999999999998,"15min":1.5369999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.6,"5min":121.54600000000001,"15min":121.54600000000001},"last":3.8980000000000001},{"interface":"front","average":{"1min":2.3010000000000002,"5min":3.7919999999999998,"15min":1.702},"min":{"1min":0.23200000000000001,"5min":0.186,"15min":0.186},"max":{"1min":11.052,"5min":99.899000000000001,"15min":99.899000000000001},"last":9.7319999999999993}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":4.2089999999999996,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":7.8920000000000003,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.593,"5min":4.6449999999999996,"15min":2.6309999999999998},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.414,"5min":138.49799999999999,"15min":138.49799999999999},"last":8.8789999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":1.8520000000000001,"15min":1.498},"min":{"1min":0.35599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":15.215999999999999,"5min":15.215999999999999,"15min":99.989999999999995},"last":8.843},{"interface":"front","average":{"1min":1.115,"5min":1.5129999999999999,"15min":1.3779999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.52,"5min":10.438000000000001,"15min":100.249},"last":0.58299999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.75,"15min":1.48},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.266,"5min":15.266,"15min":99.933999999999997},"last":8.7859999999999996},{"interface":"front","average":{"1min":1.389,"5min":1.829,"15min":1.4950000000000001},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":4.8849999999999998,"5min":10.467000000000001,"15min":100.34099999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.827,"15min":1.5880000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.585,"5min":10.497999999999999,"15min":100.309},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.671,"5min":1.496,"15min":1.5169999999999999},"min":{"1min":0.36499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9240000000000004,"5min":7.9240000000000004,"15min":100.143},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.9630000000000001,"15min":1.569},"min":{"1min":0.40000000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.9569999999999999,"5min":10.532,"15min":100.214},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.798,"5min":1.796,"15min":1.5429999999999999},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":100.113},"last":0.56699999999999995}]}]},{"osd":0,"up_from":84,"seq":360777252985,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1937132,"kb_used_data":1699044,"kb_used_omap":28,"kb_used_meta":238051,"kb_avail":91808020,"statfs":{"total":95995035648,"available":94011412480,"internally_reserved":0,"allocated":1739821056,"data_stored":1738937695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":243765072},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,0,0,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":3.6059999999999999,"15min":1.982},"min":{"1min":0.184,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":12.698,"5min":103.996,"15min":118.48},"last":8.4079999999999995},{"interface":"front","average":{"1min":3.5910000000000002,"5min":4.1959999999999997,"15min":2.2080000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":12.632999999999999,"5min":106.34399999999999,"15min":118.384},"last":7.649}]},{"osd":2,"last update":"Mon Oct 9 04:11:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":4.5990000000000002,"15min":1.9419999999999999},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.8819999999999997,"5min":118.54000000000001,"15min":118.54000000000001},"last":7.6239999999999997},{"interface":"front","average":{"1min":1.736,"5min":5.0780000000000003,"15min":2.089},"min":{"1min":0.21199999999999999,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":8.2300000000000004,"5min":118.727,"15min":118.727},"last":6.8819999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":5.9320000000000004,"15min":3.601},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.161,"5min":156.488,"15min":156.488},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.1909999999999998,"5min":5.8979999999999997,"15min":3.6890000000000001},"min":{"1min":0.40699999999999997,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":9.6010000000000009,"5min":156.50700000000001,"15min":156.50700000000001},"last":7.7240000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.3540000000000001,"15min":1.625},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8310000000000004,"5min":59.439999999999998,"15min":118.633},"last":7.5860000000000003},{"interface":"front","average":{"1min":1.151,"5min":2.9889999999999999,"15min":1.897},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.593,"5min":59.453000000000003,"15min":118.913},"last":1.5489999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":4.548,"15min":2.4169999999999998},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.6310000000000002,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.302,"5min":4.2830000000000004,"15min":2.3039999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.8919999999999995,"5min":182.66300000000001,"15min":182.66300000000001},"last":1.516}]},{"osd":6,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":8.0259999999999998,"5min":3.976,"15min":2.2109999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":137.36099999999999,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.378},{"interface":"front","average":{"1min":7.3899999999999997,"5min":3.5800000000000001,"15min":2.0670000000000002},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":136.88999999999999,"5min":136.88999999999999,"15min":136.88999999999999},"last":7.6719999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":7.2210000000000001,"5min":5.7480000000000002,"15min":2.8370000000000002},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":121.324,"5min":185.024,"15min":185.024},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.206,"5min":4.21,"15min":2.3180000000000001},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":6.6280000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":7.7000000000000002}]}]},{"osd":2,"up_from":94,"seq":403726925933,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1228360,"kb_used_data":912768,"kb_used_omap":81,"kb_used_meta":315438,"kb_avail":92516792,"statfs":{"total":95995035648,"available":94737195008,"internally_reserved":0,"allocated":934674432,"data_stored":933826170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":323008730},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.4550000000000001,"15min":1.1419999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.413,"5min":12.284000000000001,"15min":12.284000000000001},"last":5.5679999999999996},{"interface":"front","average":{"1min":2.5840000000000001,"5min":3.2650000000000001,"15min":1.5189999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":11.356999999999999,"5min":45.979999999999997,"15min":45.979999999999997},"last":5.4009999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":3.6539999999999999,"15min":1.8160000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":14.99,"5min":53.695,"15min":68.009},"last":5.4500000000000002},{"interface":"front","average":{"1min":2.5979999999999999,"5min":3.8730000000000002,"15min":1.919},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":16.597000000000001,"5min":53.439,"15min":66.975999999999999},"last":0.85499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":2.2029999999999998,"15min":1.349},"min":{"1min":0.22500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.212999999999999,"5min":12.77,"15min":27.562000000000001},"last":5.5839999999999996},{"interface":"front","average":{"1min":2.577,"5min":2.8199999999999998,"15min":1.552},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.808,"5min":46.286999999999999,"15min":46.286999999999999},"last":5.4249999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":2.419,"15min":1.3280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9209999999999998,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.524,"5min":1.7450000000000001,"15min":1.0329999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.5809999999999995,"5min":14.305,"15min":14.305},"last":5.5209999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.601,"15min":1.373},"min":{"1min":0.373,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":26.667000000000002,"5min":46.265000000000001,"15min":46.265000000000001},"last":5.3140000000000001},{"interface":"front","average":{"1min":2.7999999999999998,"5min":2.8620000000000001,"15min":1.5},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":26.654,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.61199999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0199999999999996,"5min":2.9830000000000001,"15min":1.4690000000000001},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":114.14,"5min":114.14,"15min":114.14},"last":5.5460000000000003},{"interface":"front","average":{"1min":7.8810000000000002,"5min":3.6789999999999998,"15min":1.7549999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":114.123,"5min":114.123,"15min":114.123},"last":5.359}]},{"osd":7,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.1690000000000005,"5min":3.4550000000000001,"15min":1.651},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":117.32599999999999,"5min":117.32599999999999,"15min":117.32599999999999},"last":5.4939999999999998},{"interface":"front","average":{"1min":8.1639999999999997,"5min":3.9079999999999999,"15min":1.8540000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":117.28700000000001,"5min":117.28700000000001,"15min":117.28700000000001},"last":5.3810000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762407,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2280108,"kb_used_data":1851428,"kb_used_omap":7,"kb_used_meta":428664,"kb_avail":91465044,"statfs":{"total":95995035648,"available":93660205056,"internally_reserved":0,"allocated":1895862272,"data_stored":1895038833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":438952552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8140000000000001,"5min":4.6970000000000001,"15min":3.3519999999999999},"min":{"1min":0.20100000000000001,"5min":0.189,"15min":0.185},"max":{"1min":23.449000000000002,"5min":100.512,"15min":106.22},"last":1.954},{"interface":"front","average":{"1min":4.0759999999999996,"5min":5.056,"15min":3.4039999999999999},"min":{"1min":0.32200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":13.670999999999999,"5min":101.733,"15min":106.986},"last":13.414}]},{"osd":1,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":3.5699999999999998,"15min":2.2490000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":8.6340000000000003,"5min":57.055999999999997,"15min":106.121},"last":1.716},{"interface":"front","average":{"1min":3.899,"5min":3.9180000000000001,"15min":2.1019999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":23.475999999999999,"5min":57.088999999999999,"15min":65.015000000000001},"last":1.982}]},{"osd":2,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.8130000000000002,"15min":2.1669999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":23.405999999999999,"5min":23.405999999999999,"15min":106.105},"last":2.0150000000000001},{"interface":"front","average":{"1min":3.524,"5min":3.7229999999999999,"15min":2.5190000000000001},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":13.698,"5min":98.765000000000001,"15min":106.70399999999999},"last":13.201000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":2.9060000000000001,"15min":1.927},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.308},"max":{"1min":13.536,"5min":37.652000000000001,"15min":106.85899999999999},"last":6.4660000000000002},{"interface":"front","average":{"1min":1.873,"5min":2.5209999999999999,"15min":1.351},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.1699999999999999,"5min":37.442,"15min":37.442},"last":1.79}]},{"osd":5,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.1150000000000002,"15min":1.2430000000000001},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.1940000000000008,"5min":12.819000000000001,"15min":12.819000000000001},"last":1.7609999999999999},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.9730000000000001,"15min":1.516},"min":{"1min":0.32400000000000001,"5min":0.29599999999999999,"15min":0.27900000000000003},"max":{"1min":11.938000000000001,"5min":13.228999999999999,"15min":13.228999999999999},"last":2.0680000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.8560000000000001,"15min":1.9039999999999999},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.30199999999999999},"max":{"1min":11.994999999999999,"5min":13.255000000000001,"15min":106.036},"last":2.097},{"interface":"front","average":{"1min":2.363,"5min":2.355,"15min":1.7789999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.571,"5min":15.622,"15min":106.741},"last":6.5099999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":2.5099999999999998,"15min":1.8260000000000001},"min":{"1min":0.38700000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.494999999999999,"5min":15.685,"15min":106.82599999999999},"last":6.5350000000000001},{"interface":"front","average":{"1min":2.1480000000000001,"5min":2.1749999999999998,"15min":1.7010000000000001},"min":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.2170000000000005,"5min":13.228999999999999,"15min":106.084},"last":1.819}]}]},{"osd":4,"up_from":31,"seq":133143986423,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174496,"kb_used_data":873112,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92570656,"statfs":{"total":95995035648,"available":94792351744,"internally_reserved":0,"allocated":894066688,"data_stored":893431690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":2.8290000000000002,"15min":1.3500000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":1.726,"5min":63.128,"15min":63.128},"last":0.313},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.5110000000000001,"15min":1.3149999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1600000000000001,"5min":62.645000000000003,"15min":62.645000000000003},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.7709999999999999,"15min":1.4410000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":23.157,"5min":79.075999999999993,"15min":79.075999999999993},"last":1.4910000000000001},{"interface":"front","average":{"1min":2.8410000000000002,"5min":2.8479999999999999,"15min":1.4299999999999999},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":23.091999999999999,"5min":75.314999999999998,"15min":75.314999999999998},"last":4.407}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8380000000000001,"15min":1.0900000000000001},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.698,"5min":19.356000000000002,"15min":19.356000000000002},"last":4.2439999999999998},{"interface":"front","average":{"1min":1.125,"5min":1.845,"15min":1.042},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7109999999999999,"5min":19.324000000000002,"15min":19.324000000000002},"last":4.2290000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.8779999999999999,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4880000000000004,"5min":14.694000000000001,"15min":14.694000000000001},"last":3.577},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.958,"15min":1.222},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.2110000000000003,"5min":10.476000000000001,"15min":10.476000000000001},"last":0.39900000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.70999999999999996,"15min":0.57599999999999996},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3100000000000001,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.374},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92600000000000005,"15min":0.71599999999999997},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.7290000000000001,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.93799999999999994,"15min":0.74099999999999999},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":2.9990000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.81899999999999995,"15min":0.66600000000000004},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9700000000000002,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.442}]},{"osd":7,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.96999999999999997,"15min":0.69899999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.4890000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.004,"15min":0.76100000000000001},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.306,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.45500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724787,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990672,"kb_used_data":681096,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92754480,"statfs":{"total":95995035648,"available":94980587520,"internally_reserved":0,"allocated":697442304,"data_stored":696819376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":6.391,"5min":3.4849999999999999,"15min":1.577},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":83.329999999999998,"5min":83.329999999999998,"15min":83.329999999999998},"last":52.975999999999999},{"interface":"front","average":{"1min":7.7030000000000003,"5min":3.778,"15min":1.6990000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":85.451999999999998,"5min":85.451999999999998,"15min":85.451999999999998},"last":53.359000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:11:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3869999999999996,"5min":2.6749999999999998,"15min":1.7030000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":59.588999999999999,"5min":59.588999999999999,"15min":91.212999999999994},"last":0.44},{"interface":"front","average":{"1min":4.6139999999999999,"5min":2.9550000000000001,"15min":1.913},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":59.533000000000001,"5min":59.533000000000001,"15min":91.254000000000005},"last":0.41599999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":3.0880000000000001,"15min":1.524},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.2959999999999998,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.968,"5min":3.2050000000000001,"15min":1.518},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.6929999999999996,"5min":61.154000000000003,"15min":61.154000000000003},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.772,"15min":1.7010000000000001},"min":{"1min":0.35299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.714,"5min":17.071000000000002,"15min":91.834999999999994},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.548,"5min":1.885,"15min":1.7450000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.309},"max":{"1min":5.9749999999999996,"5min":17.052,"15min":91.802000000000007},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.88900000000000001,"15min":0.65700000000000003},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.253},"max":{"1min":3.2440000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.95599999999999996,"15min":0.71799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26600000000000001},"max":{"1min":3.4420000000000002,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.33400000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2839999999999998,"5min":2.1230000000000002,"15min":1.113},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":123.239,"5min":123.239,"15min":123.239},"last":0.34999999999999998},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.0899999999999999,"15min":1.129},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":123.014,"5min":123.014,"15min":123.014},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.98999999999999999,"15min":0.78300000000000003},"min":{"1min":0.307,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2450000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90300000000000002,"15min":0.80300000000000005},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":1.8220000000000001,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1723666432,"data_stored":1723662360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1379340288,"data_stored":1379336216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":924778496,"data_stored":924778496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1879506944,"data_stored":1879506944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":884932608,"data_stored":884932608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":685572096,"data_stored":685572096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1272451072,"data_stored":1272447000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1181876224,"data_stored":1181876224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:12:02.911 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:02.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:12:02.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:02 smithi181 ceph-mon[196979]: pgmap v564: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 2197/145449 objects degraded (1.510%); 5619/145449 objects misplaced (3.863%); 876 KiB/s, 13 objects/s recovering 2023-10-09T04:12:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:02 smithi181 ceph-mon[196979]: from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/145449 objects degraded (1.510%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:03.128 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:03.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:03.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:03.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:03.698 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:03.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:03.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:03.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:03.888 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:04.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:04.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:04.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:04.603 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:12:04.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:04.640 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:04.699 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:04.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:04.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[213841]: pgmap v565: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/141591 objects degraded (1.552%); 5385/141591 objects misplaced (3.803%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1975575673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:12:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:04.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[221067]: pgmap v565: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/141591 objects degraded (1.552%); 5385/141591 objects misplaced (3.803%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:04.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1975575673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:12:04.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:04.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:04.836 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:04.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:04.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:04 smithi181 ceph-mon[196979]: pgmap v565: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/141591 objects degraded (1.552%); 5385/141591 objects misplaced (3.803%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:04.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1975575673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:12:04.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:04.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:04.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:05.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:05.079 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:05.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:05.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:05.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:05.699 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:05.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:05.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:05.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[221067]: failed to encode map e216 with expected crc 2023-10-09T04:12:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1975575673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[221067]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[221067]: failed to encode map e216 with expected crc 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[213841]: failed to encode map e216 with expected crc 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1975575673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[213841]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T04:12:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:05 smithi163 ceph-mon[213841]: failed to encode map e216 with expected crc 2023-10-09T04:12:05.847 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:05.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:05 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:05.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:05 smithi181 ceph-mon[196979]: failed to encode map e216 with expected crc 2023-10-09T04:12:05.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1975575673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:12:05.881 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:05 smithi181 ceph-mon[196979]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T04:12:05.882 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:05 smithi181 ceph-mon[196979]: failed to encode map e216 with expected crc 2023-10-09T04:12:05.886 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:05.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:05.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:06.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:06.097 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:06.157 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:06.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:06.262 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:06.292 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:06.367 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:06.451 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:12:06.451 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:12:06.626 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:06 smithi163 ceph-mon[213841]: pgmap v567: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/137907 objects degraded (1.593%); 5385/137907 objects misplaced (3.905%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:06.734 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:06.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:06.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:06.878 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:06 smithi181 ceph-mon[196979]: pgmap v567: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/137907 objects degraded (1.593%); 5385/137907 objects misplaced (3.905%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:06.880 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:06.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:06.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:07.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:07.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:06 smithi163 ceph-mon[221067]: pgmap v567: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 2197/137907 objects degraded (1.593%); 5385/137907 objects misplaced (3.905%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:07.121 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:07.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:07.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:07.762 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:12:07.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:07.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:12:07.815 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:12:07.860 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:07.871 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:12:07.944 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:12:07.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/137907 objects degraded (1.593%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:07.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/137907 objects degraded (1.593%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:07.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:07.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:07.989 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:12:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/137907 objects degraded (1.593%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:08.034 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:12:08.068 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:12:08.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:08.100 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:08.113 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:12:08.130 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.130 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.130 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.130 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.131 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.131 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.131 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.131 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:12:08.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:08.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:08.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:12:08.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:08.354 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:08.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:08.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:08.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:08.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:08 smithi181 ceph-mon[196979]: pgmap v568: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 2197/134880 objects degraded (1.629%); 5084/134880 objects misplaced (3.769%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:08.994 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1924017866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:08.996 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:09.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:08 smithi163 ceph-mon[213841]: pgmap v568: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 2197/134880 objects degraded (1.629%); 5084/134880 objects misplaced (3.769%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:09.058 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1924017866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:09.058 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:09.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:08 smithi163 ceph-mon[221067]: pgmap v568: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 2197/134880 objects degraded (1.629%); 5084/134880 objects misplaced (3.769%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:12:09.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1924017866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:09.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:09.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:09.220 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:09.678 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:09.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:09.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:09.824 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:09.826 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:09.865 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:09.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:09.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:09.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:10.012 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:10.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:10.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:10.180 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:10.273 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:10.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:10.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:10.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:10.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:10.682 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:10 smithi163 ceph-mon[213841]: pgmap v569: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2197/129747 objects degraded (1.693%); 4823/129747 objects misplaced (3.717%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:10.683 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:12:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:12:10] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:12:10.684 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:10 smithi163 ceph-mon[221067]: pgmap v569: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2197/129747 objects degraded (1.693%); 4823/129747 objects misplaced (3.717%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:10.740 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:10.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:10.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:10.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:11.007 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:11.007 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2197/129747 objects degraded (1.693%), 1 pg degraded, 2 pgs undersized","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":695,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":216,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":8,"osd_in_since":1696823548,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":94},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":43249,"data_bytes":2833323960,"bytes_used":11674656768,"bytes_avail":756285628416,"bytes_total":767960285184,"degraded_objects":2197,"degraded_total":129747,"degraded_ratio":0.016932954133814269,"misplaced_objects":4823,"misplaced_total":129747,"misplaced_ratio":0.037172343098491677,"recovering_objects_per_sec":47,"recovering_bytes_per_sec":3109893,"recovering_keys_per_sec":0,"num_objects_recovered":475,"num_bytes_recovered":31129600,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":702},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":21,"modified":"2023-10-09T04:04:35.266858+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"61f8f0b7-6515-43d1-9684-78e0e9193886":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 6s)","progress":0.94949495792388916,"add_to_ceph_s":true}}} 2023-10-09T04:12:11.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:10 smithi181 ceph-mon[196979]: pgmap v569: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2197/129747 objects degraded (1.693%); 4823/129747 objects misplaced (3.717%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:11.044 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:11.075 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:11.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:11.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:11.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:11.509 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:11.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:11.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:11.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:11.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:12:11.669 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/373164828' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:12:11.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:11.704 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:11.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:11.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:11.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:11.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/373164828' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:12:11.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:11.882 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:11.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:11.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:12.041 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:12.065 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/373164828' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:12:12.067 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:12.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:12.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:12.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:12.763 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:12:12.763 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:12:12.813 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:12.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:12 smithi163 ceph-mon[213841]: pgmap v570: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2197/124368 objects degraded (1.767%); 4472/124368 objects misplaced (3.596%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:12.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/124368 objects degraded (1.767%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:12.904 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:12 smithi163 ceph-mon[221067]: pgmap v570: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2197/124368 objects degraded (1.767%); 4472/124368 objects misplaced (3.596%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:12.905 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/124368 objects degraded (1.767%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:12.927 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:12.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:12 smithi181 ceph-mon[196979]: pgmap v570: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2197/124368 objects degraded (1.767%); 4472/124368 objects misplaced (3.596%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/124368 objects degraded (1.767%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:13.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:13.074 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:13.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:13.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:13.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:13.257 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:13.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:13.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:13.395 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:13.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:13.552 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:12:13.552 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:12:13.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:12:13.552 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:12:13.553 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.553 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119029 6.11 6:89ce47bf:::benchmark_data_smithi163_287034_object10766:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.553 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:12:13.553 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 328.19527334600002, 2023-10-09T04:12:13.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.554635572, 2023-10-09T04:12:13.553 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119029 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.554 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335398+0000", 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335406+0000", 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7929999999999998e-06 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.363872+0000", 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028466115 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.363891+0000", 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8961e-05 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364339+0000", 2023-10-09T04:12:13.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000447935 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364350+0000", 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0750999999999999e-05 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.862665+0000", 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.498315641 2023-10-09T04:12:13.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.863639+0000", 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097311200000000004 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877198+0000", 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013559584 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.559 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877283+0000", 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5289000000000003e-05 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877425+0000", 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000141619 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.881660+0000", 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042347840000000001 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882652+0000", 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0009924109999999999 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882695+0000", 2023-10-09T04:12:13.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2449e-05 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.887923+0000", 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052283570000000003 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.887979+0000", 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5566000000000003e-05 2023-10-09T04:12:13.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.890035+0000", 2023-10-09T04:12:13.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002056745 2023-10-09T04:12:13.563 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.563 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.564 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.564 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119035 6.11 6:88144e9f:::benchmark_data_smithi163_287034_object10772:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.564 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:12:13.564 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 328.16652702800002, 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.535802179, 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.565 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119035 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:12:13.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364145+0000", 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364152+0000", 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9800000000000001e-06 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364162+0000", 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5899999999999997e-06 2023-10-09T04:12:13.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.570 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.570 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364169+0000", 2023-10-09T04:12:13.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3880000000000002e-06 2023-10-09T04:12:13.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364483+0000", 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031385899999999999 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364491+0000", 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.867872+0000", 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.503381112 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.871582+0000", 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037102459999999999 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.885871+0000", 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014288876000000001 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.886009+0000", 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013767400000000001 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.886197+0000", 2023-10-09T04:12:13.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188591 2023-10-09T04:12:13.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.890058+0000", 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038603420000000001 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897207+0000", 2023-10-09T04:12:13.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071496529999999997 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897263+0000", 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5933999999999997e-05 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897307+0000", 2023-10-09T04:12:13.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3574e-05 2023-10-09T04:12:13.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897386+0000", 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.928e-05 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.899948+0000", 2023-10-09T04:12:13.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002562174 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119089 6.11 6:8c811572:::benchmark_data_smithi163_287034_object10826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.580 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 328.00089824999998, 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.375641546, 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.581 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119089 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:12:13.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529777+0000", 2023-10-09T04:12:13.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.531e-06 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529784+0000", 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1779999999999997e-06 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529789+0000", 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0509999999999996e-06 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529844+0000", 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4928e-05 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529854+0000", 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9369999999999999e-06 2023-10-09T04:12:13.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.881692+0000", 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3518380210000001 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882588+0000", 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000895884 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895389+0000", 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012800420999999999 2023-10-09T04:12:13.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895468+0000", 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9363000000000006e-05 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895563+0000", 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5142000000000005e-05 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903262+0000", 2023-10-09T04:12:13.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076987920000000003 2023-10-09T04:12:13.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.592 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.592 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.592 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903275+0000", 2023-10-09T04:12:13.592 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2758999999999999e-05 2023-10-09T04:12:13.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903318+0000", 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3312000000000001e-05 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903378+0000", 2023-10-09T04:12:13.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0386000000000001e-05 2023-10-09T04:12:13.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903429+0000", 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0822000000000002e-05 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.905416+0000", 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019873859999999998 2023-10-09T04:12:13.595 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119092 6.11 6:883a032b:::benchmark_data_smithi163_287034_object10829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:12:13.596 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 328.000852049, 2023-10-09T04:12:13.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4310658030000001, 2023-10-09T04:12:13.597 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.597 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.597 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.597 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.597 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119092 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:12:13.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529820+0000", 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529825+0000", 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7060000000000003e-06 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529837+0000", 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1731e-05 2023-10-09T04:12:13.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529844+0000", 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6240000000000002e-06 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.602 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529878+0000", 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.392e-05 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529887+0000", 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7059999999999993e-06 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.900013+0000", 2023-10-09T04:12:13.603 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.370126255 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.902419+0000", 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024062950000000001 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.927730+0000", 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.025310295999999999 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.927860+0000", 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012995199999999999 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.928026+0000", 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016576500000000001 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.945127+0000", 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017101775 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958766+0000", 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013638477 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958829+0000", 2023-10-09T04:12:13.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3446000000000002e-05 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959082+0000", 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025267399999999999 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959123+0000", 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0979999999999997e-05 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960887+0000", 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00176406 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119103 6.11 6:8d75c45e:::benchmark_data_smithi163_287034_object10840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.98890259500001, 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4205783109999999, 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119103 2023-10-09T04:12:13.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541769+0000", 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541776+0000", 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1169999999999998e-06 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541795+0000", 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9114999999999999e-05 2023-10-09T04:12:13.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541807+0000", 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1369000000000001e-05 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.613 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541880+0000", 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3739000000000007e-05 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541891+0000", 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1107000000000001e-05 2023-10-09T04:12:13.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.902472+0000", 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.36058053 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903197+0000", 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072483200000000002 2023-10-09T04:12:13.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933631+0000", 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.030434745999999999 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933720+0000", 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8212000000000002e-05 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933838+0000", 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011806 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.945137+0000", 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011299208 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958878+0000", 2023-10-09T04:12:13.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013741547 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958932+0000", 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3193999999999999e-05 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960951+0000", 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020194599999999998 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960992+0000", 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1319000000000002e-05 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962349+0000", 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013564200000000001 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119123 6.11 6:892f37da:::benchmark_data_smithi163_287034_object10860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.622 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.97085243100003, 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.403942767, 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119123 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.623 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:12:13.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559819+0000", 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559825+0000", 2023-10-09T04:12:13.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2280000000000003e-06 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559842+0000", 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6141999999999999e-05 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559850+0000", 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9079999999999995e-06 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559902+0000", 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2420999999999998e-05 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559917+0000", 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5003e-05 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.905461+0000", 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.345543943 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.906296+0000", 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083475499999999996 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.938986+0000", 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032690364 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.939133+0000", 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014743800000000001 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.939247+0000", 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113915 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958704+0000", 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019456505999999998 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958968+0000", 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026427900000000002 2023-10-09T04:12:13.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959024+0000", 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5674000000000003e-05 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962395+0000", 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003370736 2023-10-09T04:12:13.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962431+0000", 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6906000000000002e-05 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963764+0000", 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00133204 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.633 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119141 6.11 6:8c64562a:::benchmark_data_smithi163_287034_object10878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.95557381200001, 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.396054788, 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.634 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119141 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:12:13.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:12:13.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575102+0000", 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9490000000000001e-06 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575108+0000", 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1140000000000002e-06 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575113+0000", 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2360000000000004e-06 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575184+0000", 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0494000000000006e-05 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575195+0000", 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1283e-05 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.908246+0000", 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.333050761 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.909908+0000", 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001662455 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949276+0000", 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039368001 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949574+0000", 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029827500000000003 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949695+0000", 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120931 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958714+0000", 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0090190619999999996 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963822+0000", 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051071249999999997 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963923+0000", 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101548 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.969034+0000", 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051108899999999999 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.969082+0000", 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8161999999999997e-05 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971154+0000", 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020719359999999999 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119147 6.11 6:8f3284c4:::benchmark_data_smithi163_287034_object10884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:12:13.645 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.94533227699998, 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3874358629999999, 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119147 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.646 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585339+0000", 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585345+0000", 2023-10-09T04:12:13.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.02e-06 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585357+0000", 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1371e-05 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585362+0000", 2023-10-09T04:12:13.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1259999999999997e-06 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585402+0000", 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0420999999999998e-05 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585413+0000", 2023-10-09T04:12:13.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0242e-05 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.911820+0000", 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.326407552 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.912798+0000", 2023-10-09T04:12:13.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097778699999999993 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.954865+0000", 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.042066631 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.954986+0000", 2023-10-09T04:12:13.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121303 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.955142+0000", 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015607500000000001 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959037+0000", 2023-10-09T04:12:13.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003894837 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963876+0000", 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048392319999999997 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963970+0000", 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4040000000000001e-05 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971217+0000", 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072466520000000001 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971271+0000", 2023-10-09T04:12:13.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4704999999999999e-05 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.972777+0000", 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001505337 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119163 6.11 6:8b083f99:::benchmark_data_smithi163_287034_object10900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.92916293500002, 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3956605739999999, 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.657 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119163 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601509+0000", 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601516+0000", 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3549999999999999e-06 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601528+0000", 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1885000000000001e-05 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601537+0000", 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9060000000000008e-06 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601581+0000", 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4168000000000001e-05 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601589+0000", 2023-10-09T04:12:13.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3240000000000003e-06 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.964003+0000", 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.362413941 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.965225+0000", 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012218419999999999 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977192+0000", 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011966976000000001 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977278+0000", 2023-10-09T04:12:13.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5663999999999998e-05 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977394+0000", 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011626 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.982063+0000", 2023-10-09T04:12:13.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046684430000000004 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995092+0000", 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013029099000000001 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995125+0000", 2023-10-09T04:12:13.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3803999999999997e-05 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995155+0000", 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9683999999999999e-05 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995201+0000", 2023-10-09T04:12:13.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6093000000000001e-05 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997171+0000", 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019696169999999999 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119188 6.11 6:8a56da9b:::benchmark_data_smithi163_287034_object10925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.90679373500001, 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3781419879999999, 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.668 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119188 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623878+0000", 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623884+0000", 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623900+0000", 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6373000000000001e-05 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623908+0000", 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1419999999999993e-06 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623965+0000", 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6709000000000002e-05 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623977+0000", 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2267e-05 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.965277+0000", 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3412998730000001 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968073+0000", 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027958570000000001 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986363+0000", 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018290041 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986467+0000", 2023-10-09T04:12:13.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010372199999999999 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986609+0000", 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014165400000000001 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997219+0000", 2023-10-09T04:12:13.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010610519000000001 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997253+0000", 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3803999999999997e-05 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997288+0000", 2023-10-09T04:12:13.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5064999999999997e-05 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000032+0000", 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027444190000000001 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000092+0000", 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9963000000000002e-05 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002021+0000", 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001929107 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119196 6.11 6:8fdab617:::benchmark_data_smithi163_287034_object10933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.89928996800001, 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3723983319999999, 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.679 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119196 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631382+0000", 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631387+0000", 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5260000000000004e-06 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631396+0000", 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0110000000000006e-06 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631402+0000", 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631439+0000", 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6520999999999998e-05 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631446+0000", 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4059999999999999e-06 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968116+0000", 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.33666961 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968985+0000", 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086886199999999998 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993266+0000", 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024281193999999999 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993347+0000", 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1148000000000003e-05 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993458+0000", 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111158 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000020+0000", 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065613119999999997 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002036+0000", 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020161950000000001 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002091+0000", 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5470999999999998e-05 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002144+0000", 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2332e-05 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002203+0000", 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9389999999999999e-05 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.003782+0000", 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015784429999999999 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.689 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119206 6.11 6:8f80b5b4:::benchmark_data_smithi163_287034_object10943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.89014330499998, 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.397280665, 2023-10-09T04:12:13.690 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119206 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640528+0000", 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640535+0000", 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0119999999999999e-06 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640550+0000", 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4673e-05 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640559+0000", 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.935e-06 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640615+0000", 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6740999999999998e-05 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640627+0000", 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.189e-05 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997311+0000", 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3566840010000001 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.998135+0000", 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082370499999999999 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012025+0000", 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013889755 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012139+0000", 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011416099999999999 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012292+0000", 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015364299999999999 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026026+0000", 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013733176999999999 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026062+0000", 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6235999999999997e-05 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032222+0000", 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0061599669999999997 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.036596+0000", 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043742829999999996 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.036661+0000", 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5092000000000006e-05 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037811+0000", 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001149279 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119231 6.11 6:8f1bfbf5:::benchmark_data_smithi163_287034_object10968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.85959214100001, 2023-10-09T04:12:13.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3681440409999999, 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119231 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671079+0000", 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671087+0000", 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5410000000000003e-06 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671103+0000", 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6531000000000001e-05 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671113+0000", 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7170000000000003e-06 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671174+0000", 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1267999999999997e-05 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671184+0000", 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9890000000000004e-06 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.998170+0000", 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.326985353 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999028+0000", 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000858163 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018202+0000", 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019174388000000001 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018330+0000", 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127269 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018487+0000", 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015788 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032196+0000", 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013708549 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032241+0000", 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5118999999999999e-05 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037827+0000", 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055862890000000004 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037934+0000", 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106397 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037980+0000", 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6433000000000001e-05 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.039225+0000", 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001244818 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119251 6.11 6:8bf159d8:::benchmark_data_smithi163_287034_object10988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:12:13.712 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.832612531, 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3497973839999999, 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119251 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698059+0000", 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698065+0000", 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.908e-06 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698078+0000", 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.296e-05 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698086+0000", 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7570000000000004e-06 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698139+0000", 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3701e-05 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698150+0000", 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0548e-05 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999079+0000", 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.30092889 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999966+0000", 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088695600000000005 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024113+0000", 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.02414738 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024224+0000", 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011063799999999999 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024372+0000", 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000148123 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032262+0000", 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078901960000000004 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037837+0000", 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055749579999999996 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.039272+0000", 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001435012 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.046029+0000", 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067568610000000003 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.046059+0000", 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9821000000000001e-05 2023-10-09T04:12:13.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.047858+0000", 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00179923 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.723 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119258 6.11 6:89e8583e:::benchmark_data_smithi163_287034_object10995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.822681499, 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3578773550000001, 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119258 2023-10-09T04:12:13.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:12:13.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707990+0000", 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707997+0000", 2023-10-09T04:12:13.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8560000000000001e-06 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708014+0000", 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6804999999999999e-05 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708021+0000", 2023-10-09T04:12:13.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0180000000000004e-06 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708074+0000", 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2914999999999997e-05 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708088+0000", 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3818999999999999e-05 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026085+0000", 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3179974219999999 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026910+0000", 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082452299999999997 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050702+0000", 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023792316000000001 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050818+0000", 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116173 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050970+0000", 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015151300000000001 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060167+0000", 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091971970000000007 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060243+0000", 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6215000000000001e-05 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060374+0000", 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000130894 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.063837+0000", 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003463224 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.063891+0000", 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4011999999999997e-05 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.065869+0000", 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001977768 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119295 6.11 6:8feb1e91:::benchmark_data_smithi163_287034_object11032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 327.704979939, 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2504799099999999, 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.735 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119295 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825692+0000", 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825700+0000", 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1559999999999995e-06 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825719+0000", 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9177999999999998e-05 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825727+0000", 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4630000000000001e-06 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825788+0000", 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0730999999999998e-05 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825803+0000", 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4347e-05 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026947+0000", 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2011442640000001 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.027704+0000", 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00075679299999999996 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.056255+0000", 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028550981 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.057403+0000", 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011480500000000001 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.057552+0000", 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014943500000000001 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060380+0000", 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002827867 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.068125+0000", 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0077448899999999999 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.069348+0000", 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001223285 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.074303+0000", 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049545429999999996 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.074377+0000", 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4524999999999995e-05 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.076173+0000", 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017959460000000001 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80474 7.2 7:413fef99:::benchmark_data_smithi163_346082_object80473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.449723+0000", 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 96.080950279999996, 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97738634599999996, 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.746 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80474 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449723+0000", 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449723+0000", 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449721+0000", 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449750+0000", 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8515999999999999e-05 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449768+0000", 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7963000000000001e-05 2023-10-09T04:12:13.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449773+0000", 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4280000000000004e-06 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449833+0000", 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9419999999999997e-05 2023-10-09T04:12:13.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450001+0000", 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016866599999999999 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450183+0000", 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018147700000000001 2023-10-09T04:12:13.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.460339+0000", 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010156505 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.460355+0000", 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5838e-05 2023-10-09T04:12:13.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.410485+0000", 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.950129377 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.411250+0000", 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000765616 2023-10-09T04:12:13.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.427003+0000", 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01575269 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.427109+0000", 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010626800000000001 2023-10-09T04:12:13.754 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80490 7.2 7:445fadfb:::benchmark_data_smithi163_346082_object80489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.450738+0000", 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 96.079935164000005, 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97800665600000003, 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.755 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80490 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450738+0000", 2023-10-09T04:12:13.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450738+0000", 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450737+0000", 2023-10-09T04:12:13.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450794+0000", 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7027000000000001e-05 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450802+0000", 2023-10-09T04:12:13.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0530000000000001e-06 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450807+0000", 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9710000000000001e-06 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.451719+0000", 2023-10-09T04:12:13.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00091187600000000005 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.451838+0000", 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118956 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:12:13.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.451989+0000", 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015018099999999999 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472667+0000", 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020678228 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472677+0000", 2023-10-09T04:12:13.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0037999999999999e-05 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.412196+0000", 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.93951964399999999 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.412822+0000", 2023-10-09T04:12:13.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062551600000000001 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428634+0000", 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01581221 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428745+0000", 2023-10-09T04:12:13.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110512 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80507 7.2 7:421785fa:::benchmark_data_smithi163_346082_object80506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.466106+0000", 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 96.064566924000005, 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96350955699999996, 2023-10-09T04:12:13.764 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80507 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466106+0000", 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466106+0000", 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466105+0000", 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466135+0000", 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0056000000000002e-05 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466144+0000", 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9320000000000002e-06 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466150+0000", 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2009999999999996e-06 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472699+0000", 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065491109999999998 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472704+0000", 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5940000000000004e-06 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.412881+0000", 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94017724199999997 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.413037+0000", 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000156253 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.413175+0000", 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013784299999999999 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.417403+0000", 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042277529999999999 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.420553+0000", 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031505410000000002 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428790+0000", 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082364510000000005 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428843+0000", 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2750999999999997e-05 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.429616+0000", 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077321300000000003 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80519 7.2 7:42c84ec1:::benchmark_data_smithi163_346082_object80518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.483278+0000", 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 96.047395625999997, 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94722626799999998, 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:12:13.774 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80519 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483278+0000", 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483278+0000", 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483276+0000", 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483297+0000", 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0234e-05 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483308+0000", 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.165e-05 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483314+0000", 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0290000000000002e-06 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483363+0000", 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9240999999999998e-05 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483374+0000", 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0234000000000001e-05 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.414927+0000", 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.93155327300000002 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.415090+0000", 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016282499999999999 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.415295+0000", 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020525900000000001 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.418308+0000", 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003013253 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.420597+0000", 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022882129999999999 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.429667+0000", 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.009070013 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.429718+0000", 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1289000000000003e-05 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:13.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.430504+0000", 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078599099999999999 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:13.783 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:12:13.784 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:13.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:13 smithi163 ceph-mon[213841]: pgmap v571: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2197/120618 objects degraded (1.821%); 4185/120618 objects misplaced (3.470%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:13.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:13 smithi163 ceph-mon[221067]: pgmap v571: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2197/120618 objects degraded (1.821%); 4185/120618 objects misplaced (3.470%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:14.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:13 smithi181 ceph-mon[196979]: pgmap v571: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 2197/120618 objects degraded (1.821%); 4185/120618 objects misplaced (3.470%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:14.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:14.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:14.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:14.219 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:14.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:14.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:14.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:14.395 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:14.803 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:12:14.945 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:12:15.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:15.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2947074510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:15.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:15.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:14 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:15.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2947074510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:15.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:15.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:14 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:15.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:15.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:15.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2947074510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:15.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:15.170 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:14 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:15.172 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:15.226 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:15.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:15.332 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:15.360 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:15.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:15.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:15.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:15.526 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:15.593 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:15.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:15.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T04:12:15.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[221067]: pgmap v573: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 2197/116994 objects degraded (1.878%); 4185/116994 objects misplaced (3.577%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.855 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.855 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.855 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: failed to encode map e217 with expected crc 2023-10-09T04:12:15.855 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:15 smithi163 ceph-mon[213841]: pgmap v573: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 2197/116994 objects degraded (1.878%); 4185/116994 objects misplaced (3.577%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T04:12:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: failed to encode map e217 with expected crc 2023-10-09T04:12:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: failed to encode map e217 with expected crc 2023-10-09T04:12:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: failed to encode map e217 with expected crc 2023-10-09T04:12:16.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: failed to encode map e217 with expected crc 2023-10-09T04:12:16.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: failed to encode map e217 with expected crc 2023-10-09T04:12:16.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:15 smithi181 ceph-mon[196979]: pgmap v573: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 2197/116994 objects degraded (1.878%); 4185/116994 objects misplaced (3.577%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:16.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:16.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:16.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:16.497 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:16.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:16.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:16.626 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:16.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:16.717 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:16.720 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:16.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:16.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:16.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:16.976 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:17.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:17.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:17.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:17.173 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:17.224 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:17.227 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:12:17.229 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/79150373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:17.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:17.272 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:12:17.327 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:12:17.331 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:17.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/79150373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:17.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/79150373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:17.361 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:17.377 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:12:17.414 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:17.415 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:12:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:17.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:17.470 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:12:17.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:17.527 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:12:17.563 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:12:17.585 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.586 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:12:17.636 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:18.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:12:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:18 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2197/116994 objects degraded (1.878%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:18 smithi181 ceph-mon[196979]: pgmap v574: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2197/113670 objects degraded (1.933%); 3848/113670 objects misplaced (3.385%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:18 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2197/116994 objects degraded (1.878%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:18.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:18 smithi163 ceph-mon[221067]: pgmap v574: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2197/113670 objects degraded (1.933%); 3848/113670 objects misplaced (3.385%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:18 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2197/116994 objects degraded (1.878%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:18.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:18 smithi163 ceph-mon[213841]: pgmap v574: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 2197/113670 objects degraded (1.933%); 3848/113670 objects misplaced (3.385%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:12:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[221067]: osdmap e218: 8 total, 8 up, 8 in 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[221067]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[221067]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[221067]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[213841]: osdmap e218: 8 total, 8 up, 8 in 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[213841]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[213841]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[213841]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:19.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:19 smithi181 ceph-mon[196979]: osdmap e218: 8 total, 8 up, 8 in 2023-10-09T04:12:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:19 smithi181 ceph-mon[196979]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:19 smithi181 ceph-mon[196979]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:19 smithi181 ceph-mon[196979]: failed to encode map e218 with expected crc 2023-10-09T04:12:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:19.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:19.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:19.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:19.957 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:20.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:20.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:20.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:20.389 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:12:20.389 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:12:20.393 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:20.398 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:20.456 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:20.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:20.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:20.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[213841]: osdmap e219: 8 total, 8 up, 8 in 2023-10-09T04:12:20.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[213841]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[213841]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[213841]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[213841]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[213841]: pgmap v577: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2197/108156 objects degraded (2.031%); 3506/108156 objects misplaced (3.242%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:12:20.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[221067]: osdmap e219: 8 total, 8 up, 8 in 2023-10-09T04:12:20.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[221067]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[221067]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[221067]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[221067]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:20 smithi163 ceph-mon[221067]: pgmap v577: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2197/108156 objects degraded (2.031%); 3506/108156 objects misplaced (3.242%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:12:20.606 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:20 smithi181 ceph-mon[196979]: osdmap e219: 8 total, 8 up, 8 in 2023-10-09T04:12:20.606 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:20 smithi181 ceph-mon[196979]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.606 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:20 smithi181 ceph-mon[196979]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.607 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:20 smithi181 ceph-mon[196979]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.607 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:20 smithi181 ceph-mon[196979]: failed to encode map e219 with expected crc 2023-10-09T04:12:20.607 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:20 smithi181 ceph-mon[196979]: pgmap v577: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2197/108156 objects degraded (2.031%); 3506/108156 objects misplaced (3.242%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:12:20.607 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:20.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:20.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:20.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:20.840 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:20.872 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:20.874 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:12:20.917 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:12:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:12:20] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-10-09T04:12:21.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:21.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:21.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:21.666 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:21.715 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:21.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:21.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:21.920 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:21.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:12:22.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:12:22.214 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":577,"stamp":"2023-10-09T04:12:19.485893+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2361661368,"num_objects":36052,"num_object_clones":0,"num_object_copies":108156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":3506,"num_objects_unfound":0,"num_objects_dirty":36052,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":144369,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11881,"num_bytes_recovered":779750068,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164219,"ondisk_log_size":164219,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10095860,"kb_used_data":7653108,"kb_used_omap":126,"kb_used_meta":2442049,"kb_avail":739865356,"statfs":{"total":767960285184,"available":757622124544,"internally_reserved":0,"allocated":7836782592,"data_stored":7830754950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2500658811},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,1,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-272236544,"num_objects":-4154,"num_object_clones":0,"num_object_copies":-12462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-679,"num_objects_unfound":0,"num_objects_dirty":-4154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4154,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002828"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":214,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615434+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:12:04.615434+0000","last_peered":"2023-10-09T04:12:04.615434+0000","last_clean":"2023-10-09T04:12:04.615434+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:12:04.615434+0000","last_undegraded":"2023-10-09T04:12:04.615434+0000","last_fullsized":"2023-10-09T04:12:04.615434+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.797988+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:12:14.797988+0000","last_peered":"2023-10-09T04:12:14.797988+0000","last_clean":"2023-10-09T04:12:14.797988+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:12:14.797988+0000","last_undegraded":"2023-10-09T04:12:14.797988+0000","last_fullsized":"2023-10-09T04:12:14.797988+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":169,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.797704+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:12:14.797704+0000","last_peered":"2023-10-09T04:12:14.797704+0000","last_clean":"2023-10-09T04:12:14.797704+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:14.797704+0000","last_undegraded":"2023-10-09T04:12:14.797704+0000","last_fullsized":"2023-10-09T04:12:14.797704+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":173,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631461+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:12:04.631461+0000","last_peered":"2023-10-09T04:12:04.631461+0000","last_clean":"2023-10-09T04:12:04.631461+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:12:04.631461+0000","last_undegraded":"2023-10-09T04:12:04.631461+0000","last_fullsized":"2023-10-09T04:12:04.631461+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":171,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.630774+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:12:04.630774+0000","last_peered":"2023-10-09T04:12:04.630774+0000","last_clean":"2023-10-09T04:12:04.630774+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:12:04.630774+0000","last_undegraded":"2023-10-09T04:12:04.630774+0000","last_fullsized":"2023-10-09T04:12:04.630774+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":150,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.602188+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:12:04.602188+0000","last_peered":"2023-10-09T04:12:04.602188+0000","last_clean":"2023-10-09T04:12:04.602188+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:12:04.602188+0000","last_undegraded":"2023-10-09T04:12:04.602188+0000","last_fullsized":"2023-10-09T04:12:04.602188+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"217'5834","reported_seq":8790,"reported_epoch":217,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:15.700463+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:12:15.700463+0000","last_peered":"2023-10-09T04:12:15.700463+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:12:15.700463+0000","last_undegraded":"2023-10-09T04:12:15.700463+0000","last_fullsized":"2023-10-09T04:12:15.700463+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5834,"log_dups_size":0,"ondisk_log_size":5834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728000,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1125,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4663,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1125}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":172,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.637473+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:12:04.637473+0000","last_peered":"2023-10-09T04:12:04.637473+0000","last_clean":"2023-10-09T04:12:04.637473+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:12:04.637473+0000","last_undegraded":"2023-10-09T04:12:04.637473+0000","last_fullsized":"2023-10-09T04:12:04.637473+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.619542+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:12:04.619542+0000","last_peered":"2023-10-09T04:12:04.619542+0000","last_clean":"2023-10-09T04:12:04.619542+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:12:04.619542+0000","last_undegraded":"2023-10-09T04:12:04.619542+0000","last_fullsized":"2023-10-09T04:12:04.619542+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"217'5989","reported_seq":13367,"reported_epoch":217,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:12:18.078451+0000","last_change":"2023-10-09T04:10:51.646576+0000","last_active":"2023-10-09T04:12:18.078451+0000","last_peered":"2023-10-09T04:12:18.078451+0000","last_clean":"2023-10-09T04:09:59.283093+0000","last_became_active":"2023-10-09T04:10:40.892310+0000","last_became_peered":"2023-10-09T04:10:40.892310+0000","last_unstale":"2023-10-09T04:12:18.078451+0000","last_undegraded":"2023-10-09T04:12:18.078451+0000","last_fullsized":"2023-10-09T04:10:40.864699+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5989,"log_dups_size":0,"ondisk_log_size":5989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74055680,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4824,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2549,"num_bytes_recovered":167051264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["0","1","3","6"],"object_location_counts":[{"shards":"0,1,6,7","objects":4},{"shards":"0,6","objects":1126}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":176,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.603467+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:12:04.603467+0000","last_peered":"2023-10-09T04:12:04.603467+0000","last_clean":"2023-10-09T04:12:04.603467+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:12:04.603467+0000","last_undegraded":"2023-10-09T04:12:04.603467+0000","last_fullsized":"2023-10-09T04:12:04.603467+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":113,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.612572+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:12:04.612572+0000","last_peered":"2023-10-09T04:12:04.612572+0000","last_clean":"2023-10-09T04:12:04.612572+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:12:04.612572+0000","last_undegraded":"2023-10-09T04:12:04.612572+0000","last_fullsized":"2023-10-09T04:12:04.612572+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"217'5752","reported_seq":8639,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:15.567890+0000","last_change":"2023-10-09T04:10:18.287893+0000","last_active":"2023-10-09T04:12:15.567890+0000","last_peered":"2023-10-09T04:12:15.567890+0000","last_clean":"2023-10-09T04:12:15.567890+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:12:15.567890+0000","last_undegraded":"2023-10-09T04:12:15.567890+0000","last_fullsized":"2023-10-09T04:12:15.567890+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4617,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":179,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.612979+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:12:04.612979+0000","last_peered":"2023-10-09T04:12:04.612979+0000","last_clean":"2023-10-09T04:12:04.612979+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:12:04.612979+0000","last_undegraded":"2023-10-09T04:12:04.612979+0000","last_fullsized":"2023-10-09T04:12:04.612979+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":170,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.800709+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:12:14.800709+0000","last_peered":"2023-10-09T04:12:14.800709+0000","last_clean":"2023-10-09T04:12:14.800709+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:12:14.800709+0000","last_undegraded":"2023-10-09T04:12:14.800709+0000","last_fullsized":"2023-10-09T04:12:14.800709+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"217'5904","reported_seq":8842,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.779721+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:12:14.779721+0000","last_peered":"2023-10-09T04:12:14.779721+0000","last_clean":"2023-10-09T04:12:14.779721+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:12:14.779721+0000","last_undegraded":"2023-10-09T04:12:14.779721+0000","last_fullsized":"2023-10-09T04:12:14.779721+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5904,"log_dups_size":0,"ondisk_log_size":5904,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4715,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":181,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.618364+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:12:04.618364+0000","last_peered":"2023-10-09T04:12:04.618364+0000","last_clean":"2023-10-09T04:12:04.618364+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:12:04.618364+0000","last_undegraded":"2023-10-09T04:12:04.618364+0000","last_fullsized":"2023-10-09T04:12:04.618364+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":175,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615728+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:12:04.615728+0000","last_peered":"2023-10-09T04:12:04.615728+0000","last_clean":"2023-10-09T04:12:04.615728+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:12:04.615728+0000","last_undegraded":"2023-10-09T04:12:04.615728+0000","last_fullsized":"2023-10-09T04:12:04.615728+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"217'5889","reported_seq":5959,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.012630+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:12:18.012630+0000","last_peered":"2023-10-09T04:12:18.012630+0000","last_clean":"2023-10-09T04:12:18.012630+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:12:18.012630+0000","last_undegraded":"2023-10-09T04:12:18.012630+0000","last_fullsized":"2023-10-09T04:12:18.012630+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5889,"log_dups_size":0,"ondisk_log_size":5889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72679424,"num_objects":1109,"num_object_clones":0,"num_object_copies":3327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4735,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":178,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.612689+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:12:04.612689+0000","last_peered":"2023-10-09T04:12:04.612689+0000","last_clean":"2023-10-09T04:12:04.612689+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:12:04.612689+0000","last_undegraded":"2023-10-09T04:12:04.612689+0000","last_fullsized":"2023-10-09T04:12:04.612689+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":154,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.798919+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:12:14.798919+0000","last_peered":"2023-10-09T04:12:14.798919+0000","last_clean":"2023-10-09T04:12:14.798919+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:12:14.798919+0000","last_undegraded":"2023-10-09T04:12:14.798919+0000","last_fullsized":"2023-10-09T04:12:14.798919+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"217'5712","reported_seq":5781,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.072782+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:12:18.072782+0000","last_peered":"2023-10-09T04:12:18.072782+0000","last_clean":"2023-10-09T04:12:18.072782+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:12:18.072782+0000","last_undegraded":"2023-10-09T04:12:18.072782+0000","last_fullsized":"2023-10-09T04:12:18.072782+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68845568,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4595,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.636856+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:12:04.636856+0000","last_peered":"2023-10-09T04:12:04.636856+0000","last_clean":"2023-10-09T04:12:04.636856+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:12:04.636856+0000","last_undegraded":"2023-10-09T04:12:04.636856+0000","last_fullsized":"2023-10-09T04:12:04.636856+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":175,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601836+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:12:04.601836+0000","last_peered":"2023-10-09T04:12:04.601836+0000","last_clean":"2023-10-09T04:12:04.601836+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:12:04.601836+0000","last_undegraded":"2023-10-09T04:12:04.601836+0000","last_fullsized":"2023-10-09T04:12:04.601836+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"217'5818","reported_seq":8764,"reported_epoch":217,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:15.500552+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:12:15.500552+0000","last_peered":"2023-10-09T04:12:15.500552+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:12:15.500552+0000","last_undegraded":"2023-10-09T04:12:15.500552+0000","last_fullsized":"2023-10-09T04:12:15.500552+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5818,"log_dups_size":0,"ondisk_log_size":5818,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78675968,"num_objects":1201,"num_object_clones":0,"num_object_copies":3603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1201,"num_objects_unfound":0,"num_objects_dirty":1201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4703,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1201}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":159,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.612558+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:12:04.612558+0000","last_peered":"2023-10-09T04:12:04.612558+0000","last_clean":"2023-10-09T04:12:04.612558+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:12:04.612558+0000","last_undegraded":"2023-10-09T04:12:04.612558+0000","last_fullsized":"2023-10-09T04:12:04.612558+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631491+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:12:04.631491+0000","last_peered":"2023-10-09T04:12:04.631491+0000","last_clean":"2023-10-09T04:12:04.631491+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:12:04.631491+0000","last_undegraded":"2023-10-09T04:12:04.631491+0000","last_fullsized":"2023-10-09T04:12:04.631491+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"217'5901","reported_seq":5971,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.017441+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:12:18.017441+0000","last_peered":"2023-10-09T04:12:18.017441+0000","last_clean":"2023-10-09T04:12:18.017441+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:12:18.017441+0000","last_undegraded":"2023-10-09T04:12:18.017441+0000","last_fullsized":"2023-10-09T04:12:18.017441+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5901,"log_dups_size":0,"ondisk_log_size":5901,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4739,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":108,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.630609+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:12:04.630609+0000","last_peered":"2023-10-09T04:12:04.630609+0000","last_clean":"2023-10-09T04:12:04.630609+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:12:04.630609+0000","last_undegraded":"2023-10-09T04:12:04.630609+0000","last_fullsized":"2023-10-09T04:12:04.630609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":181,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.627844+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:12:04.627844+0000","last_peered":"2023-10-09T04:12:04.627844+0000","last_clean":"2023-10-09T04:12:04.627844+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:12:04.627844+0000","last_undegraded":"2023-10-09T04:12:04.627844+0000","last_fullsized":"2023-10-09T04:12:04.627844+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"218'9474","reported_seq":9535,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.637281+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:12:18.637281+0000","last_peered":"2023-10-09T04:12:18.637281+0000","last_clean":"2023-10-09T04:12:18.637281+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:12:18.637281+0000","last_undegraded":"2023-10-09T04:12:18.637281+0000","last_fullsized":"2023-10-09T04:12:18.637281+0000","mapping_epoch":167,"log_start":"175'2400","ondisk_log_start":"175'2400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144310272,"num_objects":2202,"num_object_clones":0,"num_object_copies":6606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9474,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":177,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613208+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:12:04.613208+0000","last_peered":"2023-10-09T04:12:04.613208+0000","last_clean":"2023-10-09T04:12:04.613208+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:12:04.613208+0000","last_undegraded":"2023-10-09T04:12:04.613208+0000","last_fullsized":"2023-10-09T04:12:04.613208+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":158,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601331+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:12:04.601331+0000","last_peered":"2023-10-09T04:12:04.601331+0000","last_clean":"2023-10-09T04:12:04.601331+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:12:04.601331+0000","last_undegraded":"2023-10-09T04:12:04.601331+0000","last_fullsized":"2023-10-09T04:12:04.601331+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"217'9265","reported_seq":9325,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.932726+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:12:14.932726+0000","last_peered":"2023-10-09T04:12:14.932726+0000","last_clean":"2023-10-09T04:12:14.932726+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:12:14.932726+0000","last_undegraded":"2023-10-09T04:12:14.932726+0000","last_fullsized":"2023-10-09T04:12:14.932726+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":9265,"log_dups_size":0,"ondisk_log_size":9265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150536192,"num_objects":2297,"num_object_clones":0,"num_object_copies":6891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9265,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":178,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.612875+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:12:04.612875+0000","last_peered":"2023-10-09T04:12:04.612875+0000","last_clean":"2023-10-09T04:12:04.612875+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:12:04.612875+0000","last_undegraded":"2023-10-09T04:12:04.612875+0000","last_fullsized":"2023-10-09T04:12:04.612875+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":192,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615387+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:12:04.615387+0000","last_peered":"2023-10-09T04:12:04.615387+0000","last_clean":"2023-10-09T04:12:04.615387+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:12:04.615387+0000","last_undegraded":"2023-10-09T04:12:04.615387+0000","last_fullsized":"2023-10-09T04:12:04.615387+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"217'5960","reported_seq":6137,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.064709+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:12:18.064709+0000","last_peered":"2023-10-09T04:12:18.064709+0000","last_clean":"2023-10-09T04:12:18.064709+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:12:18.064709+0000","last_undegraded":"2023-10-09T04:12:18.064709+0000","last_fullsized":"2023-10-09T04:12:18.064709+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":5960,"log_dups_size":0,"ondisk_log_size":5960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3574,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":160,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.612876+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:12:04.612876+0000","last_peered":"2023-10-09T04:12:04.612876+0000","last_clean":"2023-10-09T04:12:04.612876+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:12:04.612876+0000","last_undegraded":"2023-10-09T04:12:04.612876+0000","last_fullsized":"2023-10-09T04:12:04.612876+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":137,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.800501+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:12:14.800501+0000","last_peered":"2023-10-09T04:12:14.800501+0000","last_clean":"2023-10-09T04:12:14.800501+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:12:14.800501+0000","last_undegraded":"2023-10-09T04:12:14.800501+0000","last_fullsized":"2023-10-09T04:12:14.800501+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"217'5710","reported_seq":5784,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.993670+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:12:14.993670+0000","last_peered":"2023-10-09T04:12:14.993670+0000","last_clean":"2023-10-09T04:12:14.993670+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:12:14.993670+0000","last_undegraded":"2023-10-09T04:12:14.993670+0000","last_fullsized":"2023-10-09T04:12:14.993670+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5710,"log_dups_size":0,"ondisk_log_size":5710,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76152832,"num_objects":1162,"num_object_clones":0,"num_object_copies":3486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4570,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":176,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601593+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:12:04.601593+0000","last_peered":"2023-10-09T04:12:04.601593+0000","last_clean":"2023-10-09T04:12:04.601593+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:12:04.601593+0000","last_undegraded":"2023-10-09T04:12:04.601593+0000","last_fullsized":"2023-10-09T04:12:04.601593+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":172,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631428+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:12:04.631428+0000","last_peered":"2023-10-09T04:12:04.631428+0000","last_clean":"2023-10-09T04:12:04.631428+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:12:04.631428+0000","last_undegraded":"2023-10-09T04:12:04.631428+0000","last_fullsized":"2023-10-09T04:12:04.631428+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"217'9537","reported_seq":9597,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.068090+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:12:18.068090+0000","last_peered":"2023-10-09T04:12:18.068090+0000","last_clean":"2023-10-09T04:12:18.068090+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:12:18.068090+0000","last_undegraded":"2023-10-09T04:12:18.068090+0000","last_fullsized":"2023-10-09T04:12:18.068090+0000","mapping_epoch":167,"log_start":"172'2000","ondisk_log_start":"172'2000","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7537,"log_dups_size":0,"ondisk_log_size":7537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139264000,"num_objects":2125,"num_object_clones":0,"num_object_copies":6375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":121,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631353+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:12:04.631353+0000","last_peered":"2023-10-09T04:12:04.631353+0000","last_clean":"2023-10-09T04:12:04.631353+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:12:04.631353+0000","last_undegraded":"2023-10-09T04:12:04.631353+0000","last_fullsized":"2023-10-09T04:12:04.631353+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":175,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.618296+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:12:04.618296+0000","last_peered":"2023-10-09T04:12:04.618296+0000","last_clean":"2023-10-09T04:12:04.618296+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:12:04.618296+0000","last_undegraded":"2023-10-09T04:12:04.618296+0000","last_fullsized":"2023-10-09T04:12:04.618296+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"217'9272","reported_seq":9348,"reported_epoch":217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:12:15.509107+0000","last_change":"2023-10-09T04:10:38.444053+0000","last_active":"2023-10-09T04:12:15.509107+0000","last_peered":"2023-10-09T04:12:15.509107+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:12:15.509107+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":9272,"log_dups_size":0,"ondisk_log_size":9272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153747456,"num_objects":2346,"num_object_clones":0,"num_object_copies":7038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9272,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1534},{"shards":"0,5","objects":149}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":136,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615529+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:12:04.615529+0000","last_peered":"2023-10-09T04:12:04.615529+0000","last_clean":"2023-10-09T04:12:04.615529+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:12:04.615529+0000","last_undegraded":"2023-10-09T04:12:04.615529+0000","last_fullsized":"2023-10-09T04:12:04.615529+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":212,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615373+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:12:04.615373+0000","last_peered":"2023-10-09T04:12:04.615373+0000","last_clean":"2023-10-09T04:12:04.615373+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:12:04.615373+0000","last_undegraded":"2023-10-09T04:12:04.615373+0000","last_fullsized":"2023-10-09T04:12:04.615373+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"218'9464","reported_seq":9526,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.669028+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:12:18.669028+0000","last_peered":"2023-10-09T04:12:18.669028+0000","last_clean":"2023-10-09T04:12:18.669028+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:12:18.669028+0000","last_undegraded":"2023-10-09T04:12:18.669028+0000","last_fullsized":"2023-10-09T04:12:18.669028+0000","mapping_epoch":167,"log_start":"175'2400","ondisk_log_start":"175'2400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141230104,"num_objects":2156,"num_object_clones":0,"num_object_copies":6468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2156,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9464,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":176,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615196+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:12:04.615196+0000","last_peered":"2023-10-09T04:12:04.615196+0000","last_clean":"2023-10-09T04:12:04.615196+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:12:04.615196+0000","last_undegraded":"2023-10-09T04:12:04.615196+0000","last_fullsized":"2023-10-09T04:12:04.615196+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":168,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615838+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:12:04.615838+0000","last_peered":"2023-10-09T04:12:04.615838+0000","last_clean":"2023-10-09T04:12:04.615838+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:12:04.615838+0000","last_undegraded":"2023-10-09T04:12:04.615838+0000","last_fullsized":"2023-10-09T04:12:04.615838+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"217'9154","reported_seq":9214,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:15.780110+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:12:15.780110+0000","last_peered":"2023-10-09T04:12:15.780110+0000","last_clean":"2023-10-09T04:12:15.780110+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:12:15.780110+0000","last_undegraded":"2023-10-09T04:12:15.780110+0000","last_fullsized":"2023-10-09T04:12:15.780110+0000","mapping_epoch":167,"log_start":"176'2700","ondisk_log_start":"176'2700","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6454,"log_dups_size":0,"ondisk_log_size":6454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149159936,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9154,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":183,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.795847+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:12:14.795847+0000","last_peered":"2023-10-09T04:12:14.795847+0000","last_clean":"2023-10-09T04:12:14.795847+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:12:14.795847+0000","last_undegraded":"2023-10-09T04:12:14.795847+0000","last_fullsized":"2023-10-09T04:12:14.795847+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":170,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.797729+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:12:14.797729+0000","last_peered":"2023-10-09T04:12:14.797729+0000","last_clean":"2023-10-09T04:12:14.797729+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:12:14.797729+0000","last_undegraded":"2023-10-09T04:12:14.797729+0000","last_fullsized":"2023-10-09T04:12:14.797729+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"217'5819","reported_seq":6011,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:15.187959+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:12:15.187959+0000","last_peered":"2023-10-09T04:12:15.187959+0000","last_clean":"2023-10-09T04:12:15.187959+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:12:15.187959+0000","last_undegraded":"2023-10-09T04:12:15.187959+0000","last_fullsized":"2023-10-09T04:12:15.187959+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":5819,"log_dups_size":0,"ondisk_log_size":5819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77266944,"num_objects":1179,"num_object_clones":0,"num_object_copies":3537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2166,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":193,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.798997+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:12:14.798997+0000","last_peered":"2023-10-09T04:12:14.798997+0000","last_clean":"2023-10-09T04:12:14.798997+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:12:14.798997+0000","last_undegraded":"2023-10-09T04:12:14.798997+0000","last_fullsized":"2023-10-09T04:12:14.798997+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":193,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601906+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:12:04.601906+0000","last_peered":"2023-10-09T04:12:04.601906+0000","last_clean":"2023-10-09T04:12:04.601906+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:04.601906+0000","last_undegraded":"2023-10-09T04:12:04.601906+0000","last_fullsized":"2023-10-09T04:12:04.601906+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":155,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.800620+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:12:14.800620+0000","last_peered":"2023-10-09T04:12:14.800620+0000","last_clean":"2023-10-09T04:12:14.800620+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:12:14.800620+0000","last_undegraded":"2023-10-09T04:12:14.800620+0000","last_fullsized":"2023-10-09T04:12:14.800620+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"217'5943","reported_seq":6013,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:17.981689+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:12:17.981689+0000","last_peered":"2023-10-09T04:12:17.981689+0000","last_clean":"2023-10-09T04:12:17.981689+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:12:17.981689+0000","last_undegraded":"2023-10-09T04:12:17.981689+0000","last_fullsized":"2023-10-09T04:12:17.981689+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69271552,"num_objects":1057,"num_object_clones":0,"num_object_copies":3171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4789,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":180,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.602876+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:12:04.602876+0000","last_peered":"2023-10-09T04:12:04.602876+0000","last_clean":"2023-10-09T04:12:04.602876+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:12:04.602876+0000","last_undegraded":"2023-10-09T04:12:04.602876+0000","last_fullsized":"2023-10-09T04:12:04.602876+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":176,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615808+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:12:04.615808+0000","last_peered":"2023-10-09T04:12:04.615808+0000","last_clean":"2023-10-09T04:12:04.615808+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:12:04.615808+0000","last_undegraded":"2023-10-09T04:12:04.615808+0000","last_fullsized":"2023-10-09T04:12:04.615808+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":203,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.798959+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:12:14.798959+0000","last_peered":"2023-10-09T04:12:14.798959+0000","last_clean":"2023-10-09T04:12:14.798959+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:12:14.798959+0000","last_undegraded":"2023-10-09T04:12:14.798959+0000","last_fullsized":"2023-10-09T04:12:14.798959+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"217'5970","reported_seq":6040,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.035759+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:12:18.035759+0000","last_peered":"2023-10-09T04:12:18.035759+0000","last_clean":"2023-10-09T04:12:18.035759+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:12:18.035759+0000","last_undegraded":"2023-10-09T04:12:18.035759+0000","last_fullsized":"2023-10-09T04:12:18.035759+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4808,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":30,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.605837+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:12:04.605837+0000","last_peered":"2023-10-09T04:12:04.605837+0000","last_clean":"2023-10-09T04:12:04.605837+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:12:04.605837+0000","last_undegraded":"2023-10-09T04:12:04.605837+0000","last_fullsized":"2023-10-09T04:12:04.605837+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":187,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.803592+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:12:14.803592+0000","last_peered":"2023-10-09T04:12:14.803592+0000","last_clean":"2023-10-09T04:12:14.803592+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:12:14.803592+0000","last_undegraded":"2023-10-09T04:12:14.803592+0000","last_fullsized":"2023-10-09T04:12:14.803592+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":188,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.798037+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:12:14.798037+0000","last_peered":"2023-10-09T04:12:14.798037+0000","last_clean":"2023-10-09T04:12:14.798037+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:12:14.798037+0000","last_undegraded":"2023-10-09T04:12:14.798037+0000","last_fullsized":"2023-10-09T04:12:14.798037+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"217'5740","reported_seq":5809,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:17.907936+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:12:17.907936+0000","last_peered":"2023-10-09T04:12:17.907936+0000","last_clean":"2023-10-09T04:12:17.907936+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:12:17.907936+0000","last_undegraded":"2023-10-09T04:12:17.907936+0000","last_fullsized":"2023-10-09T04:12:17.907936+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72646656,"num_objects":1108,"num_object_clones":0,"num_object_copies":3324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4622,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":200,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601869+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:12:04.601869+0000","last_peered":"2023-10-09T04:12:04.601869+0000","last_clean":"2023-10-09T04:12:04.601869+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:12:04.601869+0000","last_undegraded":"2023-10-09T04:12:04.601869+0000","last_fullsized":"2023-10-09T04:12:04.601869+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":557,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613890+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:12:04.613890+0000","last_peered":"2023-10-09T04:12:04.613890+0000","last_clean":"2023-10-09T04:12:04.613890+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:12:04.613890+0000","last_undegraded":"2023-10-09T04:12:04.613890+0000","last_fullsized":"2023-10-09T04:12:04.613890+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":113,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.602871+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:12:04.602871+0000","last_peered":"2023-10-09T04:12:04.602871+0000","last_clean":"2023-10-09T04:12:04.602871+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:12:04.602871+0000","last_undegraded":"2023-10-09T04:12:04.602871+0000","last_fullsized":"2023-10-09T04:12:04.602871+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615524+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:12:04.615524+0000","last_peered":"2023-10-09T04:12:04.615524+0000","last_clean":"2023-10-09T04:12:04.615524+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:12:04.615524+0000","last_undegraded":"2023-10-09T04:12:04.615524+0000","last_fullsized":"2023-10-09T04:12:04.615524+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"217'5751","reported_seq":8102,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:15.532208+0000","last_change":"2023-10-09T04:10:53.276397+0000","last_active":"2023-10-09T04:12:15.532208+0000","last_peered":"2023-10-09T04:12:15.532208+0000","last_clean":"2023-10-09T04:12:15.532208+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:12:15.532208+0000","last_undegraded":"2023-10-09T04:12:15.532208+0000","last_fullsized":"2023-10-09T04:12:15.532208+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74678272,"num_objects":1139,"num_object_clones":0,"num_object_copies":3417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4635,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":197,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613848+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:12:04.613848+0000","last_peered":"2023-10-09T04:12:04.613848+0000","last_clean":"2023-10-09T04:12:04.613848+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:12:04.613848+0000","last_undegraded":"2023-10-09T04:12:04.613848+0000","last_fullsized":"2023-10-09T04:12:04.613848+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.788408+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:12:14.788408+0000","last_peered":"2023-10-09T04:12:14.788408+0000","last_clean":"2023-10-09T04:12:14.788408+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:12:14.788408+0000","last_undegraded":"2023-10-09T04:12:14.788408+0000","last_fullsized":"2023-10-09T04:12:14.788408+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":205,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631306+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:12:04.631306+0000","last_peered":"2023-10-09T04:12:04.631306+0000","last_clean":"2023-10-09T04:12:04.631306+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:12:04.631306+0000","last_undegraded":"2023-10-09T04:12:04.631306+0000","last_fullsized":"2023-10-09T04:12:04.631306+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"217'5970","reported_seq":6102,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:15.546343+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:12:15.546343+0000","last_peered":"2023-10-09T04:12:15.546343+0000","last_clean":"2023-10-09T04:12:15.546343+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:12:15.546343+0000","last_undegraded":"2023-10-09T04:12:15.546343+0000","last_fullsized":"2023-10-09T04:12:15.546343+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78315520,"num_objects":1195,"num_object_clones":0,"num_object_copies":3585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3423,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":180,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613107+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:12:04.613107+0000","last_peered":"2023-10-09T04:12:04.613107+0000","last_clean":"2023-10-09T04:12:04.613107+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:12:04.613107+0000","last_undegraded":"2023-10-09T04:12:04.613107+0000","last_fullsized":"2023-10-09T04:12:04.613107+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":187,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.799043+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:12:14.799043+0000","last_peered":"2023-10-09T04:12:14.799043+0000","last_clean":"2023-10-09T04:12:14.799043+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:12:14.799043+0000","last_undegraded":"2023-10-09T04:12:14.799043+0000","last_fullsized":"2023-10-09T04:12:14.799043+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":157,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631253+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:12:04.631253+0000","last_peered":"2023-10-09T04:12:04.631253+0000","last_clean":"2023-10-09T04:12:04.631253+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:12:04.631253+0000","last_undegraded":"2023-10-09T04:12:04.631253+0000","last_fullsized":"2023-10-09T04:12:04.631253+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"217'5971","reported_seq":9034,"reported_epoch":217,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:15.713390+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:12:15.713390+0000","last_peered":"2023-10-09T04:12:15.713390+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:12:15.713390+0000","last_undegraded":"2023-10-09T04:12:15.713390+0000","last_fullsized":"2023-10-09T04:12:15.713390+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5971,"log_dups_size":0,"ondisk_log_size":5971,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77070336,"num_objects":1176,"num_object_clones":0,"num_object_copies":3528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1176,"num_objects_unfound":0,"num_objects_dirty":1176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4800,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":1176}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":121,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613124+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:12:04.613124+0000","last_peered":"2023-10-09T04:12:04.613124+0000","last_clean":"2023-10-09T04:12:04.613124+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:12:04.613124+0000","last_undegraded":"2023-10-09T04:12:04.613124+0000","last_fullsized":"2023-10-09T04:12:04.613124+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":154,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.807273+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:12:14.807273+0000","last_peered":"2023-10-09T04:12:14.807273+0000","last_clean":"2023-10-09T04:12:14.807273+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:12:14.807273+0000","last_undegraded":"2023-10-09T04:12:14.807273+0000","last_fullsized":"2023-10-09T04:12:14.807273+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":147,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613986+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:12:04.613986+0000","last_peered":"2023-10-09T04:12:04.613986+0000","last_clean":"2023-10-09T04:12:04.613986+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:12:04.613986+0000","last_undegraded":"2023-10-09T04:12:04.613986+0000","last_fullsized":"2023-10-09T04:12:04.613986+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"218'5939","reported_seq":8153,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.644710+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:12:18.644710+0000","last_peered":"2023-10-09T04:12:18.644710+0000","last_clean":"2023-10-09T04:12:18.644710+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:12:18.644710+0000","last_undegraded":"2023-10-09T04:12:18.644710+0000","last_fullsized":"2023-10-09T04:12:18.644710+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3462,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":145,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.613192+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:12:04.613192+0000","last_peered":"2023-10-09T04:12:04.613192+0000","last_clean":"2023-10-09T04:12:04.613192+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:12:04.613192+0000","last_undegraded":"2023-10-09T04:12:04.613192+0000","last_fullsized":"2023-10-09T04:12:04.613192+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":176,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631384+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:12:04.631384+0000","last_peered":"2023-10-09T04:12:04.631384+0000","last_clean":"2023-10-09T04:12:04.631384+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:12:04.631384+0000","last_undegraded":"2023-10-09T04:12:04.631384+0000","last_fullsized":"2023-10-09T04:12:04.631384+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615242+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:12:04.615242+0000","last_peered":"2023-10-09T04:12:04.615242+0000","last_clean":"2023-10-09T04:12:04.615242+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:12:04.615242+0000","last_undegraded":"2023-10-09T04:12:04.615242+0000","last_fullsized":"2023-10-09T04:12:04.615242+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":161,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.631510+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:12:04.631510+0000","last_peered":"2023-10-09T04:12:04.631510+0000","last_clean":"2023-10-09T04:12:04.631510+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:12:04.631510+0000","last_undegraded":"2023-10-09T04:12:04.631510+0000","last_fullsized":"2023-10-09T04:12:04.631510+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":30,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.606256+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:12:04.606256+0000","last_peered":"2023-10-09T04:12:04.606256+0000","last_clean":"2023-10-09T04:12:04.606256+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:12:04.606256+0000","last_undegraded":"2023-10-09T04:12:04.606256+0000","last_fullsized":"2023-10-09T04:12:04.606256+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"217'5785","reported_seq":5857,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.946024+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:12:14.946024+0000","last_peered":"2023-10-09T04:12:14.946024+0000","last_clean":"2023-10-09T04:12:14.946024+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:12:14.946024+0000","last_undegraded":"2023-10-09T04:12:14.946024+0000","last_fullsized":"2023-10-09T04:12:14.946024+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76873728,"num_objects":1173,"num_object_clones":0,"num_object_copies":3519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4645,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":178,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601675+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:12:04.601675+0000","last_peered":"2023-10-09T04:12:04.601675+0000","last_clean":"2023-10-09T04:12:04.601675+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:12:04.601675+0000","last_undegraded":"2023-10-09T04:12:04.601675+0000","last_fullsized":"2023-10-09T04:12:04.601675+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":192,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615885+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:12:04.615885+0000","last_peered":"2023-10-09T04:12:04.615885+0000","last_clean":"2023-10-09T04:12:04.615885+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:12:04.615885+0000","last_undegraded":"2023-10-09T04:12:04.615885+0000","last_fullsized":"2023-10-09T04:12:04.615885+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"217'5980","reported_seq":6047,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:18.043902+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:12:18.043902+0000","last_peered":"2023-10-09T04:12:18.043902+0000","last_clean":"2023-10-09T04:12:18.043902+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:12:18.043902+0000","last_undegraded":"2023-10-09T04:12:18.043902+0000","last_fullsized":"2023-10-09T04:12:18.043902+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5980,"log_dups_size":0,"ondisk_log_size":5980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71172096,"num_objects":1086,"num_object_clones":0,"num_object_copies":3258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4818,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":161,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.601975+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:12:04.601975+0000","last_peered":"2023-10-09T04:12:04.601975+0000","last_clean":"2023-10-09T04:12:04.601975+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:12:04.601975+0000","last_undegraded":"2023-10-09T04:12:04.601975+0000","last_fullsized":"2023-10-09T04:12:04.601975+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":192,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615641+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:12:04.615641+0000","last_peered":"2023-10-09T04:12:04.615641+0000","last_clean":"2023-10-09T04:12:04.615641+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:12:04.615641+0000","last_undegraded":"2023-10-09T04:12:04.615641+0000","last_fullsized":"2023-10-09T04:12:04.615641+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":30,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.606835+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:12:04.606835+0000","last_peered":"2023-10-09T04:12:04.606835+0000","last_clean":"2023-10-09T04:12:04.606835+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:12:04.606835+0000","last_undegraded":"2023-10-09T04:12:04.606835+0000","last_fullsized":"2023-10-09T04:12:04.606835+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":30,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.605887+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:12:04.605887+0000","last_peered":"2023-10-09T04:12:04.605887+0000","last_clean":"2023-10-09T04:12:04.605887+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:12:04.605887+0000","last_undegraded":"2023-10-09T04:12:04.605887+0000","last_fullsized":"2023-10-09T04:12:04.605887+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":207,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-09T04:12:14.789459+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:12:14.789459+0000","last_peered":"2023-10-09T04:12:14.789459+0000","last_clean":"2023-10-09T04:12:14.789459+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:12:14.789459+0000","last_undegraded":"2023-10-09T04:12:14.789459+0000","last_fullsized":"2023-10-09T04:12:14.789459+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":175,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-09T04:12:04.615810+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:12:04.615810+0000","last_peered":"2023-10-09T04:12:04.615810+0000","last_clean":"2023-10-09T04:12:04.615810+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:12:04.615810+0000","last_undegraded":"2023-10-09T04:12:04.615810+0000","last_fullsized":"2023-10-09T04:12:04.615810+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":2361065496,"num_objects":36028,"num_object_clones":0,"num_object_copies":108084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2197,"num_objects_misplaced":3506,"num_objects_unfound":0,"num_objects_dirty":36028,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144069,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11871,"num_bytes_recovered":777977856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7727624192,"data_stored":7727611976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164003,"ondisk_log_size":164003,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201517,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1248260,"kb_used_data":938684,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92496892,"statfs":{"total":95995035648,"available":94716817408,"internally_reserved":0,"allocated":961212416,"data_stored":960558007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":10.025,"5min":3.5390000000000001,"15min":1.583},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":126.752,"5min":126.752,"15min":126.752},"last":2.6040000000000001},{"interface":"front","average":{"1min":10.521000000000001,"5min":3.609,"15min":1.6060000000000001},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":128.88399999999999,"5min":128.88399999999999,"15min":128.88399999999999},"last":0.57399999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":10.563000000000001,"5min":5.8120000000000003,"15min":2.3980000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":103.014,"5min":245.28700000000001,"15min":245.28700000000001},"last":0.27800000000000002},{"interface":"front","average":{"1min":10.186999999999999,"5min":5.3700000000000001,"15min":2.379},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":103.083,"5min":244.626,"15min":244.626},"last":1.575}]},{"osd":2,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.8799999999999999,"15min":1.268},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4009999999999998,"5min":13.887,"15min":58.615000000000002},"last":2.3029999999999999},{"interface":"front","average":{"1min":1.401,"5min":2.9279999999999999,"15min":1.6850000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.3109999999999999,"5min":99.372,"15min":99.372},"last":2.29}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.5469999999999999,"15min":1.5589999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":3.0430000000000001,"5min":13.124000000000001,"15min":99.134},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.6599999999999999,"15min":1.6399999999999999},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.0230000000000001,"5min":13.140000000000001,"15min":99.117999999999995},"last":0.50800000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.079,"15min":0.69699999999999995},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9790000000000001,"5min":16.195,"15min":16.195},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.442,"5min":1.0109999999999999,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.052,"5min":12.442,"15min":12.442},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.77800000000000002,"15min":0.59699999999999998},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81999999999999995,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.873,"15min":0.67800000000000005},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0550000000000002,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.38700000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.91300000000000003,"15min":0.67600000000000005},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.99399999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.03,"15min":0.745},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":2.0819999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.49099999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463154,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1290348,"kb_used_data":988964,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92454804,"statfs":{"total":95995035648,"available":94673719296,"internally_reserved":0,"allocated":1012699136,"data_stored":1012060571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":6.7850000000000001,"5min":4.2990000000000004,"15min":1.8460000000000001},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":60.295000000000002,"5min":78.983999999999995,"15min":78.983999999999995},"last":0.23699999999999999},{"interface":"front","average":{"1min":6.8220000000000001,"5min":4.8659999999999997,"15min":2.0830000000000002},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":60.716000000000001,"5min":78.998000000000005,"15min":78.998000000000005},"last":0.28399999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.657,"15min":1.6950000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":4.6840000000000002,"5min":62.698,"15min":62.698},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.909,"5min":3.875,"15min":1.794},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":8.3529999999999998,"5min":63.530999999999999,"15min":63.530999999999999},"last":4.1840000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:11:20 2023","interfaces":[{"interface":"back","average":{"1min":7.6840000000000002,"5min":3.8420000000000001,"15min":1.8420000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":77.953000000000003,"5min":77.953000000000003,"15min":77.953000000000003},"last":2.085},{"interface":"front","average":{"1min":7.7320000000000002,"5min":3.875,"15min":1.8879999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.28399999999999997},"max":{"1min":78.019000000000005,"5min":78.019000000000005,"15min":78.019000000000005},"last":2.1139999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":3.516,"15min":2.0339999999999998},"min":{"1min":0.35799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.336,"5min":57.509999999999998,"15min":57.509999999999998},"last":4.2190000000000003},{"interface":"front","average":{"1min":1.1120000000000001,"5min":3.7450000000000001,"15min":2.052},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1840000000000002,"5min":57.453000000000003,"15min":57.453000000000003},"last":1.1990000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.3720000000000001,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.72199999999999998,"15min":0.64400000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2970000000000002,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":0.84599999999999997,"15min":0.84899999999999998},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.081,"5min":0.97399999999999998,"15min":0.95499999999999996},"min":{"1min":0.34399999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":23.006,"5min":23.006,"15min":54.853999999999999},"last":0.40600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68600000000000005,"15min":0.628},"min":{"1min":0.317,"5min":0.313,"15min":0.313},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":1.2310000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.79000000000000004,"15min":0.71199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":3.3679999999999999,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":1.2529999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089463,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252608,"kb_used_data":1010232,"kb_used_omap":9,"kb_used_meta":242102,"kb_avail":92492544,"statfs":{"total":95995035648,"available":94712365056,"internally_reserved":0,"allocated":1034477568,"data_stored":1033599427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":247913449},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":6.4269999999999996,"5min":3.7930000000000001,"15min":1.843},"min":{"1min":0.247,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":61.689999999999998,"5min":85.977999999999994,"15min":85.977999999999994},"last":11.587999999999999},{"interface":"front","average":{"1min":7.4409999999999998,"5min":4.2519999999999998,"15min":2.3519999999999999},"min":{"1min":0.31900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":61.670000000000002,"5min":85.998999999999995,"15min":99.831999999999994},"last":11.839}]},{"osd":2,"last update":"Mon Oct 9 04:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":3.6469999999999998,"15min":1.5369999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.6,"5min":121.54600000000001,"15min":121.54600000000001},"last":11.618},{"interface":"front","average":{"1min":2.3010000000000002,"5min":3.7919999999999998,"15min":1.702},"min":{"1min":0.23200000000000001,"5min":0.186,"15min":0.186},"max":{"1min":11.052,"5min":99.899000000000001,"15min":99.899000000000001},"last":11.714}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":4.2089999999999996,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":7.8920000000000003,"5min":138.87299999999999,"15min":138.87299999999999},"last":11.467000000000001},{"interface":"front","average":{"1min":3.593,"5min":4.6449999999999996,"15min":2.6309999999999998},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.414,"5min":138.49799999999999,"15min":138.49799999999999},"last":11.771000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":1.8520000000000001,"15min":1.498},"min":{"1min":0.35599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":15.215999999999999,"5min":15.215999999999999,"15min":99.989999999999995},"last":11.682},{"interface":"front","average":{"1min":1.115,"5min":1.5129999999999999,"15min":1.3779999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.52,"5min":10.438000000000001,"15min":100.249},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.75,"15min":1.48},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.266,"5min":15.266,"15min":99.933999999999997},"last":11.742000000000001},{"interface":"front","average":{"1min":1.389,"5min":1.829,"15min":1.4950000000000001},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":4.8849999999999998,"5min":10.467000000000001,"15min":100.34099999999999},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.827,"15min":1.5880000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.585,"5min":10.497999999999999,"15min":100.309},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.671,"5min":1.496,"15min":1.5169999999999999},"min":{"1min":0.36499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9240000000000004,"5min":7.9240000000000004,"15min":100.143},"last":11.497999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.9630000000000001,"15min":1.569},"min":{"1min":0.40000000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.9569999999999999,"5min":10.532,"15min":100.214},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.798,"5min":1.796,"15min":1.5429999999999999},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":100.113},"last":11.419}]}]},{"osd":0,"up_from":84,"seq":360777252989,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1440328,"kb_used_data":1210368,"kb_used_omap":28,"kb_used_meta":229859,"kb_avail":92304824,"statfs":{"total":95995035648,"available":94520139776,"internally_reserved":0,"allocated":1239416832,"data_stored":1238521704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":235376464},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":3.6059999999999999,"15min":1.982},"min":{"1min":0.184,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":12.698,"5min":103.996,"15min":118.48},"last":0.498},{"interface":"front","average":{"1min":3.5910000000000002,"5min":4.1959999999999997,"15min":2.2080000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":12.632999999999999,"5min":106.34399999999999,"15min":118.384},"last":0.48799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:11:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":4.5990000000000002,"15min":1.9419999999999999},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":4.8819999999999997,"5min":118.54000000000001,"15min":118.54000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.736,"5min":5.0780000000000003,"15min":2.089},"min":{"1min":0.21199999999999999,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":8.2300000000000004,"5min":118.727,"15min":118.727},"last":0.46500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":5.9320000000000004,"15min":3.601},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.161,"5min":156.488,"15min":156.488},"last":1.046},{"interface":"front","average":{"1min":4.1909999999999998,"5min":5.8979999999999997,"15min":3.6890000000000001},"min":{"1min":0.40699999999999997,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":9.6010000000000009,"5min":156.50700000000001,"15min":156.50700000000001},"last":1.0669999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.3540000000000001,"15min":1.625},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8310000000000004,"5min":59.439999999999998,"15min":118.633},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.151,"5min":2.9889999999999999,"15min":1.897},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.593,"5min":59.453000000000003,"15min":118.913},"last":0.40400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":4.548,"15min":2.4169999999999998},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.6310000000000002,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.439},{"interface":"front","average":{"1min":2.302,"5min":4.2830000000000004,"15min":2.3039999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.8919999999999995,"5min":182.66300000000001,"15min":182.66300000000001},"last":0.54000000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":8.0259999999999998,"5min":3.976,"15min":2.2109999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":137.36099999999999,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":7.3899999999999997,"5min":3.5800000000000001,"15min":2.0670000000000002},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":136.88999999999999,"5min":136.88999999999999,"15min":136.88999999999999},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":7.2210000000000001,"5min":5.7480000000000002,"15min":2.8370000000000002},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":121.324,"5min":185.024,"15min":185.024},"last":0.623},{"interface":"front","average":{"1min":2.206,"5min":4.21,"15min":2.3180000000000001},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":6.6280000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.56399999999999995}]}]},{"osd":2,"up_from":94,"seq":403726925936,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1109972,"kb_used_data":794444,"kb_used_omap":81,"kb_used_meta":315438,"kb_avail":92635180,"statfs":{"total":95995035648,"available":94858424320,"internally_reserved":0,"allocated":813510656,"data_stored":812658393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":323008730},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.4550000000000001,"15min":1.1419999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.413,"5min":12.284000000000001,"15min":12.284000000000001},"last":6.1760000000000002},{"interface":"front","average":{"1min":2.5840000000000001,"5min":3.2650000000000001,"15min":1.5189999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":11.356999999999999,"5min":45.979999999999997,"15min":45.979999999999997},"last":1.196}]},{"osd":1,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":3.6539999999999999,"15min":1.8160000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":14.99,"5min":53.695,"15min":68.009},"last":1.337},{"interface":"front","average":{"1min":2.5979999999999999,"5min":3.8730000000000002,"15min":1.919},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":16.597000000000001,"5min":53.439,"15min":66.975999999999999},"last":0.224}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":2.2029999999999998,"15min":1.349},"min":{"1min":0.22500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.212999999999999,"5min":12.77,"15min":27.562000000000001},"last":5.9550000000000001},{"interface":"front","average":{"1min":2.577,"5min":2.8199999999999998,"15min":1.552},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.808,"5min":46.286999999999999,"15min":46.286999999999999},"last":1.23}]},{"osd":4,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":2.419,"15min":1.3280000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.9209999999999998,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.524,"5min":1.7450000000000001,"15min":1.0329999999999999},"min":{"1min":0.308,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.5809999999999995,"5min":14.305,"15min":14.305},"last":6.0750000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.601,"15min":1.373},"min":{"1min":0.373,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":26.667000000000002,"5min":46.265000000000001,"15min":46.265000000000001},"last":1.306},{"interface":"front","average":{"1min":2.7999999999999998,"5min":2.8620000000000001,"15min":1.5},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":26.654,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.32300000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0199999999999996,"5min":2.9830000000000001,"15min":1.4690000000000001},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":114.14,"5min":114.14,"15min":114.14},"last":6.125},{"interface":"front","average":{"1min":7.8810000000000002,"5min":3.6789999999999998,"15min":1.7549999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":114.123,"5min":114.123,"15min":114.123},"last":1.276}]},{"osd":7,"last update":"Mon Oct 9 04:11:19 2023","interfaces":[{"interface":"back","average":{"1min":8.1690000000000005,"5min":3.4550000000000001,"15min":1.651},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":117.32599999999999,"5min":117.32599999999999,"15min":117.32599999999999},"last":6.1529999999999996},{"interface":"front","average":{"1min":8.1639999999999997,"5min":3.9079999999999999,"15min":1.8540000000000001},"min":{"1min":0.40500000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":117.28700000000001,"5min":117.28700000000001,"15min":117.28700000000001},"last":1.3640000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762410,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926600,"kb_used_data":1497728,"kb_used_omap":7,"kb_used_meta":428664,"kb_avail":91818552,"statfs":{"total":95995035648,"available":94022197248,"internally_reserved":0,"allocated":1533673472,"data_stored":1532838266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":438952552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8140000000000001,"5min":4.6970000000000001,"15min":3.3519999999999999},"min":{"1min":0.20100000000000001,"5min":0.189,"15min":0.185},"max":{"1min":23.449000000000002,"5min":100.512,"15min":106.22},"last":2.8410000000000002},{"interface":"front","average":{"1min":4.0759999999999996,"5min":5.056,"15min":3.4039999999999999},"min":{"1min":0.32200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":13.670999999999999,"5min":101.733,"15min":106.986},"last":3.5430000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":3.5699999999999998,"15min":2.2490000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":8.6340000000000003,"5min":57.055999999999997,"15min":106.121},"last":2.9420000000000002},{"interface":"front","average":{"1min":3.899,"5min":3.9180000000000001,"15min":2.1019999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":23.475999999999999,"5min":57.088999999999999,"15min":65.015000000000001},"last":2.347}]},{"osd":2,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.8130000000000002,"15min":2.1669999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":23.405999999999999,"5min":23.405999999999999,"15min":106.105},"last":2.5329999999999999},{"interface":"front","average":{"1min":3.524,"5min":3.7229999999999999,"15min":2.5190000000000001},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":13.698,"5min":98.765000000000001,"15min":106.70399999999999},"last":3.4790000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":2.9060000000000001,"15min":1.927},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.308},"max":{"1min":13.536,"5min":37.652000000000001,"15min":106.85899999999999},"last":1.4650000000000001},{"interface":"front","average":{"1min":1.873,"5min":2.5209999999999999,"15min":1.351},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.1699999999999999,"5min":37.442,"15min":37.442},"last":1.397}]},{"osd":5,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.1150000000000002,"15min":1.2430000000000001},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.1940000000000008,"5min":12.819000000000001,"15min":12.819000000000001},"last":1.4179999999999999},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.9730000000000001,"15min":1.516},"min":{"1min":0.32400000000000001,"5min":0.29599999999999999,"15min":0.27900000000000003},"max":{"1min":11.938000000000001,"5min":13.228999999999999,"15min":13.228999999999999},"last":2.5630000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.8560000000000001,"15min":1.9039999999999999},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.30199999999999999},"max":{"1min":11.994999999999999,"5min":13.255000000000001,"15min":106.036},"last":2.5960000000000001},{"interface":"front","average":{"1min":2.363,"5min":2.355,"15min":1.7789999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.571,"5min":15.622,"15min":106.741},"last":3.516}]},{"osd":7,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":2.5099999999999998,"15min":1.8260000000000001},"min":{"1min":0.38700000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.494999999999999,"5min":15.685,"15min":106.82599999999999},"last":1.496},{"interface":"front","average":{"1min":2.1480000000000001,"5min":2.1749999999999998,"15min":1.7010000000000001},"min":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.2170000000000005,"5min":13.228999999999999,"15min":106.084},"last":1.3560000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986426,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994040,"kb_used_data":680368,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92751112,"statfs":{"total":95995035648,"available":94977138688,"internally_reserved":0,"allocated":696696832,"data_stored":696053832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":2.8290000000000002,"15min":1.3500000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":1.726,"5min":63.128,"15min":63.128},"last":2.9340000000000002},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.5110000000000001,"15min":1.3149999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.1600000000000001,"5min":62.645000000000003,"15min":62.645000000000003},"last":0.26500000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.7709999999999999,"15min":1.4410000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":23.157,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.8410000000000002,"5min":2.8479999999999999,"15min":1.4299999999999999},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":23.091999999999999,"5min":75.314999999999998,"15min":75.314999999999998},"last":2.875}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8380000000000001,"15min":1.0900000000000001},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.698,"5min":19.356000000000002,"15min":19.356000000000002},"last":5.4470000000000001},{"interface":"front","average":{"1min":1.125,"5min":1.845,"15min":1.042},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7109999999999999,"5min":19.324000000000002,"15min":19.324000000000002},"last":5.4610000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.8779999999999999,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4880000000000004,"5min":14.694000000000001,"15min":14.694000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.958,"15min":1.222},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.2110000000000003,"5min":10.476000000000001,"15min":10.476000000000001},"last":0.60799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.70999999999999996,"15min":0.57599999999999996},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3100000000000001,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92600000000000005,"15min":0.71599999999999997},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.7290000000000001,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.41499999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.93799999999999994,"15min":0.74099999999999999},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":2.9990000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.81899999999999995,"15min":0.66600000000000004},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9700000000000002,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":2.9129999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.96999999999999997,"15min":0.69899999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":1.4890000000000001,"5min":9.0050000000000008,"15min":9.0050000000000008},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.004,"15min":0.76100000000000001},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.306,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.50600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724790,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833704,"kb_used_data":532320,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92911448,"statfs":{"total":95995035648,"available":95141322752,"internally_reserved":0,"allocated":545095680,"data_stored":544464750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":6.391,"5min":3.4849999999999999,"15min":1.577},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":83.329999999999998,"5min":83.329999999999998,"15min":83.329999999999998},"last":2.2719999999999998},{"interface":"front","average":{"1min":7.7030000000000003,"5min":3.778,"15min":1.6990000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":85.451999999999998,"5min":85.451999999999998,"15min":85.451999999999998},"last":4.3920000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:11:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3869999999999996,"5min":2.6749999999999998,"15min":1.7030000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":59.588999999999999,"5min":59.588999999999999,"15min":91.212999999999994},"last":4.7759999999999998},{"interface":"front","average":{"1min":4.6139999999999999,"5min":2.9550000000000001,"15min":1.913},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":59.533000000000001,"5min":59.533000000000001,"15min":91.254000000000005},"last":6.7969999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":3.0880000000000001,"15min":1.524},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.2959999999999998,"5min":61.310000000000002,"15min":61.310000000000002},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.968,"5min":3.2050000000000001,"15min":1.518},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.6929999999999996,"5min":61.154000000000003,"15min":61.154000000000003},"last":7.6950000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.772,"15min":1.7010000000000001},"min":{"1min":0.35299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.714,"5min":17.071000000000002,"15min":91.834999999999994},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.548,"5min":1.885,"15min":1.7450000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.309},"max":{"1min":5.9749999999999996,"5min":17.052,"15min":91.802000000000007},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.88900000000000001,"15min":0.65700000000000003},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.253},"max":{"1min":3.2440000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.95599999999999996,"15min":0.71799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26600000000000001},"max":{"1min":3.4420000000000002,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2839999999999998,"5min":2.1230000000000002,"15min":1.113},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":123.239,"5min":123.239,"15min":123.239},"last":0.496},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.0899999999999999,"15min":1.129},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":123.014,"5min":123.014,"15min":123.014},"last":0.371}]},{"osd":7,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.98999999999999999,"15min":0.78300000000000003},"min":{"1min":0.307,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2450000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90300000000000002,"15min":0.80300000000000005},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":1.8220000000000001,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.47899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1221136384,"data_stored":1221132312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1028788224,"data_stored":1028784152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":807337984,"data_stored":807337984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1511260160,"data_stored":1511260160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":689569792,"data_stored":689569792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532807680,"data_stored":532807680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":990253056,"data_stored":990248984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":946470912,"data_stored":946470912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:12:22.431 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:22.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:22.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:22.567 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:22.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:22.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:22.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:22.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:22.776 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:22.827 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:22.831 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:22 smithi181 ceph-mon[196979]: from='client.55048 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:22.831 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:22 smithi181 ceph-mon[196979]: pgmap v578: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 785/102996 objects degraded (0.762%); 4432/102996 objects misplaced (4.303%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:12:22.832 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 785/102996 objects degraded (0.762%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:22.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:22.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:22 smithi163 ceph-mon[213841]: from='client.55048 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:22.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:22 smithi163 ceph-mon[213841]: pgmap v578: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 785/102996 objects degraded (0.762%); 4432/102996 objects misplaced (4.303%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:12:22.840 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 785/102996 objects degraded (0.762%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:22.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:22 smithi163 ceph-mon[221067]: from='client.55048 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:22.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:22 smithi163 ceph-mon[221067]: pgmap v578: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 785/102996 objects degraded (0.762%); 4432/102996 objects misplaced (4.303%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:12:22.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 785/102996 objects degraded (0.762%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:12:22.934 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:23.018 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:23.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:23.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:23.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:23.152 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:23.224 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:23.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:23.336 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:23.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:23.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:23.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:23.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:23.818 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:23.982 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:24.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:24.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:24.139 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:24.194 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:24.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:24.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:24.369 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:24.590 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:12:24.616 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:24 smithi163 ceph-mon[221067]: pgmap v579: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 785/99558 objects degraded (0.788%); 4428/99558 objects misplaced (4.448%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:12:24.616 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3192852371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:12:24.616 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:12:24.616 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:24 smithi163 ceph-mon[213841]: pgmap v579: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 785/99558 objects degraded (0.788%); 4428/99558 objects misplaced (4.448%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:12:24.616 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3192852371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:12:24.616 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:12:24.631 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:24.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:24.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:24.764 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:24.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:24.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:24.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:24.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:24.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:24 smithi181 ceph-mon[196979]: pgmap v579: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 785/99558 objects degraded (0.788%); 4428/99558 objects misplaced (4.448%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:12:24.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3192852371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:12:24.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:12:24.963 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:25.039 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:25.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:25.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:25.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:25.215 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:25.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:12:25.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:25.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:12:25.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: osdmap e220: 8 total, 8 up, 8 in 2023-10-09T04:12:25.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[213841]: 7.11 scrub starts 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: osdmap e220: 8 total, 8 up, 8 in 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.950 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: failed to encode map e220 with expected crc 2023-10-09T04:12:25.951 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:25 smithi163 ceph-mon[221067]: 7.11 scrub starts 2023-10-09T04:12:25.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:25.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: failed to encode map e220 with expected crc 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: osdmap e220: 8 total, 8 up, 8 in 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: failed to encode map e220 with expected crc 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: failed to encode map e220 with expected crc 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: failed to encode map e220 with expected crc 2023-10-09T04:12:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: failed to encode map e220 with expected crc 2023-10-09T04:12:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: failed to encode map e220 with expected crc 2023-10-09T04:12:26.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:25 smithi181 ceph-mon[196979]: 7.11 scrub starts 2023-10-09T04:12:26.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:26.112 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:26.166 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:26.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:26.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:26.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:26.371 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:26.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:26.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:26.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:26 smithi163 ceph-mon[221067]: pgmap v581: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 785/95913 objects degraded (0.818%); 4428/95913 objects misplaced (4.617%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:12:26.611 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:26 smithi163 ceph-mon[221067]: 7.5 scrub starts 2023-10-09T04:12:26.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:26.748 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:26.812 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:26.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:26.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:26.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:26 smithi163 ceph-mon[213841]: pgmap v581: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 785/95913 objects degraded (0.818%); 4428/95913 objects misplaced (4.617%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:12:26.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:26 smithi163 ceph-mon[213841]: 7.5 scrub starts 2023-10-09T04:12:26.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:26 smithi181 ceph-mon[196979]: pgmap v581: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 785/95913 objects degraded (0.818%); 4428/95913 objects misplaced (4.617%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T04:12:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:26 smithi181 ceph-mon[196979]: 7.5 scrub starts 2023-10-09T04:12:27.028 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:27.037 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:27.224 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:27.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:27.235 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:12:27.273 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:12:27.317 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:12:27.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:27.359 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:12:27.361 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:12:27.362 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:12:27.362 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:12:27.362 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:12:27.362 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:12:27.419 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:27.420 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:12:27.459 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:12:27.492 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:12:27.530 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.562 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:12:27.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1246839950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:27.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 785/95913 objects degraded (0.818%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:27.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1246839950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:27.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 785/95913 objects degraded (0.818%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:28.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1246839950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:28.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 785/95913 objects degraded (0.818%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:28.441 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:28.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:28.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:28.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:28.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:12:28.643 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:28.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:28 smithi163 ceph-mon[221067]: pgmap v582: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 785/92298 objects degraded (0.851%); 3828/92298 objects misplaced (4.147%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:12:28.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:28 smithi163 ceph-mon[213841]: pgmap v582: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 785/92298 objects degraded (0.851%); 3828/92298 objects misplaced (4.147%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:12:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:28 smithi181 ceph-mon[196979]: pgmap v582: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 785/92298 objects degraded (0.851%); 3828/92298 objects misplaced (4.147%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:12:29.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:12:29.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:29.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:29.572 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:29.616 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:29.675 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:29.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:29.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:29.826 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:29 smithi163 ceph-mon[221067]: 7.11 scrub starts 2023-10-09T04:12:29.826 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:29 smithi163 ceph-mon[213841]: 7.11 scrub starts 2023-10-09T04:12:29.836 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:29.958 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:30.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:29 smithi181 ceph-mon[196979]: 7.11 scrub starts 2023-10-09T04:12:30.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:30.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:30.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:30.228 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:30.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:30.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:30.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:30.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:30 smithi163 ceph-mon[221067]: pgmap v583: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 785/86871 objects degraded (0.904%); 3828/86871 objects misplaced (4.407%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:12:30.656 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:30 smithi163 ceph-mon[221067]: 7.5 scrub starts 2023-10-09T04:12:30.656 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:30 smithi163 ceph-mon[213841]: pgmap v583: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 785/86871 objects degraded (0.904%); 3828/86871 objects misplaced (4.407%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:12:30.657 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:30 smithi163 ceph-mon[213841]: 7.5 scrub starts 2023-10-09T04:12:30.657 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:30.657 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:12:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:12:30] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-10-09T04:12:30.665 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:30.783 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:30.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:30.819 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:12:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:30.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:30.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:30.956 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:30 smithi181 ceph-mon[196979]: pgmap v583: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 785/86871 objects degraded (0.904%); 3828/86871 objects misplaced (4.407%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:12:30.956 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:30 smithi181 ceph-mon[196979]: 7.5 scrub starts 2023-10-09T04:12:30.962 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:31.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:31.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:31.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:31.142 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:31.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:31.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:31.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:31.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:31.328 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:31.373 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:31.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:31.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:31.506 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:31.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:12:31.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:31 smithi163 ceph-mon[221067]: 7.11 scrub starts 2023-10-09T04:12:31.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:31.758 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:31 smithi163 ceph-mon[213841]: 7.11 scrub starts 2023-10-09T04:12:31.758 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:32.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:31 smithi181 ceph-mon[196979]: 7.11 scrub starts 2023-10-09T04:12:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:32.022 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:32.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:32.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:32.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:32.196 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:32.472 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:32.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:32.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:32.603 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":583,"stamp":"2023-10-09T04:12:29.492318+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1896683448,"num_objects":28957,"num_object_clones":0,"num_object_copies":86871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":3828,"num_objects_unfound":0,"num_objects_dirty":28957,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":151464,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12294,"num_bytes_recovered":806816436,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169414,"ondisk_log_size":169414,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8763312,"kb_used_data":6298480,"kb_used_omap":126,"kb_used_meta":2464193,"kb_avail":741197904,"statfs":{"total":767960285184,"available":758986653696,"internally_reserved":0,"allocated":6449643520,"data_stored":6443485686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2523334267},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-464977920,"num_objects":-7095,"num_object_clones":0,"num_object_copies":-21285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1412,"num_objects_misplaced":322,"num_objects_unfound":0,"num_objects_dirty":-7095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7095,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":413,"num_bytes_recovered":27066368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006425"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":217,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364680+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:12:19.364680+0000","last_peered":"2023-10-09T04:12:19.364680+0000","last_clean":"2023-10-09T04:12:19.364680+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:12:19.364680+0000","last_undegraded":"2023-10-09T04:12:19.364680+0000","last_fullsized":"2023-10-09T04:12:19.364680+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.360980+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:12:19.360980+0000","last_peered":"2023-10-09T04:12:19.360980+0000","last_clean":"2023-10-09T04:12:19.360980+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:12:19.360980+0000","last_undegraded":"2023-10-09T04:12:19.360980+0000","last_fullsized":"2023-10-09T04:12:19.360980+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":171,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.362284+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:12:19.362284+0000","last_peered":"2023-10-09T04:12:19.362284+0000","last_clean":"2023-10-09T04:12:19.362284+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:19.362284+0000","last_undegraded":"2023-10-09T04:12:19.362284+0000","last_fullsized":"2023-10-09T04:12:19.362284+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":176,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.325086+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:12:19.325086+0000","last_peered":"2023-10-09T04:12:19.325086+0000","last_clean":"2023-10-09T04:12:19.325086+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:12:19.325086+0000","last_undegraded":"2023-10-09T04:12:19.325086+0000","last_fullsized":"2023-10-09T04:12:19.325086+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":174,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324469+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:12:19.324469+0000","last_peered":"2023-10-09T04:12:19.324469+0000","last_clean":"2023-10-09T04:12:19.324469+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:12:19.324469+0000","last_undegraded":"2023-10-09T04:12:19.324469+0000","last_fullsized":"2023-10-09T04:12:19.324469+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":153,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335699+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:12:19.335699+0000","last_peered":"2023-10-09T04:12:19.335699+0000","last_clean":"2023-10-09T04:12:19.335699+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:12:19.335699+0000","last_undegraded":"2023-10-09T04:12:19.335699+0000","last_fullsized":"2023-10-09T04:12:19.335699+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"220'6052","reported_seq":9012,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:25.781117+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:12:25.781117+0000","last_peered":"2023-10-09T04:12:25.781117+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:12:25.781117+0000","last_undegraded":"2023-10-09T04:12:25.781117+0000","last_fullsized":"2023-10-09T04:12:25.781117+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59441152,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":907,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4881,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":907}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.325529+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:12:19.325529+0000","last_peered":"2023-10-09T04:12:19.325529+0000","last_clean":"2023-10-09T04:12:19.325529+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:12:19.325529+0000","last_undegraded":"2023-10-09T04:12:19.325529+0000","last_fullsized":"2023-10-09T04:12:19.325529+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.321639+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:12:19.321639+0000","last_peered":"2023-10-09T04:12:19.321639+0000","last_clean":"2023-10-09T04:12:19.321639+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:12:19.321639+0000","last_undegraded":"2023-10-09T04:12:19.321639+0000","last_fullsized":"2023-10-09T04:12:19.321639+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"220'6217","reported_seq":13616,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.027376+0000","last_change":"2023-10-09T04:12:19.342281+0000","last_active":"2023-10-09T04:12:28.027376+0000","last_peered":"2023-10-09T04:12:28.027376+0000","last_clean":"2023-10-09T04:12:28.027376+0000","last_became_active":"2023-10-09T04:12:19.328982+0000","last_became_peered":"2023-10-09T04:12:19.328982+0000","last_unstale":"2023-10-09T04:12:28.027376+0000","last_undegraded":"2023-10-09T04:12:28.027376+0000","last_fullsized":"2023-10-09T04:12:28.027376+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6217,"log_dups_size":0,"ondisk_log_size":6217,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59113472,"num_objects":902,"num_object_clones":0,"num_object_copies":2706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2553,"num_bytes_recovered":167313408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":179,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335697+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:12:19.335697+0000","last_peered":"2023-10-09T04:12:19.335697+0000","last_clean":"2023-10-09T04:12:19.335697+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:12:19.335697+0000","last_undegraded":"2023-10-09T04:12:19.335697+0000","last_fullsized":"2023-10-09T04:12:19.335697+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":116,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.356412+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:12:19.356412+0000","last_peered":"2023-10-09T04:12:19.356412+0000","last_clean":"2023-10-09T04:12:19.356412+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:12:19.356412+0000","last_undegraded":"2023-10-09T04:12:19.356412+0000","last_fullsized":"2023-10-09T04:12:19.356412+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"220'5979","reported_seq":8872,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.486783+0000","last_change":"2023-10-09T04:12:24.916684+0000","last_active":"2023-10-09T04:12:25.486783+0000","last_peered":"2023-10-09T04:12:25.486783+0000","last_clean":"2023-10-09T04:12:25.486783+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:12:25.486783+0000","last_undegraded":"2023-10-09T04:12:25.486783+0000","last_fullsized":"2023-10-09T04:12:25.486783+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61341696,"num_objects":936,"num_object_clones":0,"num_object_copies":2808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4844,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":182,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.357753+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:12:19.357753+0000","last_peered":"2023-10-09T04:12:19.357753+0000","last_clean":"2023-10-09T04:12:19.357753+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:12:19.357753+0000","last_undegraded":"2023-10-09T04:12:19.357753+0000","last_fullsized":"2023-10-09T04:12:19.357753+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":172,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.360046+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:12:19.360046+0000","last_peered":"2023-10-09T04:12:19.360046+0000","last_clean":"2023-10-09T04:12:19.360046+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:12:19.360046+0000","last_undegraded":"2023-10-09T04:12:19.360046+0000","last_fullsized":"2023-10-09T04:12:19.360046+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"220'6124","reported_seq":9066,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.054711+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:12:25.054711+0000","last_peered":"2023-10-09T04:12:25.054711+0000","last_clean":"2023-10-09T04:12:25.054711+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:12:25.054711+0000","last_undegraded":"2023-10-09T04:12:25.054711+0000","last_fullsized":"2023-10-09T04:12:25.054711+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6124,"log_dups_size":0,"ondisk_log_size":6124,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62980096,"num_objects":961,"num_object_clones":0,"num_object_copies":2883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4935,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":184,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.365766+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:12:19.365766+0000","last_peered":"2023-10-09T04:12:19.365766+0000","last_clean":"2023-10-09T04:12:19.365766+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:12:19.365766+0000","last_undegraded":"2023-10-09T04:12:19.365766+0000","last_fullsized":"2023-10-09T04:12:19.365766+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":178,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.366061+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:12:19.366061+0000","last_peered":"2023-10-09T04:12:19.366061+0000","last_clean":"2023-10-09T04:12:19.366061+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:12:19.366061+0000","last_undegraded":"2023-10-09T04:12:19.366061+0000","last_fullsized":"2023-10-09T04:12:19.366061+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"220'6132","reported_seq":6205,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:27.913787+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:12:27.913787+0000","last_peered":"2023-10-09T04:12:27.913787+0000","last_clean":"2023-10-09T04:12:27.913787+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:12:27.913787+0000","last_undegraded":"2023-10-09T04:12:27.913787+0000","last_fullsized":"2023-10-09T04:12:27.913787+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6132,"log_dups_size":0,"ondisk_log_size":6132,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56754176,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4978,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":181,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.356546+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:12:19.356546+0000","last_peered":"2023-10-09T04:12:19.356546+0000","last_clean":"2023-10-09T04:12:19.356546+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:12:19.356546+0000","last_undegraded":"2023-10-09T04:12:19.356546+0000","last_fullsized":"2023-10-09T04:12:19.356546+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.361035+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:12:19.361035+0000","last_peered":"2023-10-09T04:12:19.361035+0000","last_clean":"2023-10-09T04:12:19.361035+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:12:19.361035+0000","last_undegraded":"2023-10-09T04:12:19.361035+0000","last_fullsized":"2023-10-09T04:12:19.361035+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"220'5902","reported_seq":5974,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:27.889632+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:12:27.889632+0000","last_peered":"2023-10-09T04:12:27.889632+0000","last_clean":"2023-10-09T04:12:27.889632+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:12:27.889632+0000","last_undegraded":"2023-10-09T04:12:27.889632+0000","last_fullsized":"2023-10-09T04:12:27.889632+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56393728,"num_objects":861,"num_object_clones":0,"num_object_copies":2583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4785,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":175,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324667+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:12:19.324667+0000","last_peered":"2023-10-09T04:12:19.324667+0000","last_clean":"2023-10-09T04:12:19.324667+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:12:19.324667+0000","last_undegraded":"2023-10-09T04:12:19.324667+0000","last_fullsized":"2023-10-09T04:12:19.324667+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":178,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335589+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:12:19.335589+0000","last_peered":"2023-10-09T04:12:19.335589+0000","last_clean":"2023-10-09T04:12:19.335589+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:12:19.335589+0000","last_undegraded":"2023-10-09T04:12:19.335589+0000","last_fullsized":"2023-10-09T04:12:19.335589+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"220'6051","reported_seq":9000,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:25.600559+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:12:25.600559+0000","last_peered":"2023-10-09T04:12:25.600559+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:12:25.600559+0000","last_undegraded":"2023-10-09T04:12:25.600559+0000","last_fullsized":"2023-10-09T04:12:25.600559+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6051,"log_dups_size":0,"ondisk_log_size":6051,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63406080,"num_objects":968,"num_object_clones":0,"num_object_copies":2904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":968,"num_objects_unfound":0,"num_objects_dirty":968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4936,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":968}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":162,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.356339+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:12:19.356339+0000","last_peered":"2023-10-09T04:12:19.356339+0000","last_clean":"2023-10-09T04:12:19.356339+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:12:19.356339+0000","last_undegraded":"2023-10-09T04:12:19.356339+0000","last_fullsized":"2023-10-09T04:12:19.356339+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":175,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.325123+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:12:19.325123+0000","last_peered":"2023-10-09T04:12:19.325123+0000","last_clean":"2023-10-09T04:12:19.325123+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:12:19.325123+0000","last_undegraded":"2023-10-09T04:12:19.325123+0000","last_fullsized":"2023-10-09T04:12:19.325123+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"220'6131","reported_seq":6204,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.004384+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:12:28.004384+0000","last_peered":"2023-10-09T04:12:28.004384+0000","last_clean":"2023-10-09T04:12:28.004384+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:12:28.004384+0000","last_undegraded":"2023-10-09T04:12:28.004384+0000","last_fullsized":"2023-10-09T04:12:28.004384+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4969,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":111,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324277+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:12:19.324277+0000","last_peered":"2023-10-09T04:12:19.324277+0000","last_clean":"2023-10-09T04:12:19.324277+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:12:19.324277+0000","last_undegraded":"2023-10-09T04:12:19.324277+0000","last_fullsized":"2023-10-09T04:12:19.324277+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":184,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.323959+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:12:19.323959+0000","last_peered":"2023-10-09T04:12:19.323959+0000","last_clean":"2023-10-09T04:12:19.323959+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:12:19.323959+0000","last_undegraded":"2023-10-09T04:12:19.323959+0000","last_fullsized":"2023-10-09T04:12:19.323959+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"220'9908","reported_seq":9970,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.672579+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:12:28.672579+0000","last_peered":"2023-10-09T04:12:28.672579+0000","last_clean":"2023-10-09T04:12:28.672579+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:12:28.672579+0000","last_undegraded":"2023-10-09T04:12:28.672579+0000","last_fullsized":"2023-10-09T04:12:28.672579+0000","mapping_epoch":167,"log_start":"183'2900","ondisk_log_start":"183'2900","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115867648,"num_objects":1768,"num_object_clones":0,"num_object_copies":5304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9908,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":180,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.358035+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:12:19.358035+0000","last_peered":"2023-10-09T04:12:19.358035+0000","last_clean":"2023-10-09T04:12:19.358035+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:12:19.358035+0000","last_undegraded":"2023-10-09T04:12:19.358035+0000","last_fullsized":"2023-10-09T04:12:19.358035+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":161,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335343+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:12:19.335343+0000","last_peered":"2023-10-09T04:12:19.335343+0000","last_clean":"2023-10-09T04:12:19.335343+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:12:19.335343+0000","last_undegraded":"2023-10-09T04:12:19.335343+0000","last_fullsized":"2023-10-09T04:12:19.335343+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"220'9711","reported_seq":9774,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.039100+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:12:25.039100+0000","last_peered":"2023-10-09T04:12:25.039100+0000","last_clean":"2023-10-09T04:12:25.039100+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:12:25.039100+0000","last_undegraded":"2023-10-09T04:12:25.039100+0000","last_fullsized":"2023-10-09T04:12:25.039100+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":9711,"log_dups_size":0,"ondisk_log_size":9711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121307136,"num_objects":1851,"num_object_clones":0,"num_object_copies":5553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9711,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":181,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.357726+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:12:19.357726+0000","last_peered":"2023-10-09T04:12:19.357726+0000","last_clean":"2023-10-09T04:12:19.357726+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:12:19.357726+0000","last_undegraded":"2023-10-09T04:12:19.357726+0000","last_fullsized":"2023-10-09T04:12:19.357726+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":195,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364962+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:12:19.364962+0000","last_peered":"2023-10-09T04:12:19.364962+0000","last_clean":"2023-10-09T04:12:19.364962+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:12:19.364962+0000","last_undegraded":"2023-10-09T04:12:19.364962+0000","last_fullsized":"2023-10-09T04:12:19.364962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"220'6182","reported_seq":6362,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.024279+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:12:28.024279+0000","last_peered":"2023-10-09T04:12:28.024279+0000","last_clean":"2023-10-09T04:12:28.024279+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:12:28.024279+0000","last_undegraded":"2023-10-09T04:12:28.024279+0000","last_fullsized":"2023-10-09T04:12:28.024279+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57409536,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3796,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":163,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.357665+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:12:19.357665+0000","last_peered":"2023-10-09T04:12:19.357665+0000","last_clean":"2023-10-09T04:12:19.357665+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:12:19.357665+0000","last_undegraded":"2023-10-09T04:12:19.357665+0000","last_fullsized":"2023-10-09T04:12:19.357665+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.359812+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:12:19.359812+0000","last_peered":"2023-10-09T04:12:19.359812+0000","last_clean":"2023-10-09T04:12:19.359812+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:12:19.359812+0000","last_undegraded":"2023-10-09T04:12:19.359812+0000","last_fullsized":"2023-10-09T04:12:19.359812+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"220'5916","reported_seq":5993,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.054604+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:12:25.054604+0000","last_peered":"2023-10-09T04:12:25.054604+0000","last_clean":"2023-10-09T04:12:25.054604+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:12:25.054604+0000","last_undegraded":"2023-10-09T04:12:25.054604+0000","last_fullsized":"2023-10-09T04:12:25.054604+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652416,"num_objects":956,"num_object_clones":0,"num_object_copies":2868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4776,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":179,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335538+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:12:19.335538+0000","last_peered":"2023-10-09T04:12:19.335538+0000","last_clean":"2023-10-09T04:12:19.335538+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:12:19.335538+0000","last_undegraded":"2023-10-09T04:12:19.335538+0000","last_fullsized":"2023-10-09T04:12:19.335538+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":175,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324676+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:12:19.324676+0000","last_peered":"2023-10-09T04:12:19.324676+0000","last_clean":"2023-10-09T04:12:19.324676+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:12:19.324676+0000","last_undegraded":"2023-10-09T04:12:19.324676+0000","last_fullsized":"2023-10-09T04:12:19.324676+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"220'9930","reported_seq":9991,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.028229+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:12:28.028229+0000","last_peered":"2023-10-09T04:12:28.028229+0000","last_clean":"2023-10-09T04:12:28.028229+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:12:28.028229+0000","last_undegraded":"2023-10-09T04:12:28.028229+0000","last_fullsized":"2023-10-09T04:12:28.028229+0000","mapping_epoch":167,"log_start":"175'2400","ondisk_log_start":"175'2400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7530,"log_dups_size":0,"ondisk_log_size":7530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9930,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":124,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324595+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:12:19.324595+0000","last_peered":"2023-10-09T04:12:19.324595+0000","last_clean":"2023-10-09T04:12:19.324595+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:12:19.324595+0000","last_undegraded":"2023-10-09T04:12:19.324595+0000","last_fullsized":"2023-10-09T04:12:19.324595+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.365680+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:12:19.365680+0000","last_peered":"2023-10-09T04:12:19.365680+0000","last_clean":"2023-10-09T04:12:19.365680+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:12:19.365680+0000","last_undegraded":"2023-10-09T04:12:19.365680+0000","last_fullsized":"2023-10-09T04:12:19.365680+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"220'9719","reported_seq":10617,"reported_epoch":220,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:12:25.600320+0000","last_change":"2023-10-09T04:12:18.137975+0000","last_active":"2023-10-09T04:12:25.600320+0000","last_peered":"2023-10-09T04:12:25.600320+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:12:25.600320+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":9719,"log_dups_size":0,"ondisk_log_size":9719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124452864,"num_objects":1899,"num_object_clones":0,"num_object_copies":5697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":1003,"num_objects_unfound":0,"num_objects_dirty":1899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9719,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":409,"num_bytes_recovered":26804224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,3,5","objects":1003},{"shards":"0,5","objects":111}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":139,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364761+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:12:19.364761+0000","last_peered":"2023-10-09T04:12:19.364761+0000","last_clean":"2023-10-09T04:12:19.364761+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:12:19.364761+0000","last_undegraded":"2023-10-09T04:12:19.364761+0000","last_fullsized":"2023-10-09T04:12:19.364761+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":215,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.365139+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:12:19.365139+0000","last_peered":"2023-10-09T04:12:19.365139+0000","last_clean":"2023-10-09T04:12:19.365139+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:12:19.365139+0000","last_undegraded":"2023-10-09T04:12:19.365139+0000","last_fullsized":"2023-10-09T04:12:19.365139+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"220'9884","reported_seq":9948,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.638080+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:12:28.638080+0000","last_peered":"2023-10-09T04:12:28.638080+0000","last_clean":"2023-10-09T04:12:28.638080+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:12:28.638080+0000","last_undegraded":"2023-10-09T04:12:28.638080+0000","last_fullsized":"2023-10-09T04:12:28.638080+0000","mapping_epoch":167,"log_start":"183'2900","ondisk_log_start":"183'2900","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113704984,"num_objects":1736,"num_object_clones":0,"num_object_copies":5208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1736,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9884,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":179,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364695+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:12:19.364695+0000","last_peered":"2023-10-09T04:12:19.364695+0000","last_clean":"2023-10-09T04:12:19.364695+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:12:19.364695+0000","last_undegraded":"2023-10-09T04:12:19.364695+0000","last_fullsized":"2023-10-09T04:12:19.364695+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":171,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.365722+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:12:19.365722+0000","last_peered":"2023-10-09T04:12:19.365722+0000","last_clean":"2023-10-09T04:12:19.365722+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:12:19.365722+0000","last_undegraded":"2023-10-09T04:12:19.365722+0000","last_fullsized":"2023-10-09T04:12:19.365722+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"220'9590","reported_seq":9653,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.753472+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:12:25.753472+0000","last_peered":"2023-10-09T04:12:25.753472+0000","last_clean":"2023-10-09T04:12:25.753472+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:12:25.753472+0000","last_undegraded":"2023-10-09T04:12:25.753472+0000","last_fullsized":"2023-10-09T04:12:25.753472+0000","mapping_epoch":167,"log_start":"186'3200","ondisk_log_start":"186'3200","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120586240,"num_objects":1840,"num_object_clones":0,"num_object_copies":5520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9590,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.321452+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:12:19.321452+0000","last_peered":"2023-10-09T04:12:19.321452+0000","last_clean":"2023-10-09T04:12:19.321452+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:12:19.321452+0000","last_undegraded":"2023-10-09T04:12:19.321452+0000","last_fullsized":"2023-10-09T04:12:19.321452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":172,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.360386+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:12:19.360386+0000","last_peered":"2023-10-09T04:12:19.360386+0000","last_clean":"2023-10-09T04:12:19.360386+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:12:19.360386+0000","last_undegraded":"2023-10-09T04:12:19.360386+0000","last_fullsized":"2023-10-09T04:12:19.360386+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"220'6047","reported_seq":6242,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.228146+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:12:25.228146+0000","last_peered":"2023-10-09T04:12:25.228146+0000","last_clean":"2023-10-09T04:12:25.228146+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:12:25.228146+0000","last_undegraded":"2023-10-09T04:12:25.228146+0000","last_fullsized":"2023-10-09T04:12:25.228146+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":6047,"log_dups_size":0,"ondisk_log_size":6047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62324736,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2394,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":195,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.361170+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:12:19.361170+0000","last_peered":"2023-10-09T04:12:19.361170+0000","last_clean":"2023-10-09T04:12:19.361170+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:12:19.361170+0000","last_undegraded":"2023-10-09T04:12:19.361170+0000","last_fullsized":"2023-10-09T04:12:19.361170+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":196,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335648+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:12:19.335648+0000","last_peered":"2023-10-09T04:12:19.335648+0000","last_clean":"2023-10-09T04:12:19.335648+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:19.335648+0000","last_undegraded":"2023-10-09T04:12:19.335648+0000","last_fullsized":"2023-10-09T04:12:19.335648+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":157,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.360594+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:12:19.360594+0000","last_peered":"2023-10-09T04:12:19.360594+0000","last_clean":"2023-10-09T04:12:19.360594+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:12:19.360594+0000","last_undegraded":"2023-10-09T04:12:19.360594+0000","last_fullsized":"2023-10-09T04:12:19.360594+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"220'6174","reported_seq":6247,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:27.994903+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:12:27.994903+0000","last_peered":"2023-10-09T04:12:27.994903+0000","last_clean":"2023-10-09T04:12:27.994903+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:12:27.994903+0000","last_undegraded":"2023-10-09T04:12:27.994903+0000","last_fullsized":"2023-10-09T04:12:27.994903+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54132736,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5020,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":183,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335506+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:12:19.335506+0000","last_peered":"2023-10-09T04:12:19.335506+0000","last_clean":"2023-10-09T04:12:19.335506+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:12:19.335506+0000","last_undegraded":"2023-10-09T04:12:19.335506+0000","last_fullsized":"2023-10-09T04:12:19.335506+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":179,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.366137+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:12:19.366137+0000","last_peered":"2023-10-09T04:12:19.366137+0000","last_clean":"2023-10-09T04:12:19.366137+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:12:19.366137+0000","last_undegraded":"2023-10-09T04:12:19.366137+0000","last_fullsized":"2023-10-09T04:12:19.366137+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":205,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.361103+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:12:19.361103+0000","last_peered":"2023-10-09T04:12:19.361103+0000","last_clean":"2023-10-09T04:12:19.361103+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:12:19.361103+0000","last_undegraded":"2023-10-09T04:12:19.361103+0000","last_fullsized":"2023-10-09T04:12:19.361103+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"220'6183","reported_seq":6256,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.034540+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:12:28.034540+0000","last_peered":"2023-10-09T04:12:28.034540+0000","last_clean":"2023-10-09T04:12:28.034540+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:12:28.034540+0000","last_undegraded":"2023-10-09T04:12:28.034540+0000","last_fullsized":"2023-10-09T04:12:28.034540+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6183,"log_dups_size":0,"ondisk_log_size":6183,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5021,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":33,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.328304+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:12:19.328304+0000","last_peered":"2023-10-09T04:12:19.328304+0000","last_clean":"2023-10-09T04:12:19.328304+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:12:19.328304+0000","last_undegraded":"2023-10-09T04:12:19.328304+0000","last_fullsized":"2023-10-09T04:12:19.328304+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":189,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.359821+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:12:19.359821+0000","last_peered":"2023-10-09T04:12:19.359821+0000","last_clean":"2023-10-09T04:12:19.359821+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:12:19.359821+0000","last_undegraded":"2023-10-09T04:12:19.359821+0000","last_fullsized":"2023-10-09T04:12:19.359821+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.360455+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:12:19.360455+0000","last_peered":"2023-10-09T04:12:19.360455+0000","last_clean":"2023-10-09T04:12:19.360455+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:12:19.360455+0000","last_undegraded":"2023-10-09T04:12:19.360455+0000","last_fullsized":"2023-10-09T04:12:19.360455+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"220'5968","reported_seq":6040,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:27.994861+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:12:27.994861+0000","last_peered":"2023-10-09T04:12:27.994861+0000","last_clean":"2023-10-09T04:12:27.994861+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:12:27.994861+0000","last_undegraded":"2023-10-09T04:12:27.994861+0000","last_fullsized":"2023-10-09T04:12:27.994861+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":5968,"log_dups_size":0,"ondisk_log_size":5968,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57704448,"num_objects":880,"num_object_clones":0,"num_object_copies":2640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4850,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":203,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335463+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:12:19.335463+0000","last_peered":"2023-10-09T04:12:19.335463+0000","last_clean":"2023-10-09T04:12:19.335463+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:12:19.335463+0000","last_undegraded":"2023-10-09T04:12:19.335463+0000","last_fullsized":"2023-10-09T04:12:19.335463+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":560,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.356195+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:12:19.356195+0000","last_peered":"2023-10-09T04:12:19.356195+0000","last_clean":"2023-10-09T04:12:19.356195+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:12:19.356195+0000","last_undegraded":"2023-10-09T04:12:19.356195+0000","last_fullsized":"2023-10-09T04:12:19.356195+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":116,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335457+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:12:19.335457+0000","last_peered":"2023-10-09T04:12:19.335457+0000","last_clean":"2023-10-09T04:12:19.335457+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:12:19.335457+0000","last_undegraded":"2023-10-09T04:12:19.335457+0000","last_fullsized":"2023-10-09T04:12:19.335457+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":99,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.365333+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:12:19.365333+0000","last_peered":"2023-10-09T04:12:19.365333+0000","last_clean":"2023-10-09T04:12:19.365333+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:12:19.365333+0000","last_undegraded":"2023-10-09T04:12:19.365333+0000","last_fullsized":"2023-10-09T04:12:19.365333+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"220'6011","reported_seq":8365,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.395495+0000","last_change":"2023-10-09T04:10:53.276397+0000","last_active":"2023-10-09T04:12:25.395495+0000","last_peered":"2023-10-09T04:12:25.395495+0000","last_clean":"2023-10-09T04:12:25.395495+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:12:25.395495+0000","last_undegraded":"2023-10-09T04:12:25.395495+0000","last_fullsized":"2023-10-09T04:12:25.395495+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6011,"log_dups_size":0,"ondisk_log_size":6011,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57638912,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4895,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":200,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.356179+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:12:19.356179+0000","last_peered":"2023-10-09T04:12:19.356179+0000","last_clean":"2023-10-09T04:12:19.356179+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:12:19.356179+0000","last_undegraded":"2023-10-09T04:12:19.356179+0000","last_fullsized":"2023-10-09T04:12:19.356179+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":176,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324763+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:12:19.324763+0000","last_peered":"2023-10-09T04:12:19.324763+0000","last_clean":"2023-10-09T04:12:19.324763+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:12:19.324763+0000","last_undegraded":"2023-10-09T04:12:19.324763+0000","last_fullsized":"2023-10-09T04:12:19.324763+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":208,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324961+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:12:19.324961+0000","last_peered":"2023-10-09T04:12:19.324961+0000","last_clean":"2023-10-09T04:12:19.324961+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:12:19.324961+0000","last_undegraded":"2023-10-09T04:12:19.324961+0000","last_fullsized":"2023-10-09T04:12:19.324961+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"220'6206","reported_seq":6341,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.596785+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:12:25.596785+0000","last_peered":"2023-10-09T04:12:25.596785+0000","last_clean":"2023-10-09T04:12:25.596785+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:12:25.596785+0000","last_undegraded":"2023-10-09T04:12:25.596785+0000","last_fullsized":"2023-10-09T04:12:25.596785+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62849024,"num_objects":959,"num_object_clones":0,"num_object_copies":2877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3659,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":183,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.357710+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:12:19.357710+0000","last_peered":"2023-10-09T04:12:19.357710+0000","last_clean":"2023-10-09T04:12:19.357710+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:12:19.357710+0000","last_undegraded":"2023-10-09T04:12:19.357710+0000","last_fullsized":"2023-10-09T04:12:19.357710+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":189,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.361245+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:12:19.361245+0000","last_peered":"2023-10-09T04:12:19.361245+0000","last_clean":"2023-10-09T04:12:19.361245+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:12:19.361245+0000","last_undegraded":"2023-10-09T04:12:19.361245+0000","last_fullsized":"2023-10-09T04:12:19.361245+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":161,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579132+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:12:24.579132+0000","last_peered":"2023-10-09T04:12:24.579132+0000","last_clean":"2023-10-09T04:12:24.579132+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:12:24.579132+0000","last_undegraded":"2023-10-09T04:12:24.579132+0000","last_fullsized":"2023-10-09T04:12:24.579132+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"220'6197","reported_seq":9263,"reported_epoch":220,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:25.735918+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:12:25.735918+0000","last_peered":"2023-10-09T04:12:25.735918+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:12:25.735918+0000","last_undegraded":"2023-10-09T04:12:25.735918+0000","last_fullsized":"2023-10-09T04:12:25.735918+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62259200,"num_objects":950,"num_object_clones":0,"num_object_copies":2850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":950,"num_objects_unfound":0,"num_objects_dirty":950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5026,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":950}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":124,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.357922+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:12:19.357922+0000","last_peered":"2023-10-09T04:12:19.357922+0000","last_clean":"2023-10-09T04:12:19.357922+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:12:19.357922+0000","last_undegraded":"2023-10-09T04:12:19.357922+0000","last_fullsized":"2023-10-09T04:12:19.357922+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":156,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.360676+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:12:19.360676+0000","last_peered":"2023-10-09T04:12:19.360676+0000","last_clean":"2023-10-09T04:12:19.360676+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:12:19.360676+0000","last_undegraded":"2023-10-09T04:12:19.360676+0000","last_fullsized":"2023-10-09T04:12:19.360676+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":150,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.356333+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:12:19.356333+0000","last_peered":"2023-10-09T04:12:19.356333+0000","last_clean":"2023-10-09T04:12:19.356333+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:12:19.356333+0000","last_undegraded":"2023-10-09T04:12:19.356333+0000","last_fullsized":"2023-10-09T04:12:19.356333+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"220'6137","reported_seq":8353,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.660617+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:12:28.660617+0000","last_peered":"2023-10-09T04:12:28.660617+0000","last_clean":"2023-10-09T04:12:28.660617+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:12:28.660617+0000","last_undegraded":"2023-10-09T04:12:28.660617+0000","last_fullsized":"2023-10-09T04:12:28.660617+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":6137,"log_dups_size":0,"ondisk_log_size":6137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3660,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":148,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.357990+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:12:19.357990+0000","last_peered":"2023-10-09T04:12:19.357990+0000","last_clean":"2023-10-09T04:12:19.357990+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:12:19.357990+0000","last_undegraded":"2023-10-09T04:12:19.357990+0000","last_fullsized":"2023-10-09T04:12:19.357990+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":179,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.325006+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:12:19.325006+0000","last_peered":"2023-10-09T04:12:19.325006+0000","last_clean":"2023-10-09T04:12:19.325006+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:12:19.325006+0000","last_undegraded":"2023-10-09T04:12:19.325006+0000","last_fullsized":"2023-10-09T04:12:19.325006+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":161,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364768+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:12:19.364768+0000","last_peered":"2023-10-09T04:12:19.364768+0000","last_clean":"2023-10-09T04:12:19.364768+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:12:19.364768+0000","last_undegraded":"2023-10-09T04:12:19.364768+0000","last_fullsized":"2023-10-09T04:12:19.364768+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":164,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.324760+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:12:19.324760+0000","last_peered":"2023-10-09T04:12:19.324760+0000","last_clean":"2023-10-09T04:12:19.324760+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:12:19.324760+0000","last_undegraded":"2023-10-09T04:12:19.324760+0000","last_fullsized":"2023-10-09T04:12:19.324760+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":33,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.328046+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:12:19.328046+0000","last_peered":"2023-10-09T04:12:19.328046+0000","last_clean":"2023-10-09T04:12:19.328046+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:12:19.328046+0000","last_undegraded":"2023-10-09T04:12:19.328046+0000","last_fullsized":"2023-10-09T04:12:19.328046+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"220'6033","reported_seq":6108,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:25.012906+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:12:25.012906+0000","last_peered":"2023-10-09T04:12:25.012906+0000","last_clean":"2023-10-09T04:12:25.012906+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:12:25.012906+0000","last_undegraded":"2023-10-09T04:12:25.012906+0000","last_fullsized":"2023-10-09T04:12:25.012906+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6033,"log_dups_size":0,"ondisk_log_size":6033,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60620800,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4893,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":181,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335588+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:12:19.335588+0000","last_peered":"2023-10-09T04:12:19.335588+0000","last_clean":"2023-10-09T04:12:19.335588+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:12:19.335588+0000","last_undegraded":"2023-10-09T04:12:19.335588+0000","last_fullsized":"2023-10-09T04:12:19.335588+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":195,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.366210+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:12:19.366210+0000","last_peered":"2023-10-09T04:12:19.366210+0000","last_clean":"2023-10-09T04:12:19.366210+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:12:19.366210+0000","last_undegraded":"2023-10-09T04:12:19.366210+0000","last_fullsized":"2023-10-09T04:12:19.366210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"220'6214","reported_seq":6283,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.044736+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:12:28.044736+0000","last_peered":"2023-10-09T04:12:28.044736+0000","last_clean":"2023-10-09T04:12:28.044736+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:12:28.044736+0000","last_undegraded":"2023-10-09T04:12:28.044736+0000","last_fullsized":"2023-10-09T04:12:28.044736+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55836672,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":164,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.335675+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:12:19.335675+0000","last_peered":"2023-10-09T04:12:19.335675+0000","last_clean":"2023-10-09T04:12:19.335675+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:12:19.335675+0000","last_undegraded":"2023-10-09T04:12:19.335675+0000","last_fullsized":"2023-10-09T04:12:19.335675+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":195,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364834+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:12:19.364834+0000","last_peered":"2023-10-09T04:12:19.364834+0000","last_clean":"2023-10-09T04:12:19.364834+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:12:19.364834+0000","last_undegraded":"2023-10-09T04:12:19.364834+0000","last_fullsized":"2023-10-09T04:12:19.364834+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":33,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.327861+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:12:19.327861+0000","last_peered":"2023-10-09T04:12:19.327861+0000","last_clean":"2023-10-09T04:12:19.327861+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:12:19.327861+0000","last_undegraded":"2023-10-09T04:12:19.327861+0000","last_fullsized":"2023-10-09T04:12:19.327861+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":33,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.327795+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:12:19.327795+0000","last_peered":"2023-10-09T04:12:19.327795+0000","last_clean":"2023-10-09T04:12:19.327795+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:12:19.327795+0000","last_undegraded":"2023-10-09T04:12:19.327795+0000","last_fullsized":"2023-10-09T04:12:19.327795+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":210,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.325602+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:12:19.325602+0000","last_peered":"2023-10-09T04:12:19.325602+0000","last_clean":"2023-10-09T04:12:19.325602+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:12:19.325602+0000","last_undegraded":"2023-10-09T04:12:19.325602+0000","last_fullsized":"2023-10-09T04:12:19.325602+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":178,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-10-09T04:12:19.364501+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:12:19.364501+0000","last_peered":"2023-10-09T04:12:19.364501+0000","last_clean":"2023-10-09T04:12:19.364501+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:12:19.364501+0000","last_undegraded":"2023-10-09T04:12:19.364501+0000","last_fullsized":"2023-10-09T04:12:19.364501+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1896087576,"num_objects":28933,"num_object_clones":0,"num_object_copies":86799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":3828,"num_objects_unfound":0,"num_objects_dirty":28933,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":151164,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12284,"num_bytes_recovered":805044224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6336557056,"data_stored":6336544840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169198,"ondisk_log_size":169198,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201519,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1076136,"kb_used_data":770656,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92669016,"statfs":{"total":95995035648,"available":94893072384,"internally_reserved":0,"allocated":789151744,"data_stored":788485245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":10.025,"5min":3.5390000000000001,"15min":1.583},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":126.752,"5min":126.752,"15min":126.752},"last":0.89800000000000002},{"interface":"front","average":{"1min":10.521000000000001,"5min":3.609,"15min":1.6060000000000001},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":128.88399999999999,"5min":128.88399999999999,"15min":128.88399999999999},"last":1.7949999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":10.563000000000001,"5min":5.8120000000000003,"15min":2.3980000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":103.014,"5min":245.28700000000001,"15min":245.28700000000001},"last":4.6120000000000001},{"interface":"front","average":{"1min":10.186999999999999,"5min":5.3700000000000001,"15min":2.379},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":103.083,"5min":244.626,"15min":244.626},"last":1.3979999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.8799999999999999,"15min":1.268},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4009999999999998,"5min":13.887,"15min":58.615000000000002},"last":3.4550000000000001},{"interface":"front","average":{"1min":1.401,"5min":2.9279999999999999,"15min":1.6850000000000001},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.3109999999999999,"5min":99.372,"15min":99.372},"last":3.4249999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.5469999999999999,"15min":1.5589999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":3.0430000000000001,"5min":13.124000000000001,"15min":99.134},"last":3.0939999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.6599999999999999,"15min":1.6399999999999999},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.0230000000000001,"5min":13.140000000000001,"15min":99.117999999999995},"last":3.1070000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.079,"15min":0.69699999999999995},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.9790000000000001,"5min":16.195,"15min":16.195},"last":0.372},{"interface":"front","average":{"1min":0.442,"5min":1.0109999999999999,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.052,"5min":12.442,"15min":12.442},"last":0.34999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.77800000000000002,"15min":0.59699999999999998},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81999999999999995,"5min":6.5439999999999996,"15min":6.5439999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.873,"15min":0.67800000000000005},"min":{"1min":0.32100000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.0550000000000002,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.42399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:11:25 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.91300000000000003,"15min":0.67600000000000005},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.99399999999999999,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.435},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.03,"15min":0.745},"min":{"1min":0.36599999999999999,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":2.0819999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.45700000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463156,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097616,"kb_used_data":800328,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92647536,"statfs":{"total":95995035648,"available":94871076864,"internally_reserved":0,"allocated":819535872,"data_stored":818885246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":6.4000000000000004,"5min":4.4550000000000001,"15min":2.2370000000000001},"min":{"1min":0.222,"5min":0.217,"15min":0.217},"max":{"1min":77.453000000000003,"5min":77.453000000000003,"15min":78.983999999999995},"last":0.21299999999999999},{"interface":"front","average":{"1min":6.9790000000000001,"5min":4.9480000000000004,"15min":2.5129999999999999},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":77.430999999999997,"5min":77.430999999999997,"15min":78.998000000000005},"last":0.29799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.657,"15min":1.6950000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":4.6840000000000002,"5min":62.698,"15min":62.698},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.909,"5min":3.875,"15min":1.794},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":8.3529999999999998,"5min":63.530999999999999,"15min":63.530999999999999},"last":2.133}]},{"osd":2,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":3.7120000000000002,"15min":2.0350000000000001},"min":{"1min":0.33700000000000002,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":33.345999999999997,"5min":77.953000000000003,"15min":77.953000000000003},"last":2.1000000000000001},{"interface":"front","average":{"1min":3.5680000000000001,"5min":3.7610000000000001,"15min":2.085},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":33.368000000000002,"5min":78.019000000000005,"15min":78.019000000000005},"last":2.1219999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":3.516,"15min":2.0339999999999998},"min":{"1min":0.35799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.336,"5min":57.509999999999998,"15min":57.509999999999998},"last":2.0840000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":3.7450000000000001,"15min":2.052},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1840000000000002,"5min":57.453000000000003,"15min":57.453000000000003},"last":1.8100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.3720000000000001,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.72199999999999998,"15min":0.64400000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2970000000000002,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.32600000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":0.84599999999999997,"15min":0.84899999999999998},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.317},{"interface":"front","average":{"1min":2.081,"5min":0.97399999999999998,"15min":0.95499999999999996},"min":{"1min":0.34399999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":23.006,"5min":23.006,"15min":54.853999999999999},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68600000000000005,"15min":0.628},"min":{"1min":0.317,"5min":0.313,"15min":0.313},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.79000000000000004,"15min":0.71199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":3.3679999999999999,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.35699999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089465,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073056,"kb_used_data":822680,"kb_used_omap":9,"kb_used_meta":250294,"kb_avail":92672096,"statfs":{"total":95995035648,"available":94896226304,"internally_reserved":0,"allocated":842424320,"data_stored":841530597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":256302057},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":6.4269999999999996,"5min":3.7930000000000001,"15min":1.843},"min":{"1min":0.247,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":61.689999999999998,"5min":85.977999999999994,"15min":85.977999999999994},"last":0.48699999999999999},{"interface":"front","average":{"1min":7.4409999999999998,"5min":4.2519999999999998,"15min":2.3519999999999999},"min":{"1min":0.31900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":61.670000000000002,"5min":85.998999999999995,"15min":99.831999999999994},"last":0.498}]},{"osd":2,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.9359999999999999,"15min":1.7589999999999999},"min":{"1min":0.26100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":13.695,"5min":77.563000000000002,"15min":121.54600000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":4.9939999999999998,"5min":3.452,"15min":2.0110000000000001},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.186},"max":{"1min":15.298999999999999,"5min":77.504999999999995,"15min":99.899000000000001},"last":0.25}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":4.2089999999999996,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":7.8920000000000003,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.593,"5min":4.6449999999999996,"15min":2.6309999999999998},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.414,"5min":138.49799999999999,"15min":138.49799999999999},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":1.8520000000000001,"15min":1.498},"min":{"1min":0.35599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":15.215999999999999,"5min":15.215999999999999,"15min":99.989999999999995},"last":3.4279999999999999},{"interface":"front","average":{"1min":1.115,"5min":1.5129999999999999,"15min":1.3779999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.52,"5min":10.438000000000001,"15min":100.249},"last":2.4319999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.75,"15min":1.48},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.266,"5min":15.266,"15min":99.933999999999997},"last":0.316},{"interface":"front","average":{"1min":1.389,"5min":1.829,"15min":1.4950000000000001},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":4.8849999999999998,"5min":10.467000000000001,"15min":100.34099999999999},"last":0.39200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.827,"15min":1.5880000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.585,"5min":10.497999999999999,"15min":100.309},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.671,"5min":1.496,"15min":1.5169999999999999},"min":{"1min":0.36499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9240000000000004,"5min":7.9240000000000004,"15min":100.143},"last":4.7759999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.9630000000000001,"15min":1.569},"min":{"1min":0.40000000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.9569999999999999,"5min":10.532,"15min":100.214},"last":2.5419999999999998},{"interface":"front","average":{"1min":1.798,"5min":1.796,"15min":1.5429999999999999},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":100.113},"last":2.9390000000000001}]}]},{"osd":0,"up_from":84,"seq":360777252991,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1214904,"kb_used_data":970864,"kb_used_omap":28,"kb_used_meta":243811,"kb_avail":92530248,"statfs":{"total":95995035648,"available":94750973952,"internally_reserved":0,"allocated":994164736,"data_stored":993246260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":249663312},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":3.6059999999999999,"15min":1.982},"min":{"1min":0.184,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":12.698,"5min":103.996,"15min":118.48},"last":37.764000000000003},{"interface":"front","average":{"1min":3.5910000000000002,"5min":4.1959999999999997,"15min":2.2080000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":12.632999999999999,"5min":106.34399999999999,"15min":118.384},"last":38.128999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:12:21 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":3.0339999999999998,"15min":2.2490000000000001},"min":{"1min":0.38,"5min":0.221,"15min":0.221},"max":{"1min":13.108000000000001,"5min":25.055,"15min":118.54000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":4.5979999999999999,"5min":3.2890000000000001,"15min":2.367},"min":{"1min":0.22900000000000001,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":12.372,"5min":44.436999999999998,"15min":118.727},"last":0.27200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":5.9320000000000004,"15min":3.601},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.161,"5min":156.488,"15min":156.488},"last":0.23599999999999999},{"interface":"front","average":{"1min":4.1909999999999998,"5min":5.8979999999999997,"15min":3.6890000000000001},"min":{"1min":0.40699999999999997,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":9.6010000000000009,"5min":156.50700000000001,"15min":156.50700000000001},"last":0.28799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":2.3540000000000001,"15min":1.625},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.8310000000000004,"5min":59.439999999999998,"15min":118.633},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.151,"5min":2.9889999999999999,"15min":1.897},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.593,"5min":59.453000000000003,"15min":118.913},"last":0.39800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":4.548,"15min":2.4169999999999998},"min":{"1min":0.32900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.6310000000000002,"5min":182.68899999999999,"15min":182.68899999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.302,"5min":4.2830000000000004,"15min":2.3039999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.8919999999999995,"5min":182.66300000000001,"15min":182.66300000000001},"last":0.41299999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":8.0259999999999998,"5min":3.976,"15min":2.2109999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":137.36099999999999,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":7.3899999999999997,"5min":3.5800000000000001,"15min":2.0670000000000002},"min":{"1min":0.40899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":136.88999999999999,"5min":136.88999999999999,"15min":136.88999999999999},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:32 2023","interfaces":[{"interface":"back","average":{"1min":7.2210000000000001,"5min":5.7480000000000002,"15min":2.8370000000000002},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":121.324,"5min":185.024,"15min":185.024},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.206,"5min":4.21,"15min":2.3180000000000001},"min":{"1min":0.441,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":6.6280000000000001,"5min":184.98599999999999,"15min":184.98599999999999},"last":0.46700000000000003}]}]},{"osd":2,"up_from":94,"seq":403726925938,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1031936,"kb_used_data":704120,"kb_used_omap":81,"kb_used_meta":327726,"kb_avail":92713216,"statfs":{"total":95995035648,"available":94938333184,"internally_reserved":0,"allocated":721018880,"data_stored":720147034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":335591642},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2460000000000004,"5min":2.9359999999999999,"15min":1.4690000000000001},"min":{"1min":0.20200000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":22.321000000000002,"5min":22.321000000000002,"15min":22.321000000000002},"last":1.774},{"interface":"front","average":{"1min":3.6800000000000002,"5min":3.581,"15min":1.738},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.6440000000000001,"5min":45.979999999999997,"15min":45.979999999999997},"last":1.55}]},{"osd":1,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":3.5910000000000002,"15min":1.9730000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":9.9429999999999996,"5min":53.695,"15min":68.009},"last":1.496},{"interface":"front","average":{"1min":3.7149999999999999,"5min":3.7090000000000001,"15min":2.1419999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.346,"5min":53.439,"15min":66.975999999999999},"last":0.21099999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":2.2029999999999998,"15min":1.349},"min":{"1min":0.22500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.212999999999999,"5min":12.77,"15min":27.562000000000001},"last":1.7949999999999999},{"interface":"front","average":{"1min":2.577,"5min":2.8199999999999998,"15min":1.552},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.808,"5min":46.286999999999999,"15min":46.286999999999999},"last":1.575}]},{"osd":4,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.2490000000000001,"15min":1.444},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5279999999999996,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.25900000000000001},{"interface":"front","average":{"1min":3.609,"5min":1.9419999999999999,"15min":1.234},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":20.134,"5min":20.134,"15min":20.134},"last":1.7030000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.6789999999999998,"15min":1.4770000000000001},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":7.3399999999999999,"5min":46.265000000000001,"15min":46.265000000000001},"last":1.4299999999999999},{"interface":"front","average":{"1min":2.1779999999999999,"5min":2.6339999999999999,"15min":1.6020000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5039999999999996,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.28599999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6579999999999999,"5min":3.2829999999999999,"15min":1.671},"min":{"1min":0.376,"5min":0.311,"15min":0.311},"max":{"1min":20.184000000000001,"5min":114.14,"15min":114.14},"last":1.73},{"interface":"front","average":{"1min":2.194,"5min":3.7719999999999998,"15min":1.8560000000000001},"min":{"1min":0.43099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4199999999999999,"5min":114.123,"15min":114.123},"last":1.47}]},{"osd":7,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.556,"15min":1.8640000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":20.215,"5min":117.32599999999999,"15min":117.32599999999999},"last":1.752},{"interface":"front","average":{"1min":2.6349999999999998,"5min":3.9209999999999998,"15min":1.982},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.4659999999999993,"5min":117.28700000000001,"15min":117.28700000000001},"last":1.524}]}]},{"osd":3,"up_from":99,"seq":425201762412,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1673272,"kb_used_data":1248496,"kb_used_omap":7,"kb_used_meta":424568,"kb_avail":92071880,"statfs":{"total":95995035648,"available":94281605120,"internally_reserved":0,"allocated":1278459904,"data_stored":1277601350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":434758248},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8140000000000001,"5min":4.6970000000000001,"15min":3.3519999999999999},"min":{"1min":0.20100000000000001,"5min":0.189,"15min":0.185},"max":{"1min":23.449000000000002,"5min":100.512,"15min":106.22},"last":7.7290000000000001},{"interface":"front","average":{"1min":4.0759999999999996,"5min":5.056,"15min":3.4039999999999999},"min":{"1min":0.32200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":13.670999999999999,"5min":101.733,"15min":106.986},"last":8.4030000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":3.5699999999999998,"15min":2.2490000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":8.6340000000000003,"5min":57.055999999999997,"15min":106.121},"last":4.3940000000000001},{"interface":"front","average":{"1min":3.899,"5min":3.9180000000000001,"15min":2.1019999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":23.475999999999999,"5min":57.088999999999999,"15min":65.015000000000001},"last":2.6299999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.8130000000000002,"15min":2.1669999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":23.405999999999999,"5min":23.405999999999999,"15min":106.105},"last":2.665},{"interface":"front","average":{"1min":3.524,"5min":3.7229999999999999,"15min":2.5190000000000001},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":13.698,"5min":98.765000000000001,"15min":106.70399999999999},"last":8.3710000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":2.9060000000000001,"15min":1.927},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.308},"max":{"1min":13.536,"5min":37.652000000000001,"15min":106.85899999999999},"last":7.3810000000000002},{"interface":"front","average":{"1min":1.873,"5min":2.5209999999999999,"15min":1.351},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.1699999999999999,"5min":37.442,"15min":37.442},"last":4.4829999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.1150000000000002,"15min":1.2430000000000001},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.1940000000000008,"5min":12.819000000000001,"15min":12.819000000000001},"last":4.5499999999999998},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.9730000000000001,"15min":1.516},"min":{"1min":0.32400000000000001,"5min":0.29599999999999999,"15min":0.27900000000000003},"max":{"1min":11.938000000000001,"5min":13.228999999999999,"15min":13.228999999999999},"last":2.7200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.8560000000000001,"15min":1.9039999999999999},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.30199999999999999},"max":{"1min":11.994999999999999,"5min":13.255000000000001,"15min":106.036},"last":2.6949999999999998},{"interface":"front","average":{"1min":2.363,"5min":2.355,"15min":1.7789999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.571,"5min":15.622,"15min":106.741},"last":7.4379999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":2.5099999999999998,"15min":1.8260000000000001},"min":{"1min":0.38700000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.494999999999999,"5min":15.685,"15min":106.82599999999999},"last":7.4699999999999998},{"interface":"front","average":{"1min":2.1480000000000001,"5min":2.1749999999999998,"15min":1.7010000000000001},"min":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.2170000000000005,"5min":13.228999999999999,"15min":106.084},"last":4.5179999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986428,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":855708,"kb_used_data":550228,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92889444,"statfs":{"total":95995035648,"available":95118790656,"internally_reserved":0,"allocated":563433472,"data_stored":562778382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7560000000000002,"5min":3.3079999999999998,"15min":1.631},"min":{"1min":0.222,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":38.831000000000003,"5min":63.128,"15min":63.128},"last":7.3259999999999996},{"interface":"front","average":{"1min":4.7939999999999996,"5min":3.0590000000000002,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.039999999999999,"5min":62.645000000000003,"15min":62.645000000000003},"last":4.7370000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.7709999999999999,"15min":1.4410000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":23.157,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.8410000000000002,"5min":2.8479999999999999,"15min":1.4299999999999999},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":23.091999999999999,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.41499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.8380000000000001,"15min":1.0900000000000001},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.698,"5min":19.356000000000002,"15min":19.356000000000002},"last":0.5},{"interface":"front","average":{"1min":1.125,"5min":1.845,"15min":1.042},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7109999999999999,"5min":19.324000000000002,"15min":19.324000000000002},"last":0.49099999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.8779999999999999,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4880000000000004,"5min":14.694000000000001,"15min":14.694000000000001},"last":2.0899999999999999},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.958,"15min":1.222},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.2110000000000003,"5min":10.476000000000001,"15min":10.476000000000001},"last":1.232}]},{"osd":5,"last update":"Mon Oct 9 04:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.70999999999999996,"15min":0.57599999999999996},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.3100000000000001,"5min":8.5259999999999998,"15min":8.5259999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.92600000000000005,"15min":0.71599999999999997},"min":{"1min":0.27400000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.7290000000000001,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.93799999999999994,"15min":0.74099999999999999},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":2.9990000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.81899999999999995,"15min":0.66600000000000004},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9700000000000002,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.70299999999999996,"15min":0.70699999999999996},"min":{"1min":0.311,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":2.3599999999999999,"5min":3.5190000000000001,"15min":9.0050000000000008},"last":0.435},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.79100000000000004,"15min":0.77200000000000002},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.29199999999999998},"max":{"1min":2.214,"5min":5.9939999999999998,"15min":8.5440000000000005},"last":0.59299999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724792,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":740684,"kb_used_data":431108,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93004468,"statfs":{"total":95995035648,"available":95236575232,"internally_reserved":0,"allocated":441454592,"data_stored":440811572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":6.391,"5min":3.4849999999999999,"15min":1.577},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":83.329999999999998,"5min":83.329999999999998,"15min":83.329999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":7.7030000000000003,"5min":3.778,"15min":1.6990000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":85.451999999999998,"5min":85.451999999999998,"15min":85.451999999999998},"last":0.72899999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:11:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3869999999999996,"5min":2.6749999999999998,"15min":1.7030000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":59.588999999999999,"5min":59.588999999999999,"15min":91.212999999999994},"last":0.26100000000000001},{"interface":"front","average":{"1min":4.6139999999999999,"5min":2.9550000000000001,"15min":1.913},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":59.533000000000001,"5min":59.533000000000001,"15min":91.254000000000005},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":3.0880000000000001,"15min":1.524},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.2959999999999998,"5min":61.310000000000002,"15min":61.310000000000002},"last":9.0860000000000003},{"interface":"front","average":{"1min":1.968,"5min":3.2050000000000001,"15min":1.518},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.6929999999999996,"5min":61.154000000000003,"15min":61.154000000000003},"last":9.1669999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.772,"15min":1.7010000000000001},"min":{"1min":0.35299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.714,"5min":17.071000000000002,"15min":91.834999999999994},"last":0.377},{"interface":"front","average":{"1min":1.548,"5min":1.885,"15min":1.7450000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.309},"max":{"1min":5.9749999999999996,"5min":17.052,"15min":91.802000000000007},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.88900000000000001,"15min":0.65700000000000003},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.253},"max":{"1min":3.2440000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.95599999999999996,"15min":0.71799999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26600000000000001},"max":{"1min":3.4420000000000002,"5min":6.7709999999999999,"15min":6.7709999999999999},"last":0.33700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2839999999999998,"5min":2.1230000000000002,"15min":1.113},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":123.239,"5min":123.239,"15min":123.239},"last":0.29299999999999998},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.0899999999999999,"15min":1.129},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":123.014,"5min":123.014,"15min":123.014},"last":0.30199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.98999999999999999,"15min":0.78300000000000003},"min":{"1min":0.307,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2450000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.90300000000000002,"15min":0.80300000000000005},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":1.8220000000000001,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.39800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":973148160,"data_stored":973144088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":837881856,"data_stored":837877784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":708182016,"data_stored":708182016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1254621184,"data_stored":1254621184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":558628864,"data_stored":558628864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":430899200,"data_stored":430899200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":800067584,"data_stored":800063512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":773128192,"data_stored":773128192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:12:32.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:12:32.691 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:32.761 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[213841]: from='client.55066 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:32.761 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[213841]: pgmap v584: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 785/81636 objects degraded (0.962%); 3262/81636 objects misplaced (3.996%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:32.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[213841]: 7.5 scrub starts 2023-10-09T04:12:32.762 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 785/81636 objects degraded (0.962%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:32.762 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[221067]: from='client.55066 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:32.762 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[221067]: pgmap v584: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 785/81636 objects degraded (0.962%); 3262/81636 objects misplaced (3.996%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:32.762 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[221067]: 7.5 scrub starts 2023-10-09T04:12:32.762 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 785/81636 objects degraded (0.962%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:32 smithi181 ceph-mon[196979]: from='client.55066 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:32 smithi181 ceph-mon[196979]: pgmap v584: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 785/81636 objects degraded (0.962%); 3262/81636 objects misplaced (3.996%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:32 smithi181 ceph-mon[196979]: 7.5 scrub starts 2023-10-09T04:12:32.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 785/81636 objects degraded (0.962%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:33.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:33.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:33.181 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:33.226 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:33.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:33.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:33.493 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:33.585 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:33.704 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:12:33.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3639965445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:33.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:33 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:33.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3639965445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:33.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:33 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:33.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:33.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:34.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:34.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3639965445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:34.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:33 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:34.046 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:34.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:34.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:34.222 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:34.278 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:34.448 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:34.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:34.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:34.591 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:34.621 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:34.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:34.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:34.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:34.761 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:34.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: pgmap v585: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 785/78213 objects degraded (1.004%); 3262/78213 objects misplaced (4.171%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: failed to encode map e221 with expected crc 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: failed to encode map e221 with expected crc 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: failed to encode map e221 with expected crc 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:34.865 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:35.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:35.031 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:35.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: pgmap v585: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 785/78213 objects degraded (1.004%); 3262/78213 objects misplaced (4.171%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: failed to encode map e221 with expected crc 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: failed to encode map e221 with expected crc 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: failed to encode map e221 with expected crc 2023-10-09T04:12:35.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:35.044 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: pgmap v585: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 785/78213 objects degraded (1.004%); 3262/78213 objects misplaced (4.171%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: failed to encode map e221 with expected crc 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: failed to encode map e221 with expected crc 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: failed to encode map e221 with expected crc 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:35.086 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:12:35.086 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181790 6.12 6:483ee06b:::benchmark_data_smithi163_287034_object73527:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 265.87470370900002, 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2147281759999999, 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.087 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181790 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:12:35.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206414+0000", 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.855e-06 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.089 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206419+0000", 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7840000000000003e-06 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206423+0000", 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5449999999999997e-06 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207972+0000", 2023-10-09T04:12:35.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001549031 2023-10-09T04:12:35.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207978+0000", 2023-10-09T04:12:35.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9889999999999997e-06 2023-10-09T04:12:35.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388116+0000", 2023-10-09T04:12:35.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.180137601 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for rw locks", 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388262+0000", 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145642 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408226+0000", 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019964064 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408317+0000", 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1717000000000003e-05 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408461+0000", 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014404700000000001 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418786+0000", 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010325051 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420556+0000", 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017692529999999999 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420629+0000", 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2847000000000002e-05 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420693+0000", 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4324000000000003e-05 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.421141+0000", 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044771999999999998 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.097 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181893 6.12 6:48514e0f:::benchmark_data_smithi163_287034_object73630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 265.874616097, 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2117616529999999, 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181893 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206502+0000", 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0099999999999998e-06 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206513+0000", 2023-10-09T04:12:35.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206517+0000", 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5499999999999996e-06 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208079+0000", 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015617529999999999 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208087+0000", 2023-10-09T04:12:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5059999999999998e-06 2023-10-09T04:12:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395090+0000", 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1870032690000001 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395194+0000", 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010448 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395313+0000", 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011805 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400612+0000", 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052990609999999999 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407098+0000", 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064860760000000003 2023-10-09T04:12:35.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417540+0000", 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010442168999999999 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417604+0000", 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4577000000000002e-05 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418262+0000", 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00065729299999999998 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181924 6.12 6:4956f571:::benchmark_data_smithi163_287034_object73661:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 265.87457495899997, 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.212897227, 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181924 2023-10-09T04:12:35.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206543+0000", 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0640000000000001e-06 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206549+0000", 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6949999999999997e-06 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206554+0000", 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208118+0000", 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015642570000000001 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208123+0000", 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6370000000000004e-06 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397380+0000", 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1892568610000001 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397556+0000", 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017565199999999999 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397712+0000", 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015616 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400663+0000", 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002950664 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407160+0000", 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064973219999999998 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418821+0000", 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011661227 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418878+0000", 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6666999999999998e-05 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419438+0000", 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056037000000000003 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.115 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181928 6.12 6:4a3131f6:::benchmark_data_smithi163_287034_object73665:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 265.87454422899998, 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2134125929999999, 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181928 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.116 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206574+0000", 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0389999999999999e-06 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206579+0000", 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:12:35.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206585+0000", 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6330000000000001e-06 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208144+0000", 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015595439999999999 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208151+0000", 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400687+0000", 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.192535828 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400844+0000", 2023-10-09T04:12:35.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015750000000000001 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.401023+0000", 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017885000000000001 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407172+0000", 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006149374 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407223+0000", 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0392999999999997e-05 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419479+0000", 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01225652 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419533+0000", 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3735e-05 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419985+0000", 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045140000000000002 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.124 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181971 6.13 6:ce89c71c:::benchmark_data_smithi163_287034_object73708:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86483744600002, 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.344004741, 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.125 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181971 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:12:35.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216277+0000", 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216286+0000", 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7050000000000005e-06 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216564+0000", 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027831900000000002 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216577+0000", 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.29e-05 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217701+0000", 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001124002 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217710+0000", 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6999999999999997e-06 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.538610+0000", 2023-10-09T04:12:35.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3209002599999999 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.538722+0000", 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112537 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.539153+0000", 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000430362 2023-10-09T04:12:35.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542629+0000", 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034758459999999999 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.545992+0000", 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033635409999999998 2023-10-09T04:12:35.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552225+0000", 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062325189999999997 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.559513+0000", 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072881919999999998 2023-10-09T04:12:35.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.559580+0000", 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7296000000000001e-05 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560283+0000", 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070328800000000005 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.133 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181975 6.13 6:caf04be6:::benchmark_data_smithi163_287034_object73712:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86419106699998, 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3577086709999999, 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.134 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181975 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:12:35.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216924+0000", 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216930+0000", 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8719999999999999e-06 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216938+0000", 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8380000000000008e-06 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216944+0000", 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4310000000000002e-06 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217731+0000", 2023-10-09T04:12:35.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078680899999999997 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217736+0000", 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7459999999999997e-06 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542662+0000", 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3249255600000001 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542848+0000", 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000186174 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.543402+0000", 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000553489 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552236+0000", 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0088346299999999996 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552280+0000", 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3537e-05 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560349+0000", 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080691140000000005 2023-10-09T04:12:35.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.573767+0000", 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013417861 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.573826+0000", 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9379999999999997e-05 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574634+0000", 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080766300000000002 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181980 6.13 6:cdaa0fdf:::benchmark_data_smithi163_287034_object73717:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:12:35.142 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86416731899999, 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3655362260000001, 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181980 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216948+0000", 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216951+0000", 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6579999999999998e-06 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216958+0000", 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5350000000000001e-06 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216963+0000", 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3650000000000003e-06 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217756+0000", 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00079339899999999997 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217762+0000", 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6180000000000003e-06 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546040+0000", 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.328277887 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546236+0000", 2023-10-09T04:12:35.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019608900000000001 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546794+0000", 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055758299999999995 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552243+0000", 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054494230000000001 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552318+0000", 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5481000000000005e-05 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566793+0000", 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014474614 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581590+0000", 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014796659 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581645+0000", 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5427999999999999e-05 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582485+0000", 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083988900000000002 2023-10-09T04:12:35.150 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182015 6.13 6:cae1eea5:::benchmark_data_smithi163_287034_object73752:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86414879799997, 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3664138910000001, 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.151 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182015 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216970+0000", 2023-10-09T04:12:35.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5299999999999999e-06 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216975+0000", 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3689999999999998e-06 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216979+0000", 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5589999999999999e-06 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217782+0000", 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080258200000000003 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217788+0000", 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.502e-06 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.548975+0000", 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3311876169999999 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.549135+0000", 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015948399999999999 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.549566+0000", 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043101400000000002 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552286+0000", 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027206219999999998 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552369+0000", 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3139000000000005e-05 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566902+0000", 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014533001 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582622+0000", 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015720073000000001 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582685+0000", 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2966999999999995e-05 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583381+0000", 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000695814 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.159 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182032 6.13 6:cff9d57a:::benchmark_data_smithi163_287034_object73769:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86413297899998, 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3672618480000001, 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182032 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.160 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216985+0000", 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1789999999999998e-06 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216990+0000", 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3109999999999996e-06 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216995+0000", 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9280000000000001e-06 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217808+0000", 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00081240700000000002 2023-10-09T04:12:35.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217813+0000", 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4009999999999996e-06 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552391+0000", 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3345780999999999 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552538+0000", 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014700800000000001 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.553135+0000", 2023-10-09T04:12:35.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059664400000000002 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563747+0000", 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010612430000000001 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566840+0000", 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030923819999999999 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574748+0000", 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079079549999999995 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583498+0000", 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087501799999999998 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583544+0000", 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6165999999999998e-05 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584245+0000", 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070112899999999999 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182044 6.13 6:c8672a95:::benchmark_data_smithi163_287034_object73781:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86411713699999, 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.367953709, 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182044 2023-10-09T04:12:35.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217001+0000", 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9290000000000001e-06 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217007+0000", 2023-10-09T04:12:35.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1700000000000002e-06 2023-10-09T04:12:35.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217011+0000", 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3850000000000004e-06 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217834+0000", 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082231700000000003 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217839+0000", 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556028+0000", 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.338188543 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556217+0000", 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189452 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556804+0000", 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058719899999999995 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563759+0000", 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069549829999999997 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566968+0000", 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003208598 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582584+0000", 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015615655000000001 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584293+0000", 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017097309999999999 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584348+0000", 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4750999999999998e-05 2023-10-09T04:12:35.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584953+0000", 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060460600000000005 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182130 6.13 6:ce90ea3b:::benchmark_data_smithi163_287034_object73867:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:12:35.179 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86410112900001, 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.376683477, 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182130 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217017+0000", 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.464e-06 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217021+0000", 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6940000000000001e-06 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217024+0000", 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2760000000000001e-06 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217860+0000", 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083570200000000002 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217865+0000", 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5400000000000003e-06 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560446+0000", 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.342580544 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560636+0000", 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019010500000000001 2023-10-09T04:12:35.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.561233+0000", 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059681600000000001 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566854+0000", 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056207490000000004 2023-10-09T04:12:35.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567010+0000", 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015621499999999999 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583462+0000", 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016452205000000001 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.592834+0000", 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093721150000000003 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.592892+0000", 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7810999999999999e-05 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593699+0000", 2023-10-09T04:12:35.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080657999999999999 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182187 6.13 6:c86ec581:::benchmark_data_smithi163_287034_object73924:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86408973200002, 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3896110020000001, 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.188 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182187 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217028+0000", 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.906e-06 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217031+0000", 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.095e-06 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217034+0000", 2023-10-09T04:12:35.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6529999999999998e-06 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217885+0000", 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00085116200000000003 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217891+0000", 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4920000000000001e-06 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563798+0000", 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3459071169999999 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563972+0000", 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017474299999999999 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.564543+0000", 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00057071800000000001 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567022+0000", 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024785779999999999 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567079+0000", 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7258e-05 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597561+0000", 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.030482183999999999 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.605598+0000", 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080370560000000008 2023-10-09T04:12:35.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.605665+0000", 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7268999999999996e-05 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.606637+0000", 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097197400000000002 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.196 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182188 6.13 6:cef9c490:::benchmark_data_smithi163_287034_object73925:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86407913300002, 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4008471199999999, 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.197 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182188 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217039+0000", 2023-10-09T04:12:35.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8160000000000001e-06 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217044+0000", 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.169e-06 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217048+0000", 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1980000000000003e-06 2023-10-09T04:12:35.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217910+0000", 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086211000000000002 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217915+0000", 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2850000000000002e-06 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567119+0000", 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3492036760000001 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567314+0000", 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000195353 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567922+0000", 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060742399999999996 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574701+0000", 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067789490000000003 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.577940+0000", 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032392279999999998 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601664+0000", 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023723782999999998 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.616941+0000", 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015276853 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617000+0000", 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9185999999999999e-05 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617884+0000", 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088436199999999997 2023-10-09T04:12:35.205 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182235 6.13 6:cbfff063:::benchmark_data_smithi163_287034_object73972:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86406497299998, 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4014693499999999, 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.206 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182235 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217053+0000", 2023-10-09T04:12:35.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3099999999999999e-06 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217059+0000", 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3759999999999999e-06 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217063+0000", 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.318e-06 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217936+0000", 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00087284899999999998 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217941+0000", 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.643e-06 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570245+0000", 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3523035450000001 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570415+0000", 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016968699999999999 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570978+0000", 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056366499999999998 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.577951+0000", 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069725350000000002 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581550+0000", 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035993399999999999 2023-10-09T04:12:35.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601696+0000", 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020146045000000001 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617929+0000", 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016232449999999999 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617984+0000", 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5387999999999999e-05 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618521+0000", 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053653999999999996 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.214 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182259 6.13 6:ca05850d:::benchmark_data_smithi163_287034_object73996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.864049384, 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4019898450000001, 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182259 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:12:35.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217066+0000", 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217069+0000", 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.227e-06 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217074+0000", 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6829999999999996e-06 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217079+0000", 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2880000000000003e-06 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217961+0000", 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088278599999999999 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217967+0000", 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574785+0000", 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3568176510000001 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574950+0000", 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000164869 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.575563+0000", 2023-10-09T04:12:35.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061337800000000004 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582540+0000", 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069775280000000002 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593712+0000", 2023-10-09T04:12:35.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011171096 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.606689+0000", 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012977087 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618561+0000", 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011872297 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618604+0000", 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3266999999999998e-05 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619057+0000", 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045241999999999998 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.223 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182286 6.13 6:c82cebba:::benchmark_data_smithi163_287034_object74023:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86403380600001, 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4024858179999999, 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.224 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182286 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:12:35.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217089+0000", 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1029999999999996e-06 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217096+0000", 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4010000000000002e-06 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217101+0000", 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217987+0000", 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088635799999999996 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217993+0000", 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8619999999999999e-06 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.578004+0000", 2023-10-09T04:12:35.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.360011112 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.578616+0000", 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061207500000000001 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.579171+0000", 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055508900000000004 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583427+0000", 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.004255549 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593719+0000", 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010292066000000001 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.610148+0000", 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016429684 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619093+0000", 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0089444580000000006 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619134+0000", 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0794000000000003e-05 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619568+0000", 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043460500000000002 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182384 6.13 6:cc63ec28:::benchmark_data_smithi163_287034_object74121:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.864012092, 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4029806759999999, 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182384 2023-10-09T04:12:35.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217106+0000", 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1689999999999999e-06 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217113+0000", 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9739999999999996e-06 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217117+0000", 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4499999999999997e-06 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218013+0000", 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00089540900000000001 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218019+0000", 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584986+0000", 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3669674590000001 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.585163+0000", 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000176995 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.585767+0000", 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060421699999999999 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593726+0000", 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079584719999999994 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597514+0000", 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037886840000000001 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.614382+0000", 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016867727999999999 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619605+0000", 2023-10-09T04:12:35.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005223073 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619647+0000", 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2032e-05 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.620085+0000", 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043757399999999998 2023-10-09T04:12:35.241 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182402 6.13 6:cc13d46a:::benchmark_data_smithi163_287034_object74139:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86399526999998, 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.407380751, 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.242 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182402 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217123+0000", 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0609999999999998e-06 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217128+0000", 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5969999999999999e-06 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217133+0000", 2023-10-09T04:12:35.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.34e-06 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218039+0000", 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090607999999999997 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218044+0000", 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T04:12:35.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.589640+0000", 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3715955420000001 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.589766+0000", 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126234 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.590263+0000", 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000496529 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597468+0000", 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072057340000000001 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601593+0000", 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041247039999999999 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.620123+0000", 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018529468 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.623617+0000", 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034940879999999998 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.623677+0000", 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0062000000000002e-05 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624502+0000", 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000824952 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182403 6.13 6:ccb27798:::benchmark_data_smithi163_287034_object74140:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86397860400001, 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.407780327, 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.251 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182403 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217137+0000", 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217139+0000", 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2689999999999998e-06 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217145+0000", 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7980000000000002e-06 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217150+0000", 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4229999999999998e-06 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218064+0000", 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00091458200000000005 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218070+0000", 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4979999999999997e-06 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593753+0000", 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.375683465 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593914+0000", 2023-10-09T04:12:35.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016051499999999999 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.594483+0000", 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056889400000000002 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597523+0000", 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030404329999999999 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601631+0000", 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041084379999999998 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624529+0000", 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.022897438999999999 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624548+0000", 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8601999999999999e-05 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624576+0000", 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8637999999999999e-05 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624918+0000", 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00034171100000000002 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182407 6.13 6:cd9f07b1:::benchmark_data_smithi163_287034_object74144:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 263.86396269099998, 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4088424019999999, 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:12:35.260 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182407 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217155+0000", 2023-10-09T04:12:35.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2699999999999999e-06 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217161+0000", 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217166+0000", 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6439999999999999e-06 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218093+0000", 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092731699999999999 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218100+0000", 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7399999999999998e-06 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597985+0000", 2023-10-09T04:12:35.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.37988534 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.598147+0000", 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000162189 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.598729+0000", 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058185599999999995 2023-10-09T04:12:35.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601789+0000", 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003059712 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.603630+0000", 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001841051 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624943+0000", 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021312827999999999 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625489+0000", 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00054580099999999995 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625535+0000", 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6610000000000003e-05 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625996+0000", 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000460715 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:12:35.268 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:12:35.269 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:12:35.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:35.385 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:35.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:35.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:35.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:35.942 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:36.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:36.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:36.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:36.166 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:36.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:36.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:36.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:36.378 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:36.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:36.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:36 smithi181 ceph-mon[196979]: pgmap v587: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 785/74886 objects degraded (1.048%); 3262/74886 objects misplaced (4.356%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:36 smithi181 ceph-mon[196979]: 7.11 scrub starts 2023-10-09T04:12:36.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:36.620 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:36.622 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:12:36.698 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:36.698 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:36.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[213841]: pgmap v587: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 785/74886 objects degraded (1.048%); 3262/74886 objects misplaced (4.356%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:36.699 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[213841]: 7.11 scrub starts 2023-10-09T04:12:36.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:36.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:36.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[221067]: pgmap v587: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 785/74886 objects degraded (1.048%); 3262/74886 objects misplaced (4.356%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:12:36.699 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:36 smithi163 ceph-mon[221067]: 7.11 scrub starts 2023-10-09T04:12:37.403 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 785/74886 objects degraded (1.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:37.409 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 785/74886 objects degraded (1.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:37.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 785/74886 objects degraded (1.048%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:38.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:38.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:38.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:38.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:12:38.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:38.276 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:38.491 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:38.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:38.576 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:38.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:38.632 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:38.684 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:38.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:38 smithi181 ceph-mon[196979]: pgmap v588: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 785/71925 objects degraded (1.091%); 2708/71925 objects misplaced (3.765%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:38.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:38 smithi181 ceph-mon[196979]: 7.5 deep-scrub starts 2023-10-09T04:12:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:38.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:38.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:12:38.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:38.790 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:38 smithi163 ceph-mon[221067]: pgmap v588: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 785/71925 objects degraded (1.091%); 2708/71925 objects misplaced (3.765%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:38.790 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:38 smithi163 ceph-mon[221067]: 7.5 deep-scrub starts 2023-10-09T04:12:38.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:38 smithi163 ceph-mon[213841]: pgmap v588: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 785/71925 objects degraded (1.091%); 2708/71925 objects misplaced (3.765%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:38.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:38 smithi163 ceph-mon[213841]: 7.5 deep-scrub starts 2023-10-09T04:12:38.816 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:38.828 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:12:38.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:38.870 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:12:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:38.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:38.920 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:12:38.965 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:12:38.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:39.018 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:12:39.059 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:12:39.112 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:39.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.119 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.120 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:12:39.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:39.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:39.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:39.389 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:39.422 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3229198344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:39.423 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:39 smithi163 ceph-mon[213841]: 7.11 scrub starts 2023-10-09T04:12:39.423 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3229198344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:39.423 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:39 smithi163 ceph-mon[221067]: 7.11 scrub starts 2023-10-09T04:12:39.438 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:39.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:39.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:39.576 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:39.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:39.645 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3229198344' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:39.646 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:39 smithi181 ceph-mon[196979]: 7.11 scrub starts 2023-10-09T04:12:39.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:39.659 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:39.659 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 785/71925 objects degraded (1.091%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":724,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":221,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":8,"osd_in_since":1696823548,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":23975,"data_bytes":1570183096,"bytes_used":7827701760,"bytes_avail":760132583424,"bytes_total":767960285184,"degraded_objects":785,"degraded_total":71925,"degraded_ratio":0.010914146680570037,"misplaced_objects":2708,"misplaced_total":71925,"misplaced_ratio":0.037650330205074731,"recovering_objects_per_sec":47,"recovering_bytes_per_sec":3117869,"recovering_keys_per_sec":0,"num_objects_recovered":476,"num_bytes_recovered":31195136,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":678},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":21,"modified":"2023-10-09T04:04:35.266858+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"61f8f0b7-6515-43d1-9684-78e0e9193886":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 6s)","progress":0.95959597826004028,"add_to_ceph_s":true}}} 2023-10-09T04:12:39.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:39.832 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:40 smithi181 ceph-mon[196979]: pgmap v589: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 785/66309 objects degraded (1.184%); 2708/66309 objects misplaced (4.084%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2811087726' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:12:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:40 smithi163 ceph-mon[221067]: pgmap v589: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 785/66309 objects degraded (1.184%); 2708/66309 objects misplaced (4.084%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:40.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2811087726' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:12:40.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:40 smithi163 ceph-mon[213841]: pgmap v589: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 785/66309 objects degraded (1.184%); 2708/66309 objects misplaced (4.084%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:12:40.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2811087726' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:12:40.834 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:12:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:12:40] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-10-09T04:12:40.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:40.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:40.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:40.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:41.017 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:41.062 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:41.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:41.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:41.203 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:41.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:41.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:41.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:41.454 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:41.500 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:41.544 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:12:41.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:12:41.598 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:41.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:41.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:41.744 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:41.807 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:41.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:41.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:42.019 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:42.195 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:42.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:42.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:42.441 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:42.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:42 smithi163 ceph-mon[213841]: pgmap v590: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 785/61233 objects degraded (1.282%); 2150/61233 objects misplaced (3.511%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:42.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 785/61233 objects degraded (1.282%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:42.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:42.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:42.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:42.752 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:42 smithi163 ceph-mon[221067]: pgmap v590: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 785/61233 objects degraded (1.282%); 2150/61233 objects misplaced (3.511%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 785/61233 objects degraded (1.282%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:42.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:12:43.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:42 smithi181 ceph-mon[196979]: pgmap v590: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 785/61233 objects degraded (1.282%); 2150/61233 objects misplaced (3.511%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 785/61233 objects degraded (1.282%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:43.387 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:43.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:43.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:43.446 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:43.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:43.532 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:43.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:43.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:43.640 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:12:43.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1477991062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:12:43.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1477991062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:12:43.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:43.733 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:43.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:43.794 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1477991062' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:12:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:43.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:43.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:43.989 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:44.199 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:44.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:44.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:44.340 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:44.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:44.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:44.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:44.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:44.543 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:44.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[213841]: pgmap v591: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 785/58125 objects degraded (1.351%); 2150/58125 objects misplaced (3.699%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:44.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:44.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1477991062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:12:44.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[213841]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T04:12:44.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[213841]: failed to encode map e222 with expected crc 2023-10-09T04:12:44.610 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:44.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:44 smithi181 ceph-mon[196979]: pgmap v591: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 785/58125 objects degraded (1.351%); 2150/58125 objects misplaced (3.699%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:44.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:44 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:44.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1477991062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:12:44.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:44 smithi181 ceph-mon[196979]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T04:12:44.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:44 smithi181 ceph-mon[196979]: failed to encode map e222 with expected crc 2023-10-09T04:12:44.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:44.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:44.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:44.749 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:44.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:44.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:44.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[221067]: pgmap v591: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 785/58125 objects degraded (1.351%); 2150/58125 objects misplaced (3.699%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:44.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:44.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1477991062' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:12:44.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[221067]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T04:12:44.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:44 smithi163 ceph-mon[221067]: failed to encode map e222 with expected crc 2023-10-09T04:12:44.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:44.955 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:45.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:45.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:45.094 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:12:45.095 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:12:45.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:45.230 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:45.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:45 smithi163 ceph-mon[221067]: failed to encode map e222 with expected crc 2023-10-09T04:12:45.940 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:45 smithi163 ceph-mon[213841]: failed to encode map e222 with expected crc 2023-10-09T04:12:46.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:46.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:46.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:46.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:45 smithi181 ceph-mon[196979]: failed to encode map e222 with expected crc 2023-10-09T04:12:46.022 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:46.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:46.142 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:46.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:46.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:46.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:46.334 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:46.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:46.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:46.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:46.545 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:46.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:46.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:46.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:46.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:12:46.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:46.731 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:46 smithi181 ceph-mon[196979]: pgmap v593: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 785/54762 objects degraded (1.433%); 2150/54762 objects misplaced (3.926%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:46.741 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:46.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:46.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:46.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:46.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:46 smithi163 ceph-mon[221067]: pgmap v593: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 785/54762 objects degraded (1.433%); 2150/54762 objects misplaced (3.926%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:46.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:46 smithi163 ceph-mon[213841]: pgmap v593: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 785/54762 objects degraded (1.433%); 2150/54762 objects misplaced (3.926%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:12:46.967 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:47.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:47.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:47.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:47.222 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:47.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:12:48.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:48.267 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:48.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:48.315 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:48.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:48.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:48.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 785/54762 objects degraded (1.433%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:48.317 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[213841]: pgmap v594: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.0 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 785/51363 objects degraded (1.528%); 1519/51363 objects misplaced (2.957%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:12:48.345 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:12:48.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:48.462 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 785/54762 objects degraded (1.433%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:48 smithi163 ceph-mon[221067]: pgmap v594: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.0 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 785/51363 objects degraded (1.528%); 1519/51363 objects misplaced (2.957%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:12:48.656 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:48.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:48.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:48 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 785/54762 objects degraded (1.433%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:48.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:48 smithi181 ceph-mon[196979]: pgmap v594: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.0 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 785/51363 objects degraded (1.528%); 1519/51363 objects misplaced (2.957%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:12:48.794 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:48.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:48.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:48.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:49.027 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:49.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:49.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:49.281 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":594,"stamp":"2023-10-09T04:12:47.503622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1120999352,"num_objects":17121,"num_object_clones":0,"num_object_copies":51363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":1519,"num_objects_unfound":0,"num_objects_dirty":17121,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":163300,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13297,"num_bytes_recovered":872549044,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177650,"ondisk_log_size":177650,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6330484,"kb_used_data":3856244,"kb_used_omap":126,"kb_used_meta":2474049,"kb_avail":743630732,"statfs":{"total":767960285184,"available":761477869568,"internally_reserved":0,"allocated":3948793856,"data_stored":3942567777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2533426811},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-449183744,"num_objects":-6854,"num_object_clones":0,"num_object_copies":-20562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1189,"num_objects_unfound":0,"num_objects_dirty":-6854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6854,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":527,"num_bytes_recovered":34537472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009097"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":219,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685736+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:12:33.685736+0000","last_peered":"2023-10-09T04:12:33.685736+0000","last_clean":"2023-10-09T04:12:33.685736+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:12:33.685736+0000","last_undegraded":"2023-10-09T04:12:33.685736+0000","last_fullsized":"2023-10-09T04:12:33.685736+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698004+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:12:33.698004+0000","last_peered":"2023-10-09T04:12:33.698004+0000","last_clean":"2023-10-09T04:12:33.698004+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:12:33.698004+0000","last_undegraded":"2023-10-09T04:12:33.698004+0000","last_fullsized":"2023-10-09T04:12:33.698004+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698616+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:12:33.698616+0000","last_peered":"2023-10-09T04:12:33.698616+0000","last_clean":"2023-10-09T04:12:33.698616+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:33.698616+0000","last_undegraded":"2023-10-09T04:12:33.698616+0000","last_fullsized":"2023-10-09T04:12:33.698616+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":177,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.580185+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:12:24.580185+0000","last_peered":"2023-10-09T04:12:24.580185+0000","last_clean":"2023-10-09T04:12:24.580185+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:12:24.580185+0000","last_undegraded":"2023-10-09T04:12:24.580185+0000","last_fullsized":"2023-10-09T04:12:24.580185+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":175,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579447+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:12:24.579447+0000","last_peered":"2023-10-09T04:12:24.579447+0000","last_clean":"2023-10-09T04:12:24.579447+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:12:24.579447+0000","last_undegraded":"2023-10-09T04:12:24.579447+0000","last_fullsized":"2023-10-09T04:12:24.579447+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":155,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688245+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:12:33.688245+0000","last_peered":"2023-10-09T04:12:33.688245+0000","last_clean":"2023-10-09T04:12:33.688245+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:12:33.688245+0000","last_undegraded":"2023-10-09T04:12:33.688245+0000","last_fullsized":"2023-10-09T04:12:33.688245+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"222'6457","reported_seq":9419,"reported_epoch":222,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:45.812138+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:12:45.812138+0000","last_peered":"2023-10-09T04:12:45.812138+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:12:45.812138+0000","last_undegraded":"2023-10-09T04:12:45.812138+0000","last_fullsized":"2023-10-09T04:12:45.812138+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899072,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":502,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5286,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":502}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":176,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.580066+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:12:24.580066+0000","last_peered":"2023-10-09T04:12:24.580066+0000","last_clean":"2023-10-09T04:12:24.580066+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:12:24.580066+0000","last_undegraded":"2023-10-09T04:12:24.580066+0000","last_fullsized":"2023-10-09T04:12:24.580066+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":200,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.677626+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:12:33.677626+0000","last_peered":"2023-10-09T04:12:33.677626+0000","last_clean":"2023-10-09T04:12:33.677626+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:12:33.677626+0000","last_undegraded":"2023-10-09T04:12:33.677626+0000","last_fullsized":"2023-10-09T04:12:33.677626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"221'6526","reported_seq":13926,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:42.830740+0000","last_change":"2023-10-09T04:12:19.342281+0000","last_active":"2023-10-09T04:12:42.830740+0000","last_peered":"2023-10-09T04:12:42.830740+0000","last_clean":"2023-10-09T04:12:42.830740+0000","last_became_active":"2023-10-09T04:12:19.328982+0000","last_became_peered":"2023-10-09T04:12:19.328982+0000","last_unstale":"2023-10-09T04:12:42.830740+0000","last_undegraded":"2023-10-09T04:12:42.830740+0000","last_fullsized":"2023-10-09T04:12:42.830740+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6526,"log_dups_size":0,"ondisk_log_size":6526,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38862848,"num_objects":593,"num_object_clones":0,"num_object_copies":1779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5361,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2553,"num_bytes_recovered":167313408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":181,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688196+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:12:33.688196+0000","last_peered":"2023-10-09T04:12:33.688196+0000","last_clean":"2023-10-09T04:12:33.688196+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:12:33.688196+0000","last_undegraded":"2023-10-09T04:12:33.688196+0000","last_fullsized":"2023-10-09T04:12:33.688196+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678078+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:12:33.678078+0000","last_peered":"2023-10-09T04:12:33.678078+0000","last_clean":"2023-10-09T04:12:33.678078+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:12:33.678078+0000","last_undegraded":"2023-10-09T04:12:33.678078+0000","last_fullsized":"2023-10-09T04:12:33.678078+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"222'6413","reported_seq":9320,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.531656+0000","last_change":"2023-10-09T04:12:38.956478+0000","last_active":"2023-10-09T04:12:45.531656+0000","last_peered":"2023-10-09T04:12:45.531656+0000","last_clean":"2023-10-09T04:12:45.531656+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:12:45.531656+0000","last_undegraded":"2023-10-09T04:12:45.531656+0000","last_fullsized":"2023-10-09T04:12:45.531656+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6413,"log_dups_size":0,"ondisk_log_size":6413,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899072,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5278,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":184,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678938+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:12:33.678938+0000","last_peered":"2023-10-09T04:12:33.678938+0000","last_clean":"2023-10-09T04:12:33.678938+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:12:33.678938+0000","last_undegraded":"2023-10-09T04:12:33.678938+0000","last_fullsized":"2023-10-09T04:12:33.678938+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":174,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697408+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:12:33.697408+0000","last_peered":"2023-10-09T04:12:33.697408+0000","last_clean":"2023-10-09T04:12:33.697408+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:12:33.697408+0000","last_undegraded":"2023-10-09T04:12:33.697408+0000","last_fullsized":"2023-10-09T04:12:33.697408+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"222'6565","reported_seq":9509,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.008905+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:12:45.008905+0000","last_peered":"2023-10-09T04:12:45.008905+0000","last_clean":"2023-10-09T04:12:45.008905+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:12:45.008905+0000","last_undegraded":"2023-10-09T04:12:45.008905+0000","last_fullsized":"2023-10-09T04:12:45.008905+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34078720,"num_objects":520,"num_object_clones":0,"num_object_copies":1560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5376,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":186,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685898+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:12:33.685898+0000","last_peered":"2023-10-09T04:12:33.685898+0000","last_clean":"2023-10-09T04:12:33.685898+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:12:33.685898+0000","last_undegraded":"2023-10-09T04:12:33.685898+0000","last_fullsized":"2023-10-09T04:12:33.685898+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685847+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:12:33.685847+0000","last_peered":"2023-10-09T04:12:33.685847+0000","last_clean":"2023-10-09T04:12:33.685847+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:12:33.685847+0000","last_undegraded":"2023-10-09T04:12:33.685847+0000","last_fullsized":"2023-10-09T04:12:33.685847+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"221'6423","reported_seq":6497,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.110625+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:12:43.110625+0000","last_peered":"2023-10-09T04:12:43.110625+0000","last_clean":"2023-10-09T04:12:43.110625+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:12:43.110625+0000","last_undegraded":"2023-10-09T04:12:43.110625+0000","last_fullsized":"2023-10-09T04:12:43.110625+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5269,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":183,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678297+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:12:33.678297+0000","last_peered":"2023-10-09T04:12:33.678297+0000","last_clean":"2023-10-09T04:12:33.678297+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:12:33.678297+0000","last_undegraded":"2023-10-09T04:12:33.678297+0000","last_fullsized":"2023-10-09T04:12:33.678297+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":158,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698182+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:12:33.698182+0000","last_peered":"2023-10-09T04:12:33.698182+0000","last_clean":"2023-10-09T04:12:33.698182+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:12:33.698182+0000","last_undegraded":"2023-10-09T04:12:33.698182+0000","last_fullsized":"2023-10-09T04:12:33.698182+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"221'6190","reported_seq":6263,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:42.563768+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:12:42.563768+0000","last_peered":"2023-10-09T04:12:42.563768+0000","last_clean":"2023-10-09T04:12:42.563768+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:12:42.563768+0000","last_undegraded":"2023-10-09T04:12:42.563768+0000","last_fullsized":"2023-10-09T04:12:42.563768+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6190,"log_dups_size":0,"ondisk_log_size":6190,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37519360,"num_objects":573,"num_object_clones":0,"num_object_copies":1719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5073,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579672+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:12:24.579672+0000","last_peered":"2023-10-09T04:12:24.579672+0000","last_clean":"2023-10-09T04:12:24.579672+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:12:24.579672+0000","last_undegraded":"2023-10-09T04:12:24.579672+0000","last_fullsized":"2023-10-09T04:12:24.579672+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":180,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688135+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:12:33.688135+0000","last_peered":"2023-10-09T04:12:33.688135+0000","last_clean":"2023-10-09T04:12:33.688135+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:12:33.688135+0000","last_undegraded":"2023-10-09T04:12:33.688135+0000","last_fullsized":"2023-10-09T04:12:33.688135+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"222'6505","reported_seq":9456,"reported_epoch":222,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:45.575026+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:12:45.575026+0000","last_peered":"2023-10-09T04:12:45.575026+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:12:45.575026+0000","last_undegraded":"2023-10-09T04:12:45.575026+0000","last_fullsized":"2023-10-09T04:12:45.575026+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6505,"log_dups_size":0,"ondisk_log_size":6505,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33652736,"num_objects":514,"num_object_clones":0,"num_object_copies":1542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":514,"num_objects_unfound":0,"num_objects_dirty":514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5390,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":514}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":164,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678063+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:12:33.678063+0000","last_peered":"2023-10-09T04:12:33.678063+0000","last_clean":"2023-10-09T04:12:33.678063+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:12:33.678063+0000","last_undegraded":"2023-10-09T04:12:33.678063+0000","last_fullsized":"2023-10-09T04:12:33.678063+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.580226+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:12:24.580226+0000","last_peered":"2023-10-09T04:12:24.580226+0000","last_clean":"2023-10-09T04:12:24.580226+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:12:24.580226+0000","last_undegraded":"2023-10-09T04:12:24.580226+0000","last_fullsized":"2023-10-09T04:12:24.580226+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"221'6460","reported_seq":6534,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.081461+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:12:43.081461+0000","last_peered":"2023-10-09T04:12:43.081461+0000","last_clean":"2023-10-09T04:12:43.081461+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:12:43.081461+0000","last_undegraded":"2023-10-09T04:12:43.081461+0000","last_fullsized":"2023-10-09T04:12:43.081461+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6460,"log_dups_size":0,"ondisk_log_size":6460,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39976960,"num_objects":610,"num_object_clones":0,"num_object_copies":1830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5298,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":112,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579272+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:12:24.579272+0000","last_peered":"2023-10-09T04:12:24.579272+0000","last_clean":"2023-10-09T04:12:24.579272+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:12:24.579272+0000","last_undegraded":"2023-10-09T04:12:24.579272+0000","last_fullsized":"2023-10-09T04:12:24.579272+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":186,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.679287+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:12:33.679287+0000","last_peered":"2023-10-09T04:12:33.679287+0000","last_clean":"2023-10-09T04:12:33.679287+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:12:33.679287+0000","last_undegraded":"2023-10-09T04:12:33.679287+0000","last_fullsized":"2023-10-09T04:12:33.679287+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"222'10569","reported_seq":10634,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.636620+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:12:43.636620+0000","last_peered":"2023-10-09T04:12:43.636620+0000","last_clean":"2023-10-09T04:12:43.636620+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:12:43.636620+0000","last_undegraded":"2023-10-09T04:12:43.636620+0000","last_fullsized":"2023-10-09T04:12:43.636620+0000","mapping_epoch":167,"log_start":"188'3500","ondisk_log_start":"188'3500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72548352,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10569,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":182,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.679175+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:12:33.679175+0000","last_peered":"2023-10-09T04:12:33.679175+0000","last_clean":"2023-10-09T04:12:33.679175+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:12:33.679175+0000","last_undegraded":"2023-10-09T04:12:33.679175+0000","last_fullsized":"2023-10-09T04:12:33.679175+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.687846+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:12:33.687846+0000","last_peered":"2023-10-09T04:12:33.687846+0000","last_clean":"2023-10-09T04:12:33.687846+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:12:33.687846+0000","last_undegraded":"2023-10-09T04:12:33.687846+0000","last_fullsized":"2023-10-09T04:12:33.687846+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"222'10536","reported_seq":10601,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.059827+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:12:45.059827+0000","last_peered":"2023-10-09T04:12:45.059827+0000","last_clean":"2023-10-09T04:12:45.059827+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:12:45.059827+0000","last_undegraded":"2023-10-09T04:12:45.059827+0000","last_fullsized":"2023-10-09T04:12:45.059827+0000","mapping_epoch":167,"log_start":"171'500","ondisk_log_start":"171'500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67239936,"num_objects":1026,"num_object_clones":0,"num_object_copies":3078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10536,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":183,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.681239+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:12:33.681239+0000","last_peered":"2023-10-09T04:12:33.681239+0000","last_clean":"2023-10-09T04:12:33.681239+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:12:33.681239+0000","last_undegraded":"2023-10-09T04:12:33.681239+0000","last_fullsized":"2023-10-09T04:12:33.681239+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":197,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.686958+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:12:33.686958+0000","last_peered":"2023-10-09T04:12:33.686958+0000","last_clean":"2023-10-09T04:12:33.686958+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:12:33.686958+0000","last_undegraded":"2023-10-09T04:12:33.686958+0000","last_fullsized":"2023-10-09T04:12:33.686958+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"221'6526","reported_seq":6707,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.107017+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:12:43.107017+0000","last_peered":"2023-10-09T04:12:43.107017+0000","last_clean":"2023-10-09T04:12:43.107017+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:12:43.107017+0000","last_undegraded":"2023-10-09T04:12:43.107017+0000","last_fullsized":"2023-10-09T04:12:43.107017+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":6526,"log_dups_size":0,"ondisk_log_size":6526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34865152,"num_objects":532,"num_object_clones":0,"num_object_copies":1596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4140,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":165,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678852+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:12:33.678852+0000","last_peered":"2023-10-09T04:12:33.678852+0000","last_clean":"2023-10-09T04:12:33.678852+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:12:33.678852+0000","last_undegraded":"2023-10-09T04:12:33.678852+0000","last_fullsized":"2023-10-09T04:12:33.678852+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":141,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697095+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:12:33.697095+0000","last_peered":"2023-10-09T04:12:33.697095+0000","last_clean":"2023-10-09T04:12:33.697095+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:12:33.697095+0000","last_undegraded":"2023-10-09T04:12:33.697095+0000","last_fullsized":"2023-10-09T04:12:33.697095+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"222'6320","reported_seq":6399,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.030333+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:12:45.030333+0000","last_peered":"2023-10-09T04:12:45.030333+0000","last_clean":"2023-10-09T04:12:45.030333+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:12:45.030333+0000","last_undegraded":"2023-10-09T04:12:45.030333+0000","last_fullsized":"2023-10-09T04:12:45.030333+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6320,"log_dups_size":0,"ondisk_log_size":6320,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36175872,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5180,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688103+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:12:33.688103+0000","last_peered":"2023-10-09T04:12:33.688103+0000","last_clean":"2023-10-09T04:12:33.688103+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:12:33.688103+0000","last_undegraded":"2023-10-09T04:12:33.688103+0000","last_fullsized":"2023-10-09T04:12:33.688103+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":176,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579303+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:12:24.579303+0000","last_peered":"2023-10-09T04:12:24.579303+0000","last_clean":"2023-10-09T04:12:24.579303+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:12:24.579303+0000","last_undegraded":"2023-10-09T04:12:24.579303+0000","last_fullsized":"2023-10-09T04:12:24.579303+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"221'10575","reported_seq":10639,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.079195+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:12:43.079195+0000","last_peered":"2023-10-09T04:12:43.079195+0000","last_clean":"2023-10-09T04:12:43.079195+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:12:43.079195+0000","last_undegraded":"2023-10-09T04:12:43.079195+0000","last_fullsized":"2023-10-09T04:12:43.079195+0000","mapping_epoch":167,"log_start":"183'3000","ondisk_log_start":"183'3000","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71237632,"num_objects":1087,"num_object_clones":0,"num_object_copies":3261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10575,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":125,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579227+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:12:24.579227+0000","last_peered":"2023-10-09T04:12:24.579227+0000","last_clean":"2023-10-09T04:12:24.579227+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:12:24.579227+0000","last_undegraded":"2023-10-09T04:12:24.579227+0000","last_fullsized":"2023-10-09T04:12:24.579227+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":180,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685843+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:12:33.685843+0000","last_peered":"2023-10-09T04:12:33.685843+0000","last_clean":"2023-10-09T04:12:33.685843+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:12:33.685843+0000","last_undegraded":"2023-10-09T04:12:33.685843+0000","last_fullsized":"2023-10-09T04:12:33.685843+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"222'10591","reported_seq":13495,"reported_epoch":222,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:12:45.617813+0000","last_change":"2023-10-09T04:12:18.137975+0000","last_active":"2023-10-09T04:12:45.617813+0000","last_peered":"2023-10-09T04:12:45.617813+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:12:45.617813+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"171'500","ondisk_log_start":"171'500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":10091,"log_dups_size":0,"ondisk_log_size":10091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67305472,"num_objects":1027,"num_object_clones":0,"num_object_copies":3081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10591,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1412,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":242}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":141,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685831+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:12:33.685831+0000","last_peered":"2023-10-09T04:12:33.685831+0000","last_clean":"2023-10-09T04:12:33.685831+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:12:33.685831+0000","last_undegraded":"2023-10-09T04:12:33.685831+0000","last_fullsized":"2023-10-09T04:12:33.685831+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":217,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685944+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:12:33.685944+0000","last_peered":"2023-10-09T04:12:33.685944+0000","last_clean":"2023-10-09T04:12:33.685944+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:12:33.685944+0000","last_undegraded":"2023-10-09T04:12:33.685944+0000","last_fullsized":"2023-10-09T04:12:33.685944+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"222'10494","reported_seq":10560,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.637442+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:12:43.637442+0000","last_peered":"2023-10-09T04:12:43.637442+0000","last_clean":"2023-10-09T04:12:43.637442+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:12:43.637442+0000","last_undegraded":"2023-10-09T04:12:43.637442+0000","last_fullsized":"2023-10-09T04:12:43.637442+0000","mapping_epoch":167,"log_start":"188'3500","ondisk_log_start":"188'3500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728024,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10494,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":181,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.686724+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:12:33.686724+0000","last_peered":"2023-10-09T04:12:33.686724+0000","last_clean":"2023-10-09T04:12:33.686724+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:12:33.686724+0000","last_undegraded":"2023-10-09T04:12:33.686724+0000","last_fullsized":"2023-10-09T04:12:33.686724+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.686057+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:12:33.686057+0000","last_peered":"2023-10-09T04:12:33.686057+0000","last_clean":"2023-10-09T04:12:33.686057+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:12:33.686057+0000","last_undegraded":"2023-10-09T04:12:33.686057+0000","last_fullsized":"2023-10-09T04:12:33.686057+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"222'10434","reported_seq":10499,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.690568+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:12:45.690568+0000","last_peered":"2023-10-09T04:12:45.690568+0000","last_clean":"2023-10-09T04:12:45.690568+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:12:45.690568+0000","last_undegraded":"2023-10-09T04:12:45.690568+0000","last_fullsized":"2023-10-09T04:12:45.690568+0000","mapping_epoch":167,"log_start":"193'4000","ondisk_log_start":"193'4000","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65273856,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10434,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":188,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.677453+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:12:33.677453+0000","last_peered":"2023-10-09T04:12:33.677453+0000","last_clean":"2023-10-09T04:12:33.677453+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:12:33.677453+0000","last_undegraded":"2023-10-09T04:12:33.677453+0000","last_fullsized":"2023-10-09T04:12:33.677453+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.696746+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:12:33.696746+0000","last_peered":"2023-10-09T04:12:33.696746+0000","last_clean":"2023-10-09T04:12:33.696746+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:12:33.696746+0000","last_undegraded":"2023-10-09T04:12:33.696746+0000","last_fullsized":"2023-10-09T04:12:33.696746+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"222'6461","reported_seq":6658,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.293694+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:12:45.293694+0000","last_peered":"2023-10-09T04:12:45.293694+0000","last_clean":"2023-10-09T04:12:45.293694+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:12:45.293694+0000","last_undegraded":"2023-10-09T04:12:45.293694+0000","last_fullsized":"2023-10-09T04:12:45.293694+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2808,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":197,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698307+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:12:33.698307+0000","last_peered":"2023-10-09T04:12:33.698307+0000","last_clean":"2023-10-09T04:12:33.698307+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:12:33.698307+0000","last_undegraded":"2023-10-09T04:12:33.698307+0000","last_fullsized":"2023-10-09T04:12:33.698307+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":198,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688217+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:12:33.688217+0000","last_peered":"2023-10-09T04:12:33.688217+0000","last_clean":"2023-10-09T04:12:33.688217+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:33.688217+0000","last_undegraded":"2023-10-09T04:12:33.688217+0000","last_fullsized":"2023-10-09T04:12:33.688217+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":159,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697265+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:12:33.697265+0000","last_peered":"2023-10-09T04:12:33.697265+0000","last_clean":"2023-10-09T04:12:33.697265+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:12:33.697265+0000","last_undegraded":"2023-10-09T04:12:33.697265+0000","last_fullsized":"2023-10-09T04:12:33.697265+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"221'6485","reported_seq":6559,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:42.656793+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:12:42.656793+0000","last_peered":"2023-10-09T04:12:42.656793+0000","last_clean":"2023-10-09T04:12:42.656793+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:12:42.656793+0000","last_undegraded":"2023-10-09T04:12:42.656793+0000","last_fullsized":"2023-10-09T04:12:42.656793+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6485,"log_dups_size":0,"ondisk_log_size":6485,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33751040,"num_objects":515,"num_object_clones":0,"num_object_copies":1545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5331,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":185,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.687980+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:12:33.687980+0000","last_peered":"2023-10-09T04:12:33.687980+0000","last_clean":"2023-10-09T04:12:33.687980+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:12:33.687980+0000","last_undegraded":"2023-10-09T04:12:33.687980+0000","last_fullsized":"2023-10-09T04:12:33.687980+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":181,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685846+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:12:33.685846+0000","last_peered":"2023-10-09T04:12:33.685846+0000","last_clean":"2023-10-09T04:12:33.685846+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:12:33.685846+0000","last_undegraded":"2023-10-09T04:12:33.685846+0000","last_fullsized":"2023-10-09T04:12:33.685846+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698245+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:12:33.698245+0000","last_peered":"2023-10-09T04:12:33.698245+0000","last_clean":"2023-10-09T04:12:33.698245+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:12:33.698245+0000","last_undegraded":"2023-10-09T04:12:33.698245+0000","last_fullsized":"2023-10-09T04:12:33.698245+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"221'6500","reported_seq":6574,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:42.819571+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:12:42.819571+0000","last_peered":"2023-10-09T04:12:42.819571+0000","last_clean":"2023-10-09T04:12:42.819571+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:12:42.819571+0000","last_undegraded":"2023-10-09T04:12:42.819571+0000","last_fullsized":"2023-10-09T04:12:42.819571+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6500,"log_dups_size":0,"ondisk_log_size":6500,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35782656,"num_objects":546,"num_object_clones":0,"num_object_copies":1638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5338,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.615391+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:12:43.615391+0000","last_peered":"2023-10-09T04:12:43.615391+0000","last_clean":"2023-10-09T04:12:43.615391+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:12:43.615391+0000","last_undegraded":"2023-10-09T04:12:43.615391+0000","last_fullsized":"2023-10-09T04:12:43.615391+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":192,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697488+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:12:33.697488+0000","last_peered":"2023-10-09T04:12:33.697488+0000","last_clean":"2023-10-09T04:12:33.697488+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:12:33.697488+0000","last_undegraded":"2023-10-09T04:12:33.697488+0000","last_fullsized":"2023-10-09T04:12:33.697488+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.696814+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:12:33.696814+0000","last_peered":"2023-10-09T04:12:33.696814+0000","last_clean":"2023-10-09T04:12:33.696814+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:12:33.696814+0000","last_undegraded":"2023-10-09T04:12:33.696814+0000","last_fullsized":"2023-10-09T04:12:33.696814+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"221'6296","reported_seq":6369,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.060770+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:12:43.060770+0000","last_peered":"2023-10-09T04:12:43.060770+0000","last_clean":"2023-10-09T04:12:43.060770+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:12:43.060770+0000","last_undegraded":"2023-10-09T04:12:43.060770+0000","last_fullsized":"2023-10-09T04:12:43.060770+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36208640,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5178,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":205,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.687988+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:12:33.687988+0000","last_peered":"2023-10-09T04:12:33.687988+0000","last_clean":"2023-10-09T04:12:33.687988+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:12:33.687988+0000","last_undegraded":"2023-10-09T04:12:33.687988+0000","last_fullsized":"2023-10-09T04:12:33.687988+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":562,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.677910+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:12:33.677910+0000","last_peered":"2023-10-09T04:12:33.677910+0000","last_clean":"2023-10-09T04:12:33.677910+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:12:33.677910+0000","last_undegraded":"2023-10-09T04:12:33.677910+0000","last_fullsized":"2023-10-09T04:12:33.677910+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":118,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688015+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:12:33.688015+0000","last_peered":"2023-10-09T04:12:33.688015+0000","last_clean":"2023-10-09T04:12:33.688015+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:12:33.688015+0000","last_undegraded":"2023-10-09T04:12:33.688015+0000","last_fullsized":"2023-10-09T04:12:33.688015+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.686093+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:12:33.686093+0000","last_peered":"2023-10-09T04:12:33.686093+0000","last_clean":"2023-10-09T04:12:33.686093+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:12:33.686093+0000","last_undegraded":"2023-10-09T04:12:33.686093+0000","last_fullsized":"2023-10-09T04:12:33.686093+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"222'6436","reported_seq":8804,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.527507+0000","last_change":"2023-10-09T04:12:37.913403+0000","last_active":"2023-10-09T04:12:45.527507+0000","last_peered":"2023-10-09T04:12:45.527507+0000","last_clean":"2023-10-09T04:12:45.527507+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:12:45.527507+0000","last_undegraded":"2023-10-09T04:12:45.527507+0000","last_fullsized":"2023-10-09T04:12:45.527507+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6436,"log_dups_size":0,"ondisk_log_size":6436,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29786112,"num_objects":454,"num_object_clones":0,"num_object_copies":1362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5320,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":202,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.677856+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:12:33.677856+0000","last_peered":"2023-10-09T04:12:33.677856+0000","last_clean":"2023-10-09T04:12:33.677856+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:12:33.677856+0000","last_undegraded":"2023-10-09T04:12:33.677856+0000","last_fullsized":"2023-10-09T04:12:33.677856+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":177,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579099+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:12:24.579099+0000","last_peered":"2023-10-09T04:12:24.579099+0000","last_clean":"2023-10-09T04:12:24.579099+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:12:24.579099+0000","last_undegraded":"2023-10-09T04:12:24.579099+0000","last_fullsized":"2023-10-09T04:12:24.579099+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":209,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579963+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:12:24.579963+0000","last_peered":"2023-10-09T04:12:24.579963+0000","last_clean":"2023-10-09T04:12:24.579963+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:12:24.579963+0000","last_undegraded":"2023-10-09T04:12:24.579963+0000","last_fullsized":"2023-10-09T04:12:24.579963+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"222'6617","reported_seq":6754,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.548791+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:12:45.548791+0000","last_peered":"2023-10-09T04:12:45.548791+0000","last_clean":"2023-10-09T04:12:45.548791+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:12:45.548791+0000","last_undegraded":"2023-10-09T04:12:45.548791+0000","last_fullsized":"2023-10-09T04:12:45.548791+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35913728,"num_objects":548,"num_object_clones":0,"num_object_copies":1644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4070,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":185,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678902+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:12:33.678902+0000","last_peered":"2023-10-09T04:12:33.678902+0000","last_clean":"2023-10-09T04:12:33.678902+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:12:33.678902+0000","last_undegraded":"2023-10-09T04:12:33.678902+0000","last_fullsized":"2023-10-09T04:12:33.678902+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":191,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698385+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:12:33.698385+0000","last_peered":"2023-10-09T04:12:33.698385+0000","last_clean":"2023-10-09T04:12:33.698385+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:12:33.698385+0000","last_undegraded":"2023-10-09T04:12:33.698385+0000","last_fullsized":"2023-10-09T04:12:33.698385+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:28.102604+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:12:28.102604+0000","last_peered":"2023-10-09T04:12:28.102604+0000","last_clean":"2023-10-09T04:12:28.102604+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:12:28.102604+0000","last_undegraded":"2023-10-09T04:12:28.102604+0000","last_fullsized":"2023-10-09T04:12:28.102604+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"222'6644","reported_seq":9711,"reported_epoch":222,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:45.805141+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:12:45.805141+0000","last_peered":"2023-10-09T04:12:45.805141+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:12:45.805141+0000","last_undegraded":"2023-10-09T04:12:45.805141+0000","last_fullsized":"2023-10-09T04:12:45.805141+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6644,"log_dups_size":0,"ondisk_log_size":6644,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32964608,"num_objects":503,"num_object_clones":0,"num_object_copies":1509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":503,"num_objects_unfound":0,"num_objects_dirty":503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5473,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":503}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":126,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678994+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:12:33.678994+0000","last_peered":"2023-10-09T04:12:33.678994+0000","last_clean":"2023-10-09T04:12:33.678994+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:12:33.678994+0000","last_undegraded":"2023-10-09T04:12:33.678994+0000","last_fullsized":"2023-10-09T04:12:33.678994+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":158,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697700+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:12:33.697700+0000","last_peered":"2023-10-09T04:12:33.697700+0000","last_clean":"2023-10-09T04:12:33.697700+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:12:33.697700+0000","last_undegraded":"2023-10-09T04:12:33.697700+0000","last_fullsized":"2023-10-09T04:12:33.697700+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":152,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.678014+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:12:33.678014+0000","last_peered":"2023-10-09T04:12:33.678014+0000","last_clean":"2023-10-09T04:12:33.678014+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:12:33.678014+0000","last_undegraded":"2023-10-09T04:12:33.678014+0000","last_fullsized":"2023-10-09T04:12:33.678014+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"222'6452","reported_seq":8669,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.683565+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:12:43.683565+0000","last_peered":"2023-10-09T04:12:43.683565+0000","last_clean":"2023-10-09T04:12:43.683565+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:12:43.683565+0000","last_undegraded":"2023-10-09T04:12:43.683565+0000","last_fullsized":"2023-10-09T04:12:43.683565+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35061760,"num_objects":535,"num_object_clones":0,"num_object_copies":1605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3975,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":150,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.679122+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:12:33.679122+0000","last_peered":"2023-10-09T04:12:33.679122+0000","last_clean":"2023-10-09T04:12:33.679122+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:12:33.679122+0000","last_undegraded":"2023-10-09T04:12:33.679122+0000","last_fullsized":"2023-10-09T04:12:33.679122+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":180,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.580048+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:12:24.580048+0000","last_peered":"2023-10-09T04:12:24.580048+0000","last_clean":"2023-10-09T04:12:24.580048+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:12:24.580048+0000","last_undegraded":"2023-10-09T04:12:24.580048+0000","last_fullsized":"2023-10-09T04:12:24.580048+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":163,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.686796+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:12:33.686796+0000","last_peered":"2023-10-09T04:12:33.686796+0000","last_clean":"2023-10-09T04:12:33.686796+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:12:33.686796+0000","last_undegraded":"2023-10-09T04:12:33.686796+0000","last_fullsized":"2023-10-09T04:12:33.686796+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":165,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.579406+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:12:24.579406+0000","last_peered":"2023-10-09T04:12:24.579406+0000","last_clean":"2023-10-09T04:12:24.579406+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:12:24.579406+0000","last_undegraded":"2023-10-09T04:12:24.579406+0000","last_fullsized":"2023-10-09T04:12:24.579406+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":35,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.683770+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:12:33.683770+0000","last_peered":"2023-10-09T04:12:33.683770+0000","last_clean":"2023-10-09T04:12:33.683770+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:12:33.683770+0000","last_undegraded":"2023-10-09T04:12:33.683770+0000","last_fullsized":"2023-10-09T04:12:33.683770+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"222'6452","reported_seq":6529,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:44.981947+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:12:44.981947+0000","last_peered":"2023-10-09T04:12:44.981947+0000","last_clean":"2023-10-09T04:12:44.981947+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:12:44.981947+0000","last_undegraded":"2023-10-09T04:12:44.981947+0000","last_fullsized":"2023-10-09T04:12:44.981947+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33161216,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5312,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":183,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688194+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:12:33.688194+0000","last_peered":"2023-10-09T04:12:33.688194+0000","last_clean":"2023-10-09T04:12:33.688194+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:12:33.688194+0000","last_undegraded":"2023-10-09T04:12:33.688194+0000","last_fullsized":"2023-10-09T04:12:33.688194+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":197,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685936+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:12:33.685936+0000","last_peered":"2023-10-09T04:12:33.685936+0000","last_clean":"2023-10-09T04:12:33.685936+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:12:33.685936+0000","last_undegraded":"2023-10-09T04:12:33.685936+0000","last_fullsized":"2023-10-09T04:12:33.685936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"221'6507","reported_seq":6578,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.107118+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:12:43.107118+0000","last_peered":"2023-10-09T04:12:43.107118+0000","last_clean":"2023-10-09T04:12:43.107118+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:12:43.107118+0000","last_undegraded":"2023-10-09T04:12:43.107118+0000","last_fullsized":"2023-10-09T04:12:43.107118+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6507,"log_dups_size":0,"ondisk_log_size":6507,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36634624,"num_objects":559,"num_object_clones":0,"num_object_copies":1677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5345,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":166,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.688295+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:12:33.688295+0000","last_peered":"2023-10-09T04:12:33.688295+0000","last_clean":"2023-10-09T04:12:33.688295+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:12:33.688295+0000","last_undegraded":"2023-10-09T04:12:33.688295+0000","last_fullsized":"2023-10-09T04:12:33.688295+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":197,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685923+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:12:33.685923+0000","last_peered":"2023-10-09T04:12:33.685923+0000","last_clean":"2023-10-09T04:12:33.685923+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:12:33.685923+0000","last_undegraded":"2023-10-09T04:12:33.685923+0000","last_fullsized":"2023-10-09T04:12:33.685923+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":36,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:40.085088+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:12:40.085088+0000","last_peered":"2023-10-09T04:12:40.085088+0000","last_clean":"2023-10-09T04:12:40.085088+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:12:40.085088+0000","last_undegraded":"2023-10-09T04:12:40.085088+0000","last_fullsized":"2023-10-09T04:12:40.085088+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":35,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.683598+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:12:33.683598+0000","last_peered":"2023-10-09T04:12:33.683598+0000","last_clean":"2023-10-09T04:12:33.683598+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:12:33.683598+0000","last_undegraded":"2023-10-09T04:12:33.683598+0000","last_fullsized":"2023-10-09T04:12:33.683598+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":211,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-10-09T04:12:24.580096+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:12:24.580096+0000","last_peered":"2023-10-09T04:12:24.580096+0000","last_clean":"2023-10-09T04:12:24.580096+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:12:24.580096+0000","last_undegraded":"2023-10-09T04:12:24.580096+0000","last_fullsized":"2023-10-09T04:12:24.580096+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":180,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.685636+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:12:33.685636+0000","last_peered":"2023-10-09T04:12:33.685636+0000","last_clean":"2023-10-09T04:12:33.685636+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:12:33.685636+0000","last_undegraded":"2023-10-09T04:12:33.685636+0000","last_fullsized":"2023-10-09T04:12:33.685636+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1120403480,"num_objects":17097,"num_object_clones":0,"num_object_copies":51291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":785,"num_objects_misplaced":1519,"num_objects_unfound":0,"num_objects_dirty":17097,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163000,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13287,"num_bytes_recovered":870776832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3828035584,"data_stored":3828023368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177434,"ondisk_log_size":177434,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201523,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":739264,"kb_used_data":433784,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93005888,"statfs":{"total":95995035648,"available":95238029312,"internally_reserved":0,"allocated":444194816,"data_stored":443520257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":4.7140000000000004,"5min":4.0140000000000002,"15min":1.863},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":62.140000000000001,"5min":126.752,"15min":126.752},"last":0.84599999999999997},{"interface":"front","average":{"1min":4.5869999999999997,"5min":4.194,"15min":1.8779999999999999},"min":{"1min":0.308,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":62.340000000000003,"5min":128.88399999999999,"15min":128.88399999999999},"last":0.22700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":10.563000000000001,"5min":5.8120000000000003,"15min":2.3980000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":103.014,"5min":245.28700000000001,"15min":245.28700000000001},"last":8.5079999999999991},{"interface":"front","average":{"1min":10.186999999999999,"5min":5.3700000000000001,"15min":2.379},"min":{"1min":0.255,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":103.083,"5min":244.626,"15min":244.626},"last":2.843}]},{"osd":2,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.776,"15min":1.3200000000000001},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":4.351,"5min":13.887,"15min":58.615000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.97,"15min":1.77},"min":{"1min":0.38600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.2320000000000002,"5min":13.872,"15min":99.372},"last":0.371}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.5469999999999999,"15min":1.5589999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":3.0430000000000001,"5min":13.124000000000001,"15min":99.134},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.6599999999999999,"15min":1.6399999999999999},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.0230000000000001,"5min":13.140000000000001,"15min":99.117999999999995},"last":0.34599999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.0169999999999999,"15min":0.69799999999999995},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.98399999999999999,"5min":16.195,"15min":16.195},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97299999999999998,"15min":0.71799999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.343,"5min":12.442,"15min":12.442},"last":0.41099999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68700000000000006,"15min":0.61499999999999999},"min":{"1min":0.29299999999999998,"5min":0.27700000000000002,"15min":0.27400000000000002},"max":{"1min":4.4829999999999997,"5min":4.4829999999999997,"15min":6.5439999999999996},"last":0.877},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76000000000000001,"15min":0.67800000000000005},"min":{"1min":0.33600000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83699999999999997,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.35699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.80300000000000005,"15min":0.69899999999999995},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.5199999999999996,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.92600000000000005,"15min":0.75900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":2.399,"5min":5.6420000000000003,"15min":5.9039999999999999},"last":0.45200000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463160,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":752308,"kb_used_data":446828,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92992844,"statfs":{"total":95995035648,"available":95224672256,"internally_reserved":0,"allocated":457551872,"data_stored":456889156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":6.4000000000000004,"5min":4.4550000000000001,"15min":2.2370000000000001},"min":{"1min":0.222,"5min":0.217,"15min":0.217},"max":{"1min":77.453000000000003,"5min":77.453000000000003,"15min":78.983999999999995},"last":0.38300000000000001},{"interface":"front","average":{"1min":6.9790000000000001,"5min":4.9480000000000004,"15min":2.5129999999999999},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":77.430999999999997,"5min":77.430999999999997,"15min":78.998000000000005},"last":0.58799999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.657,"15min":1.6950000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":4.6840000000000002,"5min":62.698,"15min":62.698},"last":1.075},{"interface":"front","average":{"1min":1.909,"5min":3.875,"15min":1.794},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":8.3529999999999998,"5min":63.530999999999999,"15min":63.530999999999999},"last":1.1499999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":3.7120000000000002,"15min":2.0350000000000001},"min":{"1min":0.33700000000000002,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":33.345999999999997,"5min":77.953000000000003,"15min":77.953000000000003},"last":1.244},{"interface":"front","average":{"1min":3.5680000000000001,"5min":3.7610000000000001,"15min":2.085},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":33.368000000000002,"5min":78.019000000000005,"15min":78.019000000000005},"last":1.278}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":3.516,"15min":2.0339999999999998},"min":{"1min":0.35799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.336,"5min":57.509999999999998,"15min":57.509999999999998},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":3.7450000000000001,"15min":2.052},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.1840000000000002,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.72999999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.3720000000000001,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.72199999999999998,"15min":0.64400000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2970000000000002,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.26000000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":0.84599999999999997,"15min":0.84899999999999998},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.081,"5min":0.97399999999999998,"15min":0.95499999999999996},"min":{"1min":0.34399999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":23.006,"5min":23.006,"15min":54.853999999999999},"last":0.312}]},{"osd":7,"last update":"Mon Oct 9 04:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68600000000000005,"15min":0.628},"min":{"1min":0.317,"5min":0.313,"15min":0.313},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.79000000000000004,"15min":0.71199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":3.3679999999999999,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.33600000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089468,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":808300,"kb_used_data":562084,"kb_used_omap":9,"kb_used_meta":246198,"kb_avail":92936852,"statfs":{"total":95995035648,"available":95167336448,"internally_reserved":0,"allocated":575574016,"data_stored":574676263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":252107753},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":6.4269999999999996,"5min":3.7930000000000001,"15min":1.843},"min":{"1min":0.247,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":61.689999999999998,"5min":85.977999999999994,"15min":85.977999999999994},"last":3.6840000000000002},{"interface":"front","average":{"1min":7.4409999999999998,"5min":4.2519999999999998,"15min":2.3519999999999999},"min":{"1min":0.31900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":61.670000000000002,"5min":85.998999999999995,"15min":99.831999999999994},"last":4.173}]},{"osd":2,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.9359999999999999,"15min":1.7589999999999999},"min":{"1min":0.26100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":13.695,"5min":77.563000000000002,"15min":121.54600000000001},"last":5.056},{"interface":"front","average":{"1min":4.9939999999999998,"5min":3.452,"15min":2.0110000000000001},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.186},"max":{"1min":15.298999999999999,"5min":77.504999999999995,"15min":99.899000000000001},"last":5.1020000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":4.2089999999999996,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":7.8920000000000003,"5min":138.87299999999999,"15min":138.87299999999999},"last":2.1019999999999999},{"interface":"front","average":{"1min":3.593,"5min":4.6449999999999996,"15min":2.6309999999999998},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.414,"5min":138.49799999999999,"15min":138.49799999999999},"last":4.1520000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0270000000000001,"5min":1.8520000000000001,"15min":1.498},"min":{"1min":0.35599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":15.215999999999999,"5min":15.215999999999999,"15min":99.989999999999995},"last":4.0960000000000001},{"interface":"front","average":{"1min":1.115,"5min":1.5129999999999999,"15min":1.3779999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.52,"5min":10.438000000000001,"15min":100.249},"last":3.8809999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.75,"15min":1.48},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":15.266,"5min":15.266,"15min":99.933999999999997},"last":4.1289999999999996},{"interface":"front","average":{"1min":1.389,"5min":1.829,"15min":1.4950000000000001},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":4.8849999999999998,"5min":10.467000000000001,"15min":100.34099999999999},"last":3.9220000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.827,"15min":1.5880000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.585,"5min":10.497999999999999,"15min":100.309},"last":3.9969999999999999},{"interface":"front","average":{"1min":1.671,"5min":1.496,"15min":1.5169999999999999},"min":{"1min":0.36499999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.9240000000000004,"5min":7.9240000000000004,"15min":100.143},"last":2.0760000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:11:50 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.9630000000000001,"15min":1.569},"min":{"1min":0.40000000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.9569999999999999,"5min":10.532,"15min":100.214},"last":3.9540000000000002},{"interface":"front","average":{"1min":1.798,"5min":1.796,"15min":1.5429999999999999},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":7.9740000000000002,"5min":7.9740000000000002,"15min":100.113},"last":2.0430000000000001}]}]},{"osd":0,"up_from":84,"seq":360777252994,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":863176,"kb_used_data":619328,"kb_used_omap":28,"kb_used_meta":243811,"kb_avail":92881976,"statfs":{"total":95995035648,"available":95111143424,"internally_reserved":0,"allocated":634191872,"data_stored":633265601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":249663312},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":3.6059999999999999,"15min":1.982},"min":{"1min":0.184,"5min":0.17599999999999999,"15min":0.17000000000000001},"max":{"1min":12.698,"5min":103.996,"15min":118.48},"last":4.7679999999999998},{"interface":"front","average":{"1min":3.5910000000000002,"5min":4.1959999999999997,"15min":2.2080000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":12.632999999999999,"5min":106.34399999999999,"15min":118.384},"last":0.92800000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:12:21 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":3.0339999999999998,"15min":2.2490000000000001},"min":{"1min":0.38,"5min":0.221,"15min":0.221},"max":{"1min":13.108000000000001,"5min":25.055,"15min":118.54000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":4.5979999999999999,"5min":3.2890000000000001,"15min":2.367},"min":{"1min":0.22900000000000001,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":12.372,"5min":44.436999999999998,"15min":118.727},"last":11.974}]},{"osd":3,"last update":"Mon Oct 9 04:11:45 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":5.9320000000000004,"15min":3.601},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.161,"5min":156.488,"15min":156.488},"last":4.79},{"interface":"front","average":{"1min":4.1909999999999998,"5min":5.8979999999999997,"15min":3.6890000000000001},"min":{"1min":0.40699999999999997,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":9.6010000000000009,"5min":156.50700000000001,"15min":156.50700000000001},"last":0.995}]},{"osd":4,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.0219999999999998,"15min":1.8080000000000001},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7270000000000003,"5min":10.023,"15min":118.633},"last":1.0289999999999999},{"interface":"front","average":{"1min":2.714,"5min":2.4769999999999999,"15min":2.0379999999999998},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.492000000000001,"5min":44.475000000000001,"15min":118.913},"last":5.4199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.6360000000000001,"15min":2.552},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":10.445,"5min":44.494,"15min":182.68899999999999},"last":5.3840000000000003},{"interface":"front","average":{"1min":1.887,"5min":2.2210000000000001,"15min":2.387},"min":{"1min":0.41299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.4749999999999996,"5min":44.328000000000003,"15min":182.66300000000001},"last":4.673}]},{"osd":6,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":3.9180000000000001,"15min":2.3010000000000002},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.5090000000000003,"5min":137.36099999999999,"15min":137.36099999999999},"last":4.7460000000000004},{"interface":"front","average":{"1min":3.649,"5min":3.9220000000000002,"15min":2.2679999999999998},"min":{"1min":0.44500000000000001,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":12.496,"5min":136.88999999999999,"15min":136.88999999999999},"last":1.0580000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":3.3410000000000002,"15min":2.9169999999999998},"min":{"1min":0.38700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.5500000000000007,"5min":121.324,"15min":185.024},"last":4.7130000000000001},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.3239999999999998,"15min":2.508},"min":{"1min":0.46700000000000003,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":12.417,"5min":12.417,"15min":184.98599999999999},"last":1.0920000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925942,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":829924,"kb_used_data":496348,"kb_used_omap":81,"kb_used_meta":333486,"kb_avail":92915228,"statfs":{"total":95995035648,"available":95145193472,"internally_reserved":0,"allocated":508260352,"data_stored":507376416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":341489882},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2460000000000004,"5min":2.9359999999999999,"15min":1.4690000000000001},"min":{"1min":0.20200000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":22.321000000000002,"5min":22.321000000000002,"15min":22.321000000000002},"last":6.6849999999999996},{"interface":"front","average":{"1min":3.6800000000000002,"5min":3.581,"15min":1.738},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.6440000000000001,"5min":45.979999999999997,"15min":45.979999999999997},"last":0.74199999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":3.5910000000000002,"15min":1.9730000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":9.9429999999999996,"5min":53.695,"15min":68.009},"last":0.89800000000000002},{"interface":"front","average":{"1min":3.7149999999999999,"5min":3.7090000000000001,"15min":2.1419999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.346,"5min":53.439,"15min":66.975999999999999},"last":0.83099999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:11:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0150000000000001,"5min":2.2029999999999998,"15min":1.349},"min":{"1min":0.22500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.212999999999999,"5min":12.77,"15min":27.562000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":2.577,"5min":2.8199999999999998,"15min":1.552},"min":{"1min":0.26000000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.808,"5min":46.286999999999999,"15min":46.286999999999999},"last":10.522}]},{"osd":4,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.2490000000000001,"15min":1.444},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5279999999999996,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.609,"5min":1.9419999999999999,"15min":1.234},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":20.134,"5min":20.134,"15min":20.134},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.6789999999999998,"15min":1.4770000000000001},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":7.3399999999999999,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.1779999999999999,"5min":2.6339999999999999,"15min":1.6020000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5039999999999996,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6579999999999999,"5min":3.2829999999999999,"15min":1.671},"min":{"1min":0.376,"5min":0.311,"15min":0.311},"max":{"1min":20.184000000000001,"5min":114.14,"15min":114.14},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.194,"5min":3.7719999999999998,"15min":1.8560000000000001},"min":{"1min":0.43099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4199999999999999,"5min":114.123,"15min":114.123},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.556,"15min":1.8640000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":20.215,"5min":117.32599999999999,"15min":117.32599999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":2.6349999999999998,"5min":3.9209999999999998,"15min":1.982},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.4659999999999993,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.64200000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762416,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1183952,"kb_used_data":751176,"kb_used_omap":7,"kb_used_meta":432760,"kb_avail":92561200,"statfs":{"total":95995035648,"available":94782668800,"internally_reserved":0,"allocated":769204224,"data_stored":768337610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":443146856},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.8140000000000001,"5min":4.6970000000000001,"15min":3.3519999999999999},"min":{"1min":0.20100000000000001,"5min":0.189,"15min":0.185},"max":{"1min":23.449000000000002,"5min":100.512,"15min":106.22},"last":0.96699999999999997},{"interface":"front","average":{"1min":4.0759999999999996,"5min":5.056,"15min":3.4039999999999999},"min":{"1min":0.32200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":13.670999999999999,"5min":101.733,"15min":106.986},"last":4.2309999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":3.5699999999999998,"15min":2.2490000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":8.6340000000000003,"5min":57.055999999999997,"15min":106.121},"last":0.33800000000000002},{"interface":"front","average":{"1min":3.899,"5min":3.9180000000000001,"15min":2.1019999999999999},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":23.475999999999999,"5min":57.088999999999999,"15min":65.015000000000001},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.8130000000000002,"15min":2.1669999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":23.405999999999999,"5min":23.405999999999999,"15min":106.105},"last":0.40699999999999997},{"interface":"front","average":{"1min":3.524,"5min":3.7229999999999999,"15min":2.5190000000000001},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":13.698,"5min":98.765000000000001,"15min":106.70399999999999},"last":4.0170000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":2.9060000000000001,"15min":1.927},"min":{"1min":0.35799999999999998,"5min":0.31,"15min":0.308},"max":{"1min":13.536,"5min":37.652000000000001,"15min":106.85899999999999},"last":4.1079999999999997},{"interface":"front","average":{"1min":1.873,"5min":2.5209999999999999,"15min":1.351},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.1699999999999999,"5min":37.442,"15min":37.442},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.1150000000000002,"15min":1.2430000000000001},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.1940000000000008,"5min":12.819000000000001,"15min":12.819000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.9730000000000001,"15min":1.516},"min":{"1min":0.32400000000000001,"5min":0.29599999999999999,"15min":0.27900000000000003},"max":{"1min":11.938000000000001,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":1.8560000000000001,"15min":1.9039999999999999},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.30199999999999999},"max":{"1min":11.994999999999999,"5min":13.255000000000001,"15min":106.036},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.363,"5min":2.355,"15min":1.7789999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.571,"5min":15.622,"15min":106.741},"last":4.1379999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":2.5099999999999998,"15min":1.8260000000000001},"min":{"1min":0.38700000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":13.494999999999999,"5min":15.685,"15min":106.82599999999999},"last":4.1669999999999998},{"interface":"front","average":{"1min":2.1480000000000001,"5min":2.1749999999999998,"15min":1.7010000000000001},"min":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.2170000000000005,"5min":13.228999999999999,"15min":106.084},"last":3.8860000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986432,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":607092,"kb_used_data":305708,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93138060,"statfs":{"total":95995035648,"available":95373373440,"internally_reserved":0,"allocated":313044992,"data_stored":312381842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7560000000000002,"5min":3.3079999999999998,"15min":1.631},"min":{"1min":0.222,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":38.831000000000003,"5min":63.128,"15min":63.128},"last":0.30399999999999999},{"interface":"front","average":{"1min":4.7939999999999996,"5min":3.0590000000000002,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.039999999999999,"5min":62.645000000000003,"15min":62.645000000000003},"last":10.763999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.7709999999999999,"15min":1.4410000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":23.157,"5min":79.075999999999993,"15min":79.075999999999993},"last":7.6660000000000004},{"interface":"front","average":{"1min":2.8410000000000002,"5min":2.8479999999999999,"15min":1.4299999999999999},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":23.091999999999999,"5min":75.314999999999998,"15min":75.314999999999998},"last":5.1980000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.5720000000000001,"15min":1.179},"min":{"1min":0.255,"5min":0.255,"15min":0.245},"max":{"1min":5.4470000000000001,"5min":8.8030000000000008,"15min":19.356000000000002},"last":5.0099999999999998},{"interface":"front","average":{"1min":1.881,"5min":1.6299999999999999,"15min":1.1299999999999999},"min":{"1min":0.38700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.4610000000000003,"5min":8.5920000000000005,"15min":19.324000000000002},"last":4.9989999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.8779999999999999,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4880000000000004,"5min":14.694000000000001,"15min":14.694000000000001},"last":4.7400000000000002},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.958,"15min":1.222},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.2110000000000003,"5min":10.476000000000001,"15min":10.476000000000001},"last":15.1}]},{"osd":5,"last update":"Mon Oct 9 04:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55000000000000004,"15min":0.58099999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3050000000000002,"5min":3.4510000000000001,"15min":8.5259999999999998},"last":1.161},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.956,"5min":5.9569999999999999,"15min":8.3149999999999995},"last":1.22}]},{"osd":6,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.93799999999999994,"15min":0.74099999999999999},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":2.9990000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.81899999999999995,"15min":0.66600000000000004},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9700000000000002,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.39000000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.70299999999999996,"15min":0.70699999999999996},"min":{"1min":0.311,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":2.3599999999999999,"5min":3.5190000000000001,"15min":9.0050000000000008},"last":1.121},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.79100000000000004,"15min":0.77200000000000002},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.29199999999999998},"max":{"1min":2.214,"5min":5.9939999999999998,"15min":8.5440000000000005},"last":0.93400000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724796,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":546468,"kb_used_data":240988,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93198684,"statfs":{"total":95995035648,"available":95435452416,"internally_reserved":0,"allocated":246771712,"data_stored":246120632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":24.518000000000001,"5min":7.5119999999999996,"15min":3.1709999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":151.23599999999999,"5min":151.23599999999999,"15min":151.23599999999999},"last":2.726},{"interface":"front","average":{"1min":25.135000000000002,"5min":7.8940000000000001,"15min":3.3359999999999999},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":151.566,"5min":151.566,"15min":151.566},"last":10.193}]},{"osd":1,"last update":"Mon Oct 9 04:11:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3869999999999996,"5min":2.6749999999999998,"15min":1.7030000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":59.588999999999999,"5min":59.588999999999999,"15min":91.212999999999994},"last":1.1930000000000001},{"interface":"front","average":{"1min":4.6139999999999999,"5min":2.9550000000000001,"15min":1.913},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":59.533000000000001,"5min":59.533000000000001,"15min":91.254000000000005},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":5.7990000000000004,"5min":3.677,"15min":1.8720000000000001},"min":{"1min":0.29699999999999999,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":67.685000000000002,"5min":67.685000000000002,"15min":67.685000000000002},"last":1.502},{"interface":"front","average":{"1min":6.0250000000000004,"5min":3.96,"15min":1.8839999999999999},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.596000000000004,"5min":67.596000000000004,"15min":67.596000000000004},"last":3.391}]},{"osd":3,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.772,"15min":1.7010000000000001},"min":{"1min":0.35299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.714,"5min":17.071000000000002,"15min":91.834999999999994},"last":2.5960000000000001},{"interface":"front","average":{"1min":1.548,"5min":1.885,"15min":1.7450000000000001},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.309},"max":{"1min":5.9749999999999996,"5min":17.052,"15min":91.802000000000007},"last":2.569}]},{"osd":4,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.75600000000000001,"15min":0.65300000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.64600000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.307},{"interface":"front","average":{"1min":0.371,"5min":0.78200000000000003,"15min":0.70799999999999996},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.51700000000000002,"5min":5.29,"15min":6.7709999999999999},"last":0.38500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:11:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2839999999999998,"5min":2.1230000000000002,"15min":1.113},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":123.239,"5min":123.239,"15min":123.239},"last":0.35699999999999998},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.0899999999999999,"15min":1.129},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":123.014,"5min":123.014,"15min":123.014},"last":0.41899999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.877,"15min":0.77500000000000002},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":0.64100000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.747,"15min":0.79800000000000004},"min":{"1min":0.35599999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.73499999999999999,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.45400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":624824320,"data_stored":624820248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":556208128,"data_stored":556204056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":495190016,"data_stored":495190016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":745406464,"data_stored":745406464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":308215808,"data_stored":308215808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":233963520,"data_stored":233963520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":436670464,"data_stored":436666392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":427556864,"data_stored":427556864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:12:49.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:49.378 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:49.383 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:49.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:49.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:49.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:49.556 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[213841]: from='client.45299 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[221067]: from='client.45299 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:49.641 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:49.731 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:12:49.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:49 smithi181 ceph-mon[196979]: from='client.45299 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:12:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:49.816 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:12:49.868 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:12:49.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:49.907 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:12:49.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:49.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:49.964 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:12:50.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:50.015 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:12:50.041 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:50.059 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:12:50.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:50.107 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:12:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:50.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:50.150 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.150 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.150 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.150 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.151 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.151 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.151 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:12:50.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:50.304 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:50 smithi163 ceph-mon[221067]: pgmap v595: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 956 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 785/45951 objects degraded (1.708%); 1519/45951 objects misplaced (3.306%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:12:50.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3062067250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:50 smithi163 ceph-mon[213841]: pgmap v595: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 956 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 785/45951 objects degraded (1.708%); 1519/45951 objects misplaced (3.306%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:12:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3062067250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:50 smithi181 ceph-mon[196979]: pgmap v595: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 956 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 785/45951 objects degraded (1.708%); 1519/45951 objects misplaced (3.306%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:12:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3062067250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:12:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:12:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:12:50] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-10-09T04:12:51.335 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:51.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:51.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:51.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:51.505 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:51.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:51.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:52.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:52.103 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:52.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:52.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:52.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:12:52.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:52.519 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:52.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:52 smithi163 ceph-mon[221067]: pgmap v596: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 856 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 511/41148 objects degraded (1.242%); 1227/41148 objects misplaced (2.982%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:52.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 511/41148 objects degraded (1.242%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:52.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:52 smithi163 ceph-mon[213841]: pgmap v596: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 856 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 511/41148 objects degraded (1.242%); 1227/41148 objects misplaced (2.982%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:52.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 511/41148 objects degraded (1.242%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:52.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:52.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:52.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:52.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:52.803 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:52.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:52.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:52 smithi181 ceph-mon[196979]: pgmap v596: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 856 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 511/41148 objects degraded (1.242%); 1227/41148 objects misplaced (2.982%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:52.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 511/41148 objects degraded (1.242%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:52.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:52.908 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:12:52.908 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:12:52.908 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:12:52.908 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:12:52.908 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:12:52.965 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:53.070 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:53.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:53.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:53.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:53.423 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:53.540 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:53.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:53.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:53.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:53.733 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:53.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:53.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:53.903 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:53.981 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:54.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:54.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:54.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:12:54.348 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:54.366 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:54.395 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:54.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:54.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:54.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:12:54.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:54.586 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:54.639 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:54.643 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:54 smithi181 ceph-mon[196979]: pgmap v597: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 783 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 511/37644 objects degraded (1.357%); 1227/37644 objects misplaced (3.259%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:54.644 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1713392482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:54.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:54.646 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:12:54.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:54.843 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:54 smithi163 ceph-mon[213841]: pgmap v597: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 783 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 511/37644 objects degraded (1.357%); 1227/37644 objects misplaced (3.259%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:54.843 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1713392482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:54.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:54 smithi163 ceph-mon[221067]: pgmap v597: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 783 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 511/37644 objects degraded (1.357%); 1227/37644 objects misplaced (3.259%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:54.847 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1713392482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:12:54.856 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:55.703 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:55.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:55.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:12:55.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:55 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:55.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1713392482' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:55.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:55 smithi181 ceph-mon[196979]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T04:12:55.843 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:55 smithi181 ceph-mon[196979]: failed to encode map e223 with expected crc 2023-10-09T04:12:55.847 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:55.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:12:55.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:55.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:12:56.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:56.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1713392482' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[213841]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[213841]: failed to encode map e223 with expected crc 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1713392482' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[221067]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T04:12:56.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:55 smithi163 ceph-mon[221067]: failed to encode map e223 with expected crc 2023-10-09T04:12:56.067 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:56.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:56.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:56.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:56.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:56.409 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:56.423 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:56.442 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:12:56.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:56.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:56.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:56.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:56.654 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:56 smithi181 ceph-mon[196979]: failed to encode map e223 with expected crc 2023-10-09T04:12:56.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:56 smithi181 ceph-mon[196979]: pgmap v599: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 709 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 511/34080 objects degraded (1.499%); 1227/34080 objects misplaced (3.600%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:56.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:56.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:56.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:12:56.892 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:56 smithi163 ceph-mon[221067]: failed to encode map e223 with expected crc 2023-10-09T04:12:56.892 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:56 smithi163 ceph-mon[221067]: pgmap v599: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 709 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 511/34080 objects degraded (1.499%); 1227/34080 objects misplaced (3.600%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:56.893 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:56 smithi163 ceph-mon[213841]: failed to encode map e223 with expected crc 2023-10-09T04:12:56.893 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:56 smithi163 ceph-mon[213841]: pgmap v599: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 709 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 511/34080 objects degraded (1.499%); 1227/34080 objects misplaced (3.600%); 1.7 MiB/s, 54 objects/s recovering 2023-10-09T04:12:56.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:12:57.105 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:57.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:12:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:57.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:12:57.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:12:57.375 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:57.426 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:12:57.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:57.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:12:57.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:12:57.547 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:12:57.549 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:12:57.616 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:57.757 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 511/34080 objects degraded (1.499%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:57.757 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 511/34080 objects degraded (1.499%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:57.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:12:57.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:57.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:12:57.908 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:12:57.940 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 511/34080 objects degraded (1.499%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:12:57.947 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:58.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:12:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:58.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:12:58.119 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:12:58.152 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:58.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:12:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:58.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:12:58.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:12:58.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:12:58.335 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:58 smithi163 ceph-mon[221067]: pgmap v600: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 644 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 177/30966 objects degraded (0.572%); 907/30966 objects misplaced (2.929%); 0 B/s, 60 objects/s recovering 2023-10-09T04:12:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:12:58 smithi163 ceph-mon[221067]: from='client.35382 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:58 smithi163 ceph-mon[213841]: pgmap v600: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 644 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 177/30966 objects degraded (0.572%); 907/30966 objects misplaced (2.929%); 0 B/s, 60 objects/s recovering 2023-10-09T04:12:58.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:12:58 smithi163 ceph-mon[213841]: from='client.35382 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:58 smithi181 ceph-mon[196979]: pgmap v600: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 644 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 177/30966 objects degraded (0.572%); 907/30966 objects misplaced (2.929%); 0 B/s, 60 objects/s recovering 2023-10-09T04:12:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:12:58 smithi181 ceph-mon[196979]: from='client.35382 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:12:59.525 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:12:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:59.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:12:59.622 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":600,"stamp":"2023-10-09T04:12:57.510106+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":675420088,"num_objects":10322,"num_object_clones":0,"num_object_copies":30966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":177,"num_objects_misplaced":907,"num_objects_unfound":0,"num_objects_dirty":10322,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":170099,"num_write_kb":5561166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13905,"num_bytes_recovered":872549044,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181849,"ondisk_log_size":181849,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4970312,"kb_used_data":2491848,"kb_used_omap":126,"kb_used_meta":2478145,"kb_avail":744990904,"statfs":{"total":767960285184,"available":762870685696,"internally_reserved":0,"allocated":2551652352,"data_stored":2545398063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2537621115},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,2,5,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-445579264,"num_objects":-6799,"num_object_clones":0,"num_object_copies":-20397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-608,"num_objects_misplaced":-612,"num_objects_unfound":0,"num_objects_dirty":-6799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6799,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":608,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006483"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":220,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625187+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:12:43.625187+0000","last_peered":"2023-10-09T04:12:43.625187+0000","last_clean":"2023-10-09T04:12:43.625187+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:12:43.625187+0000","last_undegraded":"2023-10-09T04:12:43.625187+0000","last_fullsized":"2023-10-09T04:12:43.625187+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698004+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:12:33.698004+0000","last_peered":"2023-10-09T04:12:33.698004+0000","last_clean":"2023-10-09T04:12:33.698004+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:12:33.698004+0000","last_undegraded":"2023-10-09T04:12:33.698004+0000","last_fullsized":"2023-10-09T04:12:33.698004+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698616+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:12:33.698616+0000","last_peered":"2023-10-09T04:12:33.698616+0000","last_clean":"2023-10-09T04:12:33.698616+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:33.698616+0000","last_undegraded":"2023-10-09T04:12:33.698616+0000","last_fullsized":"2023-10-09T04:12:33.698616+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.681863+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:12:33.681863+0000","last_peered":"2023-10-09T04:12:33.681863+0000","last_clean":"2023-10-09T04:12:33.681863+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:12:33.681863+0000","last_undegraded":"2023-10-09T04:12:33.681863+0000","last_fullsized":"2023-10-09T04:12:33.681863+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":176,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672811+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:12:33.672811+0000","last_peered":"2023-10-09T04:12:33.672811+0000","last_clean":"2023-10-09T04:12:33.672811+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:12:33.672811+0000","last_undegraded":"2023-10-09T04:12:33.672811+0000","last_fullsized":"2023-10-09T04:12:33.672811+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":156,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622431+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:12:43.622431+0000","last_peered":"2023-10-09T04:12:43.622431+0000","last_clean":"2023-10-09T04:12:43.622431+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:12:43.622431+0000","last_undegraded":"2023-10-09T04:12:43.622431+0000","last_fullsized":"2023-10-09T04:12:43.622431+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"223'6654","reported_seq":9617,"reported_epoch":223,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:55.779737+0000","last_change":"2023-10-09T04:10:39.425608+0000","last_active":"2023-10-09T04:12:55.779737+0000","last_peered":"2023-10-09T04:12:55.779737+0000","last_clean":"2023-10-09T04:10:37.142206+0000","last_became_active":"2023-10-09T04:10:39.421981+0000","last_became_peered":"2023-10-09T04:10:39.421981+0000","last_unstale":"2023-10-09T04:12:55.779737+0000","last_undegraded":"2023-10-09T04:12:55.779737+0000","last_fullsized":"2023-10-09T04:12:55.779737+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6654,"log_dups_size":0,"ondisk_log_size":6654,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19988480,"num_objects":305,"num_object_clones":0,"num_object_copies":915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":305,"num_objects_unfound":0,"num_objects_dirty":305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5483,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1429,"num_bytes_recovered":93650944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":305}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":177,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.673496+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:12:33.673496+0000","last_peered":"2023-10-09T04:12:33.673496+0000","last_clean":"2023-10-09T04:12:33.673496+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:12:33.673496+0000","last_undegraded":"2023-10-09T04:12:33.673496+0000","last_fullsized":"2023-10-09T04:12:33.673496+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":202,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:54.639958+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:12:54.639958+0000","last_peered":"2023-10-09T04:12:54.639958+0000","last_clean":"2023-10-09T04:12:54.639958+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:12:54.639958+0000","last_undegraded":"2023-10-09T04:12:54.639958+0000","last_fullsized":"2023-10-09T04:12:54.639958+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"222'6757","reported_seq":14158,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.070179+0000","last_change":"2023-10-09T04:12:19.342281+0000","last_active":"2023-10-09T04:12:53.070179+0000","last_peered":"2023-10-09T04:12:53.070179+0000","last_clean":"2023-10-09T04:12:53.070179+0000","last_became_active":"2023-10-09T04:12:19.328982+0000","last_became_peered":"2023-10-09T04:12:19.328982+0000","last_unstale":"2023-10-09T04:12:53.070179+0000","last_undegraded":"2023-10-09T04:12:53.070179+0000","last_fullsized":"2023-10-09T04:12:53.070179+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23724032,"num_objects":362,"num_object_clones":0,"num_object_copies":1086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5592,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2553,"num_bytes_recovered":167313408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.623013+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:12:43.623013+0000","last_peered":"2023-10-09T04:12:43.623013+0000","last_clean":"2023-10-09T04:12:43.623013+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:12:43.623013+0000","last_undegraded":"2023-10-09T04:12:43.623013+0000","last_fullsized":"2023-10-09T04:12:43.623013+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.612767+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:12:43.612767+0000","last_peered":"2023-10-09T04:12:43.612767+0000","last_clean":"2023-10-09T04:12:43.612767+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:12:43.612767+0000","last_undegraded":"2023-10-09T04:12:43.612767+0000","last_fullsized":"2023-10-09T04:12:43.612767+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"223'6622","reported_seq":9530,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:55.584422+0000","last_change":"2023-10-09T04:12:38.956478+0000","last_active":"2023-10-09T04:12:55.584422+0000","last_peered":"2023-10-09T04:12:55.584422+0000","last_clean":"2023-10-09T04:12:55.584422+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:12:55.584422+0000","last_undegraded":"2023-10-09T04:12:55.584422+0000","last_fullsized":"2023-10-09T04:12:55.584422+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6622,"log_dups_size":0,"ondisk_log_size":6622,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19202048,"num_objects":293,"num_object_clones":0,"num_object_copies":879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":184960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":91881472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":185,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613180+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:12:43.613180+0000","last_peered":"2023-10-09T04:12:43.613180+0000","last_clean":"2023-10-09T04:12:43.613180+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:12:43.613180+0000","last_undegraded":"2023-10-09T04:12:43.613180+0000","last_fullsized":"2023-10-09T04:12:43.613180+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":174,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697408+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:12:33.697408+0000","last_peered":"2023-10-09T04:12:33.697408+0000","last_clean":"2023-10-09T04:12:33.697408+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:12:33.697408+0000","last_undegraded":"2023-10-09T04:12:33.697408+0000","last_fullsized":"2023-10-09T04:12:33.697408+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"223'6768","reported_seq":9713,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:54.982840+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:12:54.982840+0000","last_peered":"2023-10-09T04:12:54.982840+0000","last_clean":"2023-10-09T04:12:54.982840+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:12:54.982840+0000","last_undegraded":"2023-10-09T04:12:54.982840+0000","last_fullsized":"2023-10-09T04:12:54.982840+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6768,"log_dups_size":0,"ondisk_log_size":6768,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":187,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625506+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:12:43.625506+0000","last_peered":"2023-10-09T04:12:43.625506+0000","last_clean":"2023-10-09T04:12:43.625506+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:12:43.625506+0000","last_undegraded":"2023-10-09T04:12:43.625506+0000","last_fullsized":"2023-10-09T04:12:43.625506+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625709+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:12:43.625709+0000","last_peered":"2023-10-09T04:12:43.625709+0000","last_clean":"2023-10-09T04:12:43.625709+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:12:43.625709+0000","last_undegraded":"2023-10-09T04:12:43.625709+0000","last_fullsized":"2023-10-09T04:12:43.625709+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"222'6631","reported_seq":6706,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:52.993914+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:12:52.993914+0000","last_peered":"2023-10-09T04:12:52.993914+0000","last_clean":"2023-10-09T04:12:52.993914+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:12:52.993914+0000","last_undegraded":"2023-10-09T04:12:52.993914+0000","last_fullsized":"2023-10-09T04:12:52.993914+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6631,"log_dups_size":0,"ondisk_log_size":6631,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5477,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":184,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.612793+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:12:43.612793+0000","last_peered":"2023-10-09T04:12:43.612793+0000","last_clean":"2023-10-09T04:12:43.612793+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:12:43.612793+0000","last_undegraded":"2023-10-09T04:12:43.612793+0000","last_fullsized":"2023-10-09T04:12:43.612793+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":158,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698182+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:12:33.698182+0000","last_peered":"2023-10-09T04:12:33.698182+0000","last_clean":"2023-10-09T04:12:33.698182+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:12:33.698182+0000","last_undegraded":"2023-10-09T04:12:33.698182+0000","last_fullsized":"2023-10-09T04:12:33.698182+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"222'6413","reported_seq":6487,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.112543+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:12:53.112543+0000","last_peered":"2023-10-09T04:12:53.112543+0000","last_clean":"2023-10-09T04:12:53.112543+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:12:53.112543+0000","last_undegraded":"2023-10-09T04:12:53.112543+0000","last_fullsized":"2023-10-09T04:12:53.112543+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6413,"log_dups_size":0,"ondisk_log_size":6413,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22904832,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5296,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672830+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:12:33.672830+0000","last_peered":"2023-10-09T04:12:33.672830+0000","last_clean":"2023-10-09T04:12:33.672830+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:12:33.672830+0000","last_undegraded":"2023-10-09T04:12:33.672830+0000","last_fullsized":"2023-10-09T04:12:33.672830+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.623663+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:12:43.623663+0000","last_peered":"2023-10-09T04:12:43.623663+0000","last_clean":"2023-10-09T04:12:43.623663+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:12:43.623663+0000","last_undegraded":"2023-10-09T04:12:43.623663+0000","last_fullsized":"2023-10-09T04:12:43.623663+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"223'6699","reported_seq":9651,"reported_epoch":223,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:55.536477+0000","last_change":"2023-10-09T04:10:39.414681+0000","last_active":"2023-10-09T04:12:55.536477+0000","last_peered":"2023-10-09T04:12:55.536477+0000","last_clean":"2023-10-09T04:10:37.123638+0000","last_became_active":"2023-10-09T04:10:39.412132+0000","last_became_peered":"2023-10-09T04:10:39.412132+0000","last_unstale":"2023-10-09T04:12:55.536477+0000","last_undegraded":"2023-10-09T04:12:55.536477+0000","last_fullsized":"2023-10-09T04:12:55.536477+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20938752,"num_objects":320,"num_object_clones":0,"num_object_copies":960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":320,"num_objects_unfound":0,"num_objects_dirty":320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5584,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":320}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.612773+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:12:43.612773+0000","last_peered":"2023-10-09T04:12:43.612773+0000","last_clean":"2023-10-09T04:12:43.612773+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:12:43.612773+0000","last_undegraded":"2023-10-09T04:12:43.612773+0000","last_fullsized":"2023-10-09T04:12:43.612773+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":177,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.681934+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:12:33.681934+0000","last_peered":"2023-10-09T04:12:33.681934+0000","last_clean":"2023-10-09T04:12:33.681934+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:12:33.681934+0000","last_undegraded":"2023-10-09T04:12:33.681934+0000","last_fullsized":"2023-10-09T04:12:33.681934+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"222'6688","reported_seq":6763,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.031019+0000","last_change":"2023-10-09T04:09:49.273058+0000","last_active":"2023-10-09T04:12:53.031019+0000","last_peered":"2023-10-09T04:12:53.031019+0000","last_clean":"2023-10-09T04:12:53.031019+0000","last_became_active":"2023-10-09T04:09:49.271803+0000","last_became_peered":"2023-10-09T04:09:49.271803+0000","last_unstale":"2023-10-09T04:12:53.031019+0000","last_undegraded":"2023-10-09T04:12:53.031019+0000","last_fullsized":"2023-10-09T04:12:53.031019+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25034752,"num_objects":382,"num_object_clones":0,"num_object_copies":1146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5526,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":113,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672660+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:12:33.672660+0000","last_peered":"2023-10-09T04:12:33.672660+0000","last_clean":"2023-10-09T04:12:33.672660+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:12:33.672660+0000","last_undegraded":"2023-10-09T04:12:33.672660+0000","last_fullsized":"2023-10-09T04:12:33.672660+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":187,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.618810+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:12:43.618810+0000","last_peered":"2023-10-09T04:12:43.618810+0000","last_clean":"2023-10-09T04:12:43.618810+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:12:43.618810+0000","last_undegraded":"2023-10-09T04:12:43.618810+0000","last_fullsized":"2023-10-09T04:12:43.618810+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"222'11009","reported_seq":11074,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.703138+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:12:53.703138+0000","last_peered":"2023-10-09T04:12:53.703138+0000","last_clean":"2023-10-09T04:12:53.703138+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:12:53.703138+0000","last_undegraded":"2023-10-09T04:12:53.703138+0000","last_fullsized":"2023-10-09T04:12:53.703138+0000","mapping_epoch":167,"log_start":"193'4000","ondisk_log_start":"193'4000","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7009,"log_dups_size":0,"ondisk_log_size":7009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11009,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":183,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613115+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:12:43.613115+0000","last_peered":"2023-10-09T04:12:43.613115+0000","last_clean":"2023-10-09T04:12:43.613115+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:12:43.613115+0000","last_undegraded":"2023-10-09T04:12:43.613115+0000","last_fullsized":"2023-10-09T04:12:43.613115+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622161+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:12:43.622161+0000","last_peered":"2023-10-09T04:12:43.622161+0000","last_clean":"2023-10-09T04:12:43.622161+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:12:43.622161+0000","last_undegraded":"2023-10-09T04:12:43.622161+0000","last_fullsized":"2023-10-09T04:12:43.622161+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"223'10922","reported_seq":10988,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:54.963754+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:12:54.963754+0000","last_peered":"2023-10-09T04:12:54.963754+0000","last_clean":"2023-10-09T04:12:54.963754+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:12:54.963754+0000","last_undegraded":"2023-10-09T04:12:54.963754+0000","last_fullsized":"2023-10-09T04:12:54.963754+0000","mapping_epoch":167,"log_start":"171'900","ondisk_log_start":"171'900","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41943040,"num_objects":640,"num_object_clones":0,"num_object_copies":1920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10922,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":184,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613128+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:12:43.613128+0000","last_peered":"2023-10-09T04:12:43.613128+0000","last_clean":"2023-10-09T04:12:43.613128+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:12:43.613128+0000","last_undegraded":"2023-10-09T04:12:43.613128+0000","last_fullsized":"2023-10-09T04:12:43.613128+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":198,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.626999+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:12:43.626999+0000","last_peered":"2023-10-09T04:12:43.626999+0000","last_clean":"2023-10-09T04:12:43.626999+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:12:43.626999+0000","last_undegraded":"2023-10-09T04:12:43.626999+0000","last_fullsized":"2023-10-09T04:12:43.626999+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"222'6725","reported_seq":6907,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.120604+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:12:53.120604+0000","last_peered":"2023-10-09T04:12:53.120604+0000","last_clean":"2023-10-09T04:12:53.120604+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:12:53.120604+0000","last_undegraded":"2023-10-09T04:12:53.120604+0000","last_fullsized":"2023-10-09T04:12:53.120604+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21823488,"num_objects":333,"num_object_clones":0,"num_object_copies":999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4339,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":166,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613069+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:12:43.613069+0000","last_peered":"2023-10-09T04:12:43.613069+0000","last_clean":"2023-10-09T04:12:43.613069+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:12:43.613069+0000","last_undegraded":"2023-10-09T04:12:43.613069+0000","last_fullsized":"2023-10-09T04:12:43.613069+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":141,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697095+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:12:33.697095+0000","last_peered":"2023-10-09T04:12:33.697095+0000","last_clean":"2023-10-09T04:12:33.697095+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:12:33.697095+0000","last_undegraded":"2023-10-09T04:12:33.697095+0000","last_fullsized":"2023-10-09T04:12:33.697095+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"223'6544","reported_seq":6624,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:54.949140+0000","last_change":"2023-10-09T04:10:37.089940+0000","last_active":"2023-10-09T04:12:54.949140+0000","last_peered":"2023-10-09T04:12:54.949140+0000","last_clean":"2023-10-09T04:12:54.949140+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:12:54.949140+0000","last_undegraded":"2023-10-09T04:12:54.949140+0000","last_fullsized":"2023-10-09T04:12:54.949140+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21495808,"num_objects":328,"num_object_clones":0,"num_object_copies":984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5404,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622349+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:12:43.622349+0000","last_peered":"2023-10-09T04:12:43.622349+0000","last_clean":"2023-10-09T04:12:43.622349+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:12:43.622349+0000","last_undegraded":"2023-10-09T04:12:43.622349+0000","last_fullsized":"2023-10-09T04:12:43.622349+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":177,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672997+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:12:33.672997+0000","last_peered":"2023-10-09T04:12:33.672997+0000","last_clean":"2023-10-09T04:12:33.672997+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:12:33.672997+0000","last_undegraded":"2023-10-09T04:12:33.672997+0000","last_fullsized":"2023-10-09T04:12:33.672997+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"222'10996","reported_seq":11061,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.075214+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:12:53.075214+0000","last_peered":"2023-10-09T04:12:53.075214+0000","last_clean":"2023-10-09T04:12:53.075214+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:12:53.075214+0000","last_undegraded":"2023-10-09T04:12:53.075214+0000","last_fullsized":"2023-10-09T04:12:53.075214+0000","mapping_epoch":167,"log_start":"188'3400","ondisk_log_start":"188'3400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10996,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":126,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672968+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:12:33.672968+0000","last_peered":"2023-10-09T04:12:33.672968+0000","last_clean":"2023-10-09T04:12:33.672968+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:12:33.672968+0000","last_undegraded":"2023-10-09T04:12:33.672968+0000","last_fullsized":"2023-10-09T04:12:33.672968+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625438+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:12:43.625438+0000","last_peered":"2023-10-09T04:12:43.625438+0000","last_clean":"2023-10-09T04:12:43.625438+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:12:43.625438+0000","last_undegraded":"2023-10-09T04:12:43.625438+0000","last_fullsized":"2023-10-09T04:12:43.625438+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"223'11006","reported_seq":15129,"reported_epoch":223,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:12:55.600941+0000","last_change":"2023-10-09T04:12:18.137975+0000","last_active":"2023-10-09T04:12:55.600941+0000","last_peered":"2023-10-09T04:12:55.600941+0000","last_clean":"2023-10-09T04:09:58.130629+0000","last_became_active":"2023-10-09T04:10:38.435437+0000","last_became_peered":"2023-10-09T04:10:38.435437+0000","last_unstale":"2023-10-09T04:12:55.600941+0000","last_undegraded":"2023-10-09T04:10:38.333280+0000","last_fullsized":"2023-10-09T04:10:38.321121+0000","mapping_epoch":197,"log_start":"171'1000","ondisk_log_start":"171'1000","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:52.163382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40108032,"num_objects":612,"num_object_clones":0,"num_object_copies":1836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":177,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11006,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2020,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":435}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625272+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:12:43.625272+0000","last_peered":"2023-10-09T04:12:43.625272+0000","last_clean":"2023-10-09T04:12:43.625272+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:12:43.625272+0000","last_undegraded":"2023-10-09T04:12:43.625272+0000","last_fullsized":"2023-10-09T04:12:43.625272+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":218,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625412+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:12:43.625412+0000","last_peered":"2023-10-09T04:12:43.625412+0000","last_clean":"2023-10-09T04:12:43.625412+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:12:43.625412+0000","last_undegraded":"2023-10-09T04:12:43.625412+0000","last_fullsized":"2023-10-09T04:12:43.625412+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"222'10958","reported_seq":11024,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.662720+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:12:53.662720+0000","last_peered":"2023-10-09T04:12:53.662720+0000","last_clean":"2023-10-09T04:12:53.662720+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:12:53.662720+0000","last_undegraded":"2023-10-09T04:12:53.662720+0000","last_fullsized":"2023-10-09T04:12:53.662720+0000","mapping_epoch":167,"log_start":"193'3900","ondisk_log_start":"193'3900","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43319320,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10958,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.626866+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:12:43.626866+0000","last_peered":"2023-10-09T04:12:43.626866+0000","last_clean":"2023-10-09T04:12:43.626866+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:12:43.626866+0000","last_undegraded":"2023-10-09T04:12:43.626866+0000","last_fullsized":"2023-10-09T04:12:43.626866+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":174,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625555+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:12:43.625555+0000","last_peered":"2023-10-09T04:12:43.625555+0000","last_clean":"2023-10-09T04:12:43.625555+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:12:43.625555+0000","last_undegraded":"2023-10-09T04:12:43.625555+0000","last_fullsized":"2023-10-09T04:12:43.625555+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"223'10831","reported_seq":10896,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:55.822099+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:12:55.822099+0000","last_peered":"2023-10-09T04:12:55.822099+0000","last_clean":"2023-10-09T04:12:55.822099+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:12:55.822099+0000","last_undegraded":"2023-10-09T04:12:55.822099+0000","last_fullsized":"2023-10-09T04:12:55.822099+0000","mapping_epoch":167,"log_start":"195'4400","ondisk_log_start":"195'4400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39256064,"num_objects":599,"num_object_clones":0,"num_object_copies":1797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10831,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":189,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.618221+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:12:43.618221+0000","last_peered":"2023-10-09T04:12:43.618221+0000","last_clean":"2023-10-09T04:12:43.618221+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:12:43.618221+0000","last_undegraded":"2023-10-09T04:12:43.618221+0000","last_fullsized":"2023-10-09T04:12:43.618221+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.696746+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:12:33.696746+0000","last_peered":"2023-10-09T04:12:33.696746+0000","last_clean":"2023-10-09T04:12:33.696746+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:12:33.696746+0000","last_undegraded":"2023-10-09T04:12:33.696746+0000","last_fullsized":"2023-10-09T04:12:33.696746+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"223'6682","reported_seq":6880,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:54.992682+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:12:54.992682+0000","last_peered":"2023-10-09T04:12:54.992682+0000","last_clean":"2023-10-09T04:12:54.992682+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:12:54.992682+0000","last_undegraded":"2023-10-09T04:12:54.992682+0000","last_fullsized":"2023-10-09T04:12:54.992682+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":6682,"log_dups_size":0,"ondisk_log_size":6682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20709376,"num_objects":316,"num_object_clones":0,"num_object_copies":948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3029,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":197,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698307+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:12:33.698307+0000","last_peered":"2023-10-09T04:12:33.698307+0000","last_clean":"2023-10-09T04:12:33.698307+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:12:33.698307+0000","last_undegraded":"2023-10-09T04:12:33.698307+0000","last_fullsized":"2023-10-09T04:12:33.698307+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":199,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.623749+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:12:43.623749+0000","last_peered":"2023-10-09T04:12:43.623749+0000","last_clean":"2023-10-09T04:12:43.623749+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:12:43.623749+0000","last_undegraded":"2023-10-09T04:12:43.623749+0000","last_fullsized":"2023-10-09T04:12:43.623749+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":159,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697265+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:12:33.697265+0000","last_peered":"2023-10-09T04:12:33.697265+0000","last_clean":"2023-10-09T04:12:33.697265+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:12:33.697265+0000","last_undegraded":"2023-10-09T04:12:33.697265+0000","last_fullsized":"2023-10-09T04:12:33.697265+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"222'6717","reported_seq":6791,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.111071+0000","last_change":"2023-10-09T04:09:49.253721+0000","last_active":"2023-10-09T04:12:53.111071+0000","last_peered":"2023-10-09T04:12:53.111071+0000","last_clean":"2023-10-09T04:12:53.111071+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:12:53.111071+0000","last_undegraded":"2023-10-09T04:12:53.111071+0000","last_fullsized":"2023-10-09T04:12:53.111071+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6717,"log_dups_size":0,"ondisk_log_size":6717,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":187072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":186,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622315+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:12:43.622315+0000","last_peered":"2023-10-09T04:12:43.622315+0000","last_clean":"2023-10-09T04:12:43.622315+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:12:43.622315+0000","last_undegraded":"2023-10-09T04:12:43.622315+0000","last_fullsized":"2023-10-09T04:12:43.622315+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625784+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:12:43.625784+0000","last_peered":"2023-10-09T04:12:43.625784+0000","last_clean":"2023-10-09T04:12:43.625784+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:12:43.625784+0000","last_undegraded":"2023-10-09T04:12:43.625784+0000","last_fullsized":"2023-10-09T04:12:43.625784+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698245+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:12:33.698245+0000","last_peered":"2023-10-09T04:12:33.698245+0000","last_clean":"2023-10-09T04:12:33.698245+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:12:33.698245+0000","last_undegraded":"2023-10-09T04:12:33.698245+0000","last_fullsized":"2023-10-09T04:12:33.698245+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"222'6712","reported_seq":6787,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.113260+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:12:53.113260+0000","last_peered":"2023-10-09T04:12:53.113260+0000","last_clean":"2023-10-09T04:12:53.113260+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:12:53.113260+0000","last_undegraded":"2023-10-09T04:12:53.113260+0000","last_fullsized":"2023-10-09T04:12:53.113260+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21889024,"num_objects":334,"num_object_clones":0,"num_object_copies":1002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5550,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":37,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:45.085505+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:12:45.085505+0000","last_peered":"2023-10-09T04:12:45.085505+0000","last_clean":"2023-10-09T04:12:45.085505+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:12:45.085505+0000","last_undegraded":"2023-10-09T04:12:45.085505+0000","last_fullsized":"2023-10-09T04:12:45.085505+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":192,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697488+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:12:33.697488+0000","last_peered":"2023-10-09T04:12:33.697488+0000","last_clean":"2023-10-09T04:12:33.697488+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:12:33.697488+0000","last_undegraded":"2023-10-09T04:12:33.697488+0000","last_fullsized":"2023-10-09T04:12:33.697488+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.696814+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:12:33.696814+0000","last_peered":"2023-10-09T04:12:33.696814+0000","last_clean":"2023-10-09T04:12:33.696814+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:12:33.696814+0000","last_undegraded":"2023-10-09T04:12:33.696814+0000","last_fullsized":"2023-10-09T04:12:33.696814+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"222'6498","reported_seq":6572,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:52.983105+0000","last_change":"2023-10-09T04:09:49.258962+0000","last_active":"2023-10-09T04:12:52.983105+0000","last_peered":"2023-10-09T04:12:52.983105+0000","last_clean":"2023-10-09T04:12:52.983105+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:12:52.983105+0000","last_undegraded":"2023-10-09T04:12:52.983105+0000","last_fullsized":"2023-10-09T04:12:52.983105+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6498,"log_dups_size":0,"ondisk_log_size":6498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22970368,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5380,"num_write_kb":183392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":206,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622347+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:12:43.622347+0000","last_peered":"2023-10-09T04:12:43.622347+0000","last_clean":"2023-10-09T04:12:43.622347+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:12:43.622347+0000","last_undegraded":"2023-10-09T04:12:43.622347+0000","last_fullsized":"2023-10-09T04:12:43.622347+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":563,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.612324+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:12:43.612324+0000","last_peered":"2023-10-09T04:12:43.612324+0000","last_clean":"2023-10-09T04:12:43.612324+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:12:43.612324+0000","last_undegraded":"2023-10-09T04:12:43.612324+0000","last_fullsized":"2023-10-09T04:12:43.612324+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622560+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:12:43.622560+0000","last_peered":"2023-10-09T04:12:43.622560+0000","last_clean":"2023-10-09T04:12:43.622560+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:12:43.622560+0000","last_undegraded":"2023-10-09T04:12:43.622560+0000","last_fullsized":"2023-10-09T04:12:43.622560+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625549+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:12:43.625549+0000","last_peered":"2023-10-09T04:12:43.625549+0000","last_clean":"2023-10-09T04:12:43.625549+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:12:43.625549+0000","last_undegraded":"2023-10-09T04:12:43.625549+0000","last_fullsized":"2023-10-09T04:12:43.625549+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"223'6613","reported_seq":8982,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:55.529810+0000","last_change":"2023-10-09T04:12:37.913403+0000","last_active":"2023-10-09T04:12:55.529810+0000","last_peered":"2023-10-09T04:12:55.529810+0000","last_clean":"2023-10-09T04:12:55.529810+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:12:55.529810+0000","last_undegraded":"2023-10-09T04:12:55.529810+0000","last_fullsized":"2023-10-09T04:12:55.529810+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18186240,"num_objects":277,"num_object_clones":0,"num_object_copies":831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5497,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":203,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.612289+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:12:43.612289+0000","last_peered":"2023-10-09T04:12:43.612289+0000","last_clean":"2023-10-09T04:12:43.612289+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:12:43.612289+0000","last_undegraded":"2023-10-09T04:12:43.612289+0000","last_fullsized":"2023-10-09T04:12:43.612289+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":178,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672819+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:12:33.672819+0000","last_peered":"2023-10-09T04:12:33.672819+0000","last_clean":"2023-10-09T04:12:33.672819+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:12:33.672819+0000","last_undegraded":"2023-10-09T04:12:33.672819+0000","last_fullsized":"2023-10-09T04:12:33.672819+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":211,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.638190+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:12:43.638190+0000","last_peered":"2023-10-09T04:12:43.638190+0000","last_clean":"2023-10-09T04:12:43.638190+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:12:43.638190+0000","last_undegraded":"2023-10-09T04:12:43.638190+0000","last_fullsized":"2023-10-09T04:12:43.638190+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"223'6839","reported_seq":6977,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:55.522960+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:12:55.522960+0000","last_peered":"2023-10-09T04:12:55.522960+0000","last_clean":"2023-10-09T04:12:55.522960+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:12:55.522960+0000","last_undegraded":"2023-10-09T04:12:55.522960+0000","last_fullsized":"2023-10-09T04:12:55.522960+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21364736,"num_objects":326,"num_object_clones":0,"num_object_copies":978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4292,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":186,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613156+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:12:43.613156+0000","last_peered":"2023-10-09T04:12:43.613156+0000","last_clean":"2023-10-09T04:12:43.613156+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:12:43.613156+0000","last_undegraded":"2023-10-09T04:12:43.613156+0000","last_fullsized":"2023-10-09T04:12:43.613156+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":191,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.698385+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:12:33.698385+0000","last_peered":"2023-10-09T04:12:33.698385+0000","last_clean":"2023-10-09T04:12:33.698385+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:12:33.698385+0000","last_undegraded":"2023-10-09T04:12:33.698385+0000","last_fullsized":"2023-10-09T04:12:33.698385+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":163,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.672891+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:12:33.672891+0000","last_peered":"2023-10-09T04:12:33.672891+0000","last_clean":"2023-10-09T04:12:33.672891+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:12:33.672891+0000","last_undegraded":"2023-10-09T04:12:33.672891+0000","last_fullsized":"2023-10-09T04:12:33.672891+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"223'6865","reported_seq":9935,"reported_epoch":223,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:12:55.705654+0000","last_change":"2023-10-09T04:10:39.421844+0000","last_active":"2023-10-09T04:12:55.705654+0000","last_peered":"2023-10-09T04:12:55.705654+0000","last_clean":"2023-10-09T04:10:37.105400+0000","last_became_active":"2023-10-09T04:10:39.417429+0000","last_became_peered":"2023-10-09T04:10:39.417429+0000","last_unstale":"2023-10-09T04:12:55.705654+0000","last_undegraded":"2023-10-09T04:12:55.705654+0000","last_fullsized":"2023-10-09T04:12:55.705654+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6865,"log_dups_size":0,"ondisk_log_size":6865,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18481152,"num_objects":282,"num_object_clones":0,"num_object_copies":846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":282,"num_objects_unfound":0,"num_objects_dirty":282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5694,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1483,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4,6","objects":282}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":127,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613231+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:12:43.613231+0000","last_peered":"2023-10-09T04:12:43.613231+0000","last_clean":"2023-10-09T04:12:43.613231+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:12:43.613231+0000","last_undegraded":"2023-10-09T04:12:43.613231+0000","last_fullsized":"2023-10-09T04:12:43.613231+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":158,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.697700+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:12:33.697700+0000","last_peered":"2023-10-09T04:12:33.697700+0000","last_clean":"2023-10-09T04:12:33.697700+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:12:33.697700+0000","last_undegraded":"2023-10-09T04:12:33.697700+0000","last_fullsized":"2023-10-09T04:12:33.697700+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":153,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.612681+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:12:43.612681+0000","last_peered":"2023-10-09T04:12:43.612681+0000","last_clean":"2023-10-09T04:12:43.612681+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:12:43.612681+0000","last_undegraded":"2023-10-09T04:12:43.612681+0000","last_fullsized":"2023-10-09T04:12:43.612681+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"222'6685","reported_seq":8903,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.694711+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:12:53.694711+0000","last_peered":"2023-10-09T04:12:53.694711+0000","last_clean":"2023-10-09T04:12:53.694711+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:12:53.694711+0000","last_undegraded":"2023-10-09T04:12:53.694711+0000","last_fullsized":"2023-10-09T04:12:53.694711+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19791872,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4208,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":151,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.613296+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:12:43.613296+0000","last_peered":"2023-10-09T04:12:43.613296+0000","last_clean":"2023-10-09T04:12:43.613296+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:12:43.613296+0000","last_undegraded":"2023-10-09T04:12:43.613296+0000","last_fullsized":"2023-10-09T04:12:43.613296+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.638225+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:12:43.638225+0000","last_peered":"2023-10-09T04:12:43.638225+0000","last_clean":"2023-10-09T04:12:43.638225+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:12:43.638225+0000","last_undegraded":"2023-10-09T04:12:43.638225+0000","last_fullsized":"2023-10-09T04:12:43.638225+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.626916+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:12:43.626916+0000","last_peered":"2023-10-09T04:12:43.626916+0000","last_clean":"2023-10-09T04:12:43.626916+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:12:43.626916+0000","last_undegraded":"2023-10-09T04:12:43.626916+0000","last_fullsized":"2023-10-09T04:12:43.626916+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":166,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.673049+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:12:33.673049+0000","last_peered":"2023-10-09T04:12:33.673049+0000","last_clean":"2023-10-09T04:12:33.673049+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:12:33.673049+0000","last_undegraded":"2023-10-09T04:12:33.673049+0000","last_fullsized":"2023-10-09T04:12:33.673049+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":36,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.615658+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:12:43.615658+0000","last_peered":"2023-10-09T04:12:43.615658+0000","last_clean":"2023-10-09T04:12:43.615658+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:12:43.615658+0000","last_undegraded":"2023-10-09T04:12:43.615658+0000","last_fullsized":"2023-10-09T04:12:43.615658+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"223'6671","reported_seq":6749,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-10-09T04:12:55.035013+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:12:55.035013+0000","last_peered":"2023-10-09T04:12:55.035013+0000","last_clean":"2023-10-09T04:12:55.035013+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:12:55.035013+0000","last_undegraded":"2023-10-09T04:12:55.035013+0000","last_fullsized":"2023-10-09T04:12:55.035013+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6671,"log_dups_size":0,"ondisk_log_size":6671,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18808832,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":184,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.622415+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:12:43.622415+0000","last_peered":"2023-10-09T04:12:43.622415+0000","last_clean":"2023-10-09T04:12:43.622415+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:12:43.622415+0000","last_undegraded":"2023-10-09T04:12:43.622415+0000","last_fullsized":"2023-10-09T04:12:43.622415+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625853+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:12:43.625853+0000","last_peered":"2023-10-09T04:12:43.625853+0000","last_clean":"2023-10-09T04:12:43.625853+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:12:43.625853+0000","last_undegraded":"2023-10-09T04:12:43.625853+0000","last_fullsized":"2023-10-09T04:12:43.625853+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"222'6728","reported_seq":6800,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:53.082305+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:12:53.082305+0000","last_peered":"2023-10-09T04:12:53.082305+0000","last_clean":"2023-10-09T04:12:53.082305+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:12:53.082305+0000","last_undegraded":"2023-10-09T04:12:53.082305+0000","last_fullsized":"2023-10-09T04:12:53.082305+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22151168,"num_objects":338,"num_object_clones":0,"num_object_copies":1014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5566,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":167,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.623840+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:12:43.623840+0000","last_peered":"2023-10-09T04:12:43.623840+0000","last_clean":"2023-10-09T04:12:43.623840+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:12:43.623840+0000","last_undegraded":"2023-10-09T04:12:43.623840+0000","last_fullsized":"2023-10-09T04:12:43.623840+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":198,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625268+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:12:43.625268+0000","last_peered":"2023-10-09T04:12:43.625268+0000","last_clean":"2023-10-09T04:12:43.625268+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:12:43.625268+0000","last_undegraded":"2023-10-09T04:12:43.625268+0000","last_fullsized":"2023-10-09T04:12:43.625268+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":37,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.615610+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:12:43.615610+0000","last_peered":"2023-10-09T04:12:43.615610+0000","last_clean":"2023-10-09T04:12:43.615610+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:12:43.615610+0000","last_undegraded":"2023-10-09T04:12:43.615610+0000","last_fullsized":"2023-10-09T04:12:43.615610+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":36,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.615465+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:12:43.615465+0000","last_peered":"2023-10-09T04:12:43.615465+0000","last_clean":"2023-10-09T04:12:43.615465+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:12:43.615465+0000","last_undegraded":"2023-10-09T04:12:43.615465+0000","last_fullsized":"2023-10-09T04:12:43.615465+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":212,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-10-09T04:12:33.673529+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:12:33.673529+0000","last_peered":"2023-10-09T04:12:33.673529+0000","last_clean":"2023-10-09T04:12:33.673529+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:12:33.673529+0000","last_undegraded":"2023-10-09T04:12:33.673529+0000","last_fullsized":"2023-10-09T04:12:33.673529+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-10-09T04:12:43.625057+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:12:43.625057+0000","last_peered":"2023-10-09T04:12:43.625057+0000","last_clean":"2023-10-09T04:12:43.625057+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:12:43.625057+0000","last_undegraded":"2023-10-09T04:12:43.625057+0000","last_fullsized":"2023-10-09T04:12:43.625057+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":674824216,"num_objects":10298,"num_object_clones":0,"num_object_copies":30894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":177,"num_objects_misplaced":907,"num_objects_unfound":0,"num_objects_dirty":10298,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169799,"num_write_kb":5558721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13895,"num_bytes_recovered":870776832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2450206720,"data_stored":2450194504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181633,"ondisk_log_size":181633,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201525,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":570636,"kb_used_data":269252,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93174516,"statfs":{"total":95995035648,"available":95410704384,"internally_reserved":0,"allocated":275714048,"data_stored":275035459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":4.7140000000000004,"5min":4.0140000000000002,"15min":1.863},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":62.140000000000001,"5min":126.752,"15min":126.752},"last":0.33800000000000002},{"interface":"front","average":{"1min":4.5869999999999997,"5min":4.194,"15min":1.8779999999999999},"min":{"1min":0.308,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":62.340000000000003,"5min":128.88399999999999,"15min":128.88399999999999},"last":7.2160000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.0430000000000001,"5min":6.6180000000000003,"15min":2.8260000000000001},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":101.471,"5min":245.28700000000001,"15min":245.28700000000001},"last":11.081},{"interface":"front","average":{"1min":6.7069999999999999,"5min":6.3150000000000004,"15min":2.7799999999999998},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":98.527000000000001,"5min":244.626,"15min":244.626},"last":0.33100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.776,"15min":1.3200000000000001},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":4.351,"5min":13.887,"15min":58.615000000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.97,"15min":1.77},"min":{"1min":0.38600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.2320000000000002,"5min":13.872,"15min":99.372},"last":0.46400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:12:49 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":1.7649999999999999,"15min":1.7370000000000001},"min":{"1min":0.34000000000000002,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":35.771000000000001,"5min":35.771000000000001,"15min":99.134},"last":4.0140000000000002},{"interface":"front","average":{"1min":3.6019999999999999,"5min":1.9419999999999999,"15min":1.839},"min":{"1min":0.34599999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":35.848999999999997,"5min":35.848999999999997,"15min":99.117999999999995},"last":4.0060000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.0169999999999999,"15min":0.69799999999999995},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.98399999999999999,"5min":16.195,"15min":16.195},"last":0.252},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97299999999999998,"15min":0.71799999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.343,"5min":12.442,"15min":12.442},"last":0.28699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68700000000000006,"15min":0.61499999999999999},"min":{"1min":0.29299999999999998,"5min":0.27700000000000002,"15min":0.27400000000000002},"max":{"1min":4.4829999999999997,"5min":4.4829999999999997,"15min":6.5439999999999996},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76000000000000001,"15min":0.67800000000000005},"min":{"1min":0.33600000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83699999999999997,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":0.31900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.80300000000000005,"15min":0.69899999999999995},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.5199999999999996,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.92600000000000005,"15min":0.75900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":2.399,"5min":5.6420000000000003,"15min":5.9039999999999999},"last":0.34899999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463162,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":562304,"kb_used_data":265016,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93182848,"statfs":{"total":95995035648,"available":95419236352,"internally_reserved":0,"allocated":271376384,"data_stored":270709638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":6.4000000000000004,"5min":4.4550000000000001,"15min":2.2370000000000001},"min":{"1min":0.222,"5min":0.217,"15min":0.217},"max":{"1min":77.453000000000003,"5min":77.453000000000003,"15min":78.983999999999995},"last":0.376},{"interface":"front","average":{"1min":6.9790000000000001,"5min":4.9480000000000004,"15min":2.5129999999999999},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":77.430999999999997,"5min":77.430999999999997,"15min":78.998000000000005},"last":1.4370000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:12:54 2023","interfaces":[{"interface":"back","average":{"1min":4.2729999999999997,"5min":3.2250000000000001,"15min":1.9450000000000001},"min":{"1min":0.25800000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":53.337000000000003,"5min":62.698,"15min":62.698},"last":4.4870000000000001},{"interface":"front","average":{"1min":4.0529999999999999,"5min":3.4670000000000001,"15min":2.0310000000000001},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":53.146000000000001,"5min":63.530999999999999,"15min":63.530999999999999},"last":0.58799999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":3.7120000000000002,"15min":2.0350000000000001},"min":{"1min":0.33700000000000002,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":33.345999999999997,"5min":77.953000000000003,"15min":77.953000000000003},"last":2.512},{"interface":"front","average":{"1min":3.5680000000000001,"5min":3.7610000000000001,"15min":2.085},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":33.368000000000002,"5min":78.019000000000005,"15min":78.019000000000005},"last":2.5569999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5570000000000004,"5min":4.157,"15min":2.2970000000000002},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":51.082000000000001,"5min":57.509999999999998,"15min":57.509999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":4.556,"5min":3.9620000000000002,"15min":2.3119999999999998},"min":{"1min":0.32600000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":51.031999999999996,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.5}]},{"osd":4,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.3720000000000001,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.434},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.72199999999999998,"15min":0.64400000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2970000000000002,"5min":7.7000000000000002,"15min":7.7000000000000002},"last":0.45600000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":0.84599999999999997,"15min":0.84899999999999998},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.081,"5min":0.97399999999999998,"15min":0.95499999999999996},"min":{"1min":0.34399999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":23.006,"5min":23.006,"15min":54.853999999999999},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68600000000000005,"15min":0.628},"min":{"1min":0.317,"5min":0.313,"15min":0.313},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.79000000000000004,"15min":0.71199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":3.3679999999999999,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.53700000000000003}]}]},{"osd":1,"up_from":89,"seq":382252089470,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":632312,"kb_used_data":377776,"kb_used_omap":9,"kb_used_meta":254390,"kb_avail":93112840,"statfs":{"total":95995035648,"available":95347548160,"internally_reserved":0,"allocated":386842624,"data_stored":385940841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":260496361},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":8.9399999999999995,"5min":4.9530000000000003,"15min":2.4140000000000001},"min":{"1min":0.318,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":68.048000000000002,"5min":85.977999999999994,"15min":85.977999999999994},"last":0.46800000000000003},{"interface":"front","average":{"1min":6.0739999999999998,"5min":4.7610000000000001,"15min":2.73},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":25.620999999999999,"5min":85.998999999999995,"15min":99.831999999999994},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.9359999999999999,"15min":1.7589999999999999},"min":{"1min":0.26100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":13.695,"5min":77.563000000000002,"15min":121.54600000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":4.9939999999999998,"5min":3.452,"15min":2.0110000000000001},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.186},"max":{"1min":15.298999999999999,"5min":77.504999999999995,"15min":99.899000000000001},"last":0.52200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":9.3320000000000007,"5min":5.5259999999999998,"15min":2.7280000000000002},"min":{"1min":0.309,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":50.542999999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":40.634999999999998},{"interface":"front","average":{"1min":9.9339999999999993,"5min":6.0990000000000002,"15min":3.2669999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":44.716000000000001,"5min":138.49799999999999,"15min":138.49799999999999},"last":44.210999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.214,"15min":1.6879999999999999},"min":{"1min":0.29099999999999998,"5min":0.254,"15min":0.254},"max":{"1min":11.973000000000001,"5min":15.215999999999999,"15min":99.989999999999995},"last":0.40500000000000003},{"interface":"front","average":{"1min":4.8760000000000003,"5min":1.972,"15min":1.657},"min":{"1min":0.42999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":25.957999999999998,"5min":25.957999999999998,"15min":100.249},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":2.113,"15min":1.663},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.013999999999999,"5min":15.266,"15min":99.933999999999997},"last":0.44900000000000001},{"interface":"front","average":{"1min":4.8319999999999999,"5min":2.2040000000000002,"15min":1.768},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":100.34099999999999},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":4.7990000000000004,"5min":2.2719999999999998,"15min":1.8600000000000001},"min":{"1min":0.45300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":25.931000000000001,"5min":25.931000000000001,"15min":100.309},"last":0.53100000000000003},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.72,"15min":1.6779999999999999},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.446,"5min":13.446,"15min":100.143},"last":0.63100000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":4.9329999999999998,"5min":2.3370000000000002,"15min":1.8480000000000001},"min":{"1min":0.53300000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":25.984999999999999,"5min":25.984999999999999,"15min":100.214},"last":0.57699999999999996},{"interface":"front","average":{"1min":3.0899999999999999,"5min":1.9970000000000001,"15min":1.698},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.317},"max":{"1min":13.471,"5min":13.471,"15min":100.113},"last":0.438}]}]},{"osd":0,"up_from":84,"seq":360777252996,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642196,"kb_used_data":390156,"kb_used_omap":28,"kb_used_meta":252003,"kb_avail":93102956,"statfs":{"total":95995035648,"available":95337426944,"internally_reserved":0,"allocated":399519744,"data_stored":398589443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":258051920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":5.2050000000000001,"5min":3.0289999999999999,"15min":2.3050000000000002},"min":{"1min":0.20499999999999999,"5min":0.184,"15min":0.17000000000000001},"max":{"1min":37.764000000000003,"5min":44.287999999999997,"15min":118.48},"last":5.258},{"interface":"front","average":{"1min":5.718,"5min":3.5859999999999999,"15min":2.5649999999999999},"min":{"1min":0.28000000000000003,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":38.128999999999998,"5min":43.906999999999996,"15min":118.384},"last":5.7619999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:12:21 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":3.0339999999999998,"15min":2.2490000000000001},"min":{"1min":0.38,"5min":0.221,"15min":0.221},"max":{"1min":13.108000000000001,"5min":25.055,"15min":118.54000000000001},"last":4.3520000000000003},{"interface":"front","average":{"1min":4.5979999999999999,"5min":3.2890000000000001,"15min":2.367},"min":{"1min":0.22900000000000001,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":12.372,"5min":44.436999999999998,"15min":118.727},"last":4.1440000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.6109999999999998,"5min":5.0629999999999997,"15min":3.8820000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":18.196000000000002,"5min":156.488,"15min":156.488},"last":6.1630000000000003},{"interface":"front","average":{"1min":3.944,"5min":5.0300000000000002,"15min":3.923},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":12.465999999999999,"5min":156.50700000000001,"15min":156.50700000000001},"last":1.8700000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.0219999999999998,"15min":1.8080000000000001},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7270000000000003,"5min":10.023,"15min":118.633},"last":1.673},{"interface":"front","average":{"1min":2.714,"5min":2.4769999999999999,"15min":2.0379999999999998},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.492000000000001,"5min":44.475000000000001,"15min":118.913},"last":1.5800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.6360000000000001,"15min":2.552},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":10.445,"5min":44.494,"15min":182.68899999999999},"last":1.5329999999999999},{"interface":"front","average":{"1min":1.887,"5min":2.2210000000000001,"15min":2.387},"min":{"1min":0.41299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.4749999999999996,"5min":44.328000000000003,"15min":182.66300000000001},"last":4.3879999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":3.9180000000000001,"15min":2.3010000000000002},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.5090000000000003,"5min":137.36099999999999,"15min":137.36099999999999},"last":4.4539999999999997},{"interface":"front","average":{"1min":3.649,"5min":3.9220000000000002,"15min":2.2679999999999998},"min":{"1min":0.44500000000000001,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":12.496,"5min":136.88999999999999,"15min":136.88999999999999},"last":1.625}]},{"osd":7,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":3.3410000000000002,"15min":2.9169999999999998},"min":{"1min":0.38700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.5500000000000007,"5min":121.324,"15min":185.024},"last":4.4249999999999998},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.3239999999999998,"15min":2.508},"min":{"1min":0.46700000000000003,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":12.417,"5min":12.417,"15min":184.98599999999999},"last":1.647}]}]},{"osd":2,"up_from":94,"seq":403726925944,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":693540,"kb_used_data":364060,"kb_used_omap":81,"kb_used_meta":329390,"kb_avail":93051612,"statfs":{"total":95995035648,"available":95284850688,"internally_reserved":0,"allocated":372797440,"data_stored":371913504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":337295578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2460000000000004,"5min":2.9359999999999999,"15min":1.4690000000000001},"min":{"1min":0.20200000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":22.321000000000002,"5min":22.321000000000002,"15min":22.321000000000002},"last":46.750999999999998},{"interface":"front","average":{"1min":3.6800000000000002,"5min":3.581,"15min":1.738},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.6440000000000001,"5min":45.979999999999997,"15min":45.979999999999997},"last":48.350999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":3.5910000000000002,"15min":1.9730000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":9.9429999999999996,"5min":53.695,"15min":68.009},"last":0.30199999999999999},{"interface":"front","average":{"1min":3.7149999999999999,"5min":3.7090000000000001,"15min":2.1419999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.346,"5min":53.439,"15min":66.975999999999999},"last":0.25}]},{"osd":3,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":2.4380000000000002,"15min":1.5700000000000001},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":20.242999999999999,"5min":20.242999999999999,"15min":27.562000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":5.0439999999999996,"5min":3.2469999999999999,"15min":1.863},"min":{"1min":0.318,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":17.076000000000001,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.27400000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.2490000000000001,"15min":1.444},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5279999999999996,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.609,"5min":1.9419999999999999,"15min":1.234},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":20.134,"5min":20.134,"15min":20.134},"last":0.39600000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.6789999999999998,"15min":1.4770000000000001},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":7.3399999999999999,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.44},{"interface":"front","average":{"1min":2.1779999999999999,"5min":2.6339999999999999,"15min":1.6020000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5039999999999996,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6579999999999999,"5min":3.2829999999999999,"15min":1.671},"min":{"1min":0.376,"5min":0.311,"15min":0.311},"max":{"1min":20.184000000000001,"5min":114.14,"15min":114.14},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.194,"5min":3.7719999999999998,"15min":1.8560000000000001},"min":{"1min":0.43099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4199999999999999,"5min":114.123,"15min":114.123},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.556,"15min":1.8640000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":20.215,"5min":117.32599999999999,"15min":117.32599999999999},"last":0.375},{"interface":"front","average":{"1min":2.6349999999999998,"5min":3.9209999999999998,"15min":1.982},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.4659999999999993,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.46999999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762418,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927836,"kb_used_data":503252,"kb_used_omap":7,"kb_used_meta":424568,"kb_avail":92817316,"statfs":{"total":95995035648,"available":95044931584,"internally_reserved":0,"allocated":515330048,"data_stored":514459404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":434758248},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":6.2770000000000001,"5min":5.2089999999999996,"15min":3.71},"min":{"1min":0.214,"5min":0.19400000000000001,"15min":0.185},"max":{"1min":24.077999999999999,"5min":100.512,"15min":106.22},"last":3.6619999999999999},{"interface":"front","average":{"1min":7.8559999999999999,"5min":5.694,"15min":3.8660000000000001},"min":{"1min":0.22800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.091999999999999,"5min":101.733,"15min":106.986},"last":9.2699999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8450000000000002,"5min":3.819,"15min":2.484},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":19.262,"5min":57.055999999999997,"15min":106.121},"last":3.1059999999999999},{"interface":"front","average":{"1min":3.8180000000000001,"5min":4.1529999999999996,"15min":2.3330000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":16.646000000000001,"5min":57.088999999999999,"15min":65.015000000000001},"last":8.5969999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":2.9289999999999998,"15min":2.367},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":10.333,"5min":23.405999999999999,"15min":106.105},"last":5.7880000000000003},{"interface":"front","average":{"1min":4.3140000000000001,"5min":3.7370000000000001,"15min":2.782},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":13.201000000000001,"5min":98.765000000000001,"15min":106.70399999999999},"last":0.27600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7400000000000002,"15min":2.0910000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.475,"5min":37.652000000000001,"15min":106.85899999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.4060000000000001,"5min":2.4580000000000002,"15min":1.4710000000000001},"min":{"1min":0.38700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.255000000000001,"5min":37.442,"15min":37.442},"last":2.98}]},{"osd":5,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.0329999999999999,"15min":1.381},"min":{"1min":0.36899999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.285,"5min":10.285,"15min":12.819000000000001},"last":3.0339999999999998},{"interface":"front","average":{"1min":2.8610000000000002,"5min":2.0510000000000002,"15min":1.6319999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.968,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.52600000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.9399999999999999,"15min":1.9910000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":9.9290000000000003,"5min":13.255000000000001,"15min":106.036},"last":0.48599999999999999},{"interface":"front","average":{"1min":3.3290000000000002,"5min":2.371,"15min":1.96},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":11.425000000000001,"5min":15.622,"15min":106.741},"last":0.36399999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":2.3919999999999999,"15min":2.0030000000000001},"min":{"1min":0.34599999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.506,"5min":15.685,"15min":106.82599999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.9609999999999999,"5min":2.214,"15min":1.859},"min":{"1min":0.42599999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":10.212999999999999,"5min":10.212999999999999,"15min":106.084},"last":3.0710000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986434,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":490368,"kb_used_data":180792,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93254784,"statfs":{"total":95995035648,"available":95492898816,"internally_reserved":0,"allocated":185131008,"data_stored":184463828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7560000000000002,"5min":3.3079999999999998,"15min":1.631},"min":{"1min":0.222,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":38.831000000000003,"5min":63.128,"15min":63.128},"last":1.8700000000000001},{"interface":"front","average":{"1min":4.7939999999999996,"5min":3.0590000000000002,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.039999999999999,"5min":62.645000000000003,"15min":62.645000000000003},"last":1.524}]},{"osd":1,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.7709999999999999,"15min":1.4410000000000001},"min":{"1min":0.25700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":23.157,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.8410000000000002,"5min":2.8479999999999999,"15min":1.4299999999999999},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":23.091999999999999,"5min":75.314999999999998,"15min":75.314999999999998},"last":1.24}]},{"osd":2,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.5720000000000001,"15min":1.179},"min":{"1min":0.255,"5min":0.255,"15min":0.245},"max":{"1min":5.4470000000000001,"5min":8.8030000000000008,"15min":19.356000000000002},"last":2.109},{"interface":"front","average":{"1min":1.881,"5min":1.6299999999999999,"15min":1.1299999999999999},"min":{"1min":0.38700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.4610000000000003,"5min":8.5920000000000005,"15min":19.324000000000002},"last":2.1259999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.8779999999999999,"15min":1.1000000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4880000000000004,"5min":14.694000000000001,"15min":14.694000000000001},"last":6.431},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.958,"15min":1.222},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.2110000000000003,"5min":10.476000000000001,"15min":10.476000000000001},"last":6.306}]},{"osd":5,"last update":"Mon Oct 9 04:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55000000000000004,"15min":0.58099999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3050000000000002,"5min":3.4510000000000001,"15min":8.5259999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.956,"5min":5.9569999999999999,"15min":8.3149999999999995},"last":0.40600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.93799999999999994,"15min":0.74099999999999999},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":2.9990000000000001,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.81899999999999995,"15min":0.66600000000000004},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9700000000000002,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.70299999999999996,"15min":0.70699999999999996},"min":{"1min":0.311,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":2.3599999999999999,"5min":3.5190000000000001,"15min":9.0050000000000008},"last":0.438},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.79100000000000004,"15min":0.77200000000000002},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.29199999999999998},"max":{"1min":2.214,"5min":5.9939999999999998,"15min":8.5440000000000005},"last":0.44600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724798,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":451120,"kb_used_data":141544,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93294032,"statfs":{"total":95995035648,"available":95533088768,"internally_reserved":0,"allocated":144941056,"data_stored":144285946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":24.518000000000001,"5min":7.5119999999999996,"15min":3.1709999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":151.23599999999999,"5min":151.23599999999999,"15min":151.23599999999999},"last":21.120999999999999},{"interface":"front","average":{"1min":25.135000000000002,"5min":7.8940000000000001,"15min":3.3359999999999999},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":151.566,"5min":151.566,"15min":151.566},"last":21.265999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:12:50 2023","interfaces":[{"interface":"back","average":{"1min":7.6600000000000001,"5min":3.6869999999999998,"15min":2.177},"min":{"1min":0.26100000000000001,"5min":0.219,"15min":0.219},"max":{"1min":58.408000000000001,"5min":59.588999999999999,"15min":91.212999999999994},"last":72.734999999999999},{"interface":"front","average":{"1min":7.4909999999999997,"5min":3.8450000000000002,"15min":2.3740000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":58.393999999999998,"5min":59.533000000000001,"15min":91.254000000000005},"last":72.762}]},{"osd":2,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":5.7990000000000004,"5min":3.677,"15min":1.8720000000000001},"min":{"1min":0.29699999999999999,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":67.685000000000002,"5min":67.685000000000002,"15min":67.685000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":6.0250000000000004,"5min":3.96,"15min":1.8839999999999999},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.596000000000004,"5min":67.596000000000004,"15min":67.596000000000004},"last":0.45300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":1.756,"15min":1.885},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":26.645,"5min":26.645,"15min":91.834999999999994},"last":0.497},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.9390000000000001,"15min":1.9279999999999999},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.309},"max":{"1min":26.536000000000001,"5min":26.536000000000001,"15min":91.802000000000007},"last":0.51600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.75600000000000001,"15min":0.65300000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.64600000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.371,"5min":0.78200000000000003,"15min":0.70799999999999996},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.51700000000000002,"5min":5.29,"15min":6.7709999999999999},"last":0.54900000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":2.0409999999999999,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":1.407,"5min":123.239,"15min":123.239},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.0139999999999998,"15min":1.1319999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0289999999999999,"5min":123.014,"15min":123.014},"last":0.59999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.877,"15min":0.77500000000000002},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":0.64100000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.747,"15min":0.79800000000000004},"min":{"1min":0.35599999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.73499999999999999,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.625}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":383651840,"data_stored":383647768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":366219264,"data_stored":366215192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":361037824,"data_stored":361037824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":493879296,"data_stored":493879296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":182190080,"data_stored":182190080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":141950976,"data_stored":141950976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":260706304,"data_stored":260702232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":260571136,"data_stored":260571136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:12:59.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:12:59.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:12:59.733 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:12:59.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:12:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:12:59.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:00.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:00.037 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:00.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:00.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:00.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:00.214 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:13:00.214 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:13:00.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:13:00.214 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49817 6.c 6:3ffecb2a:::benchmark_data_smithi163_287034_object49816:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 453.52385973899999, 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21672543999999999, 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:00.215 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49817 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687659+0000", 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687682+0000", 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3472000000000002e-05 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687694+0000", 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1080999999999999e-05 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687701+0000", 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9000000000000006e-06 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687750+0000", 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8877e-05 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687874+0000", 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123812 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.688008+0000", 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134256 2023-10-09T04:13:00.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.690036+0000", 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020273660000000001 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692667+0000", 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026314289999999998 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904254+0000", 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21158665099999999 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904341+0000", 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7026000000000003e-05 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904386+0000", 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4781999999999999e-05 2023-10-09T04:13:00.221 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49825 6.c 6:38f6223e:::benchmark_data_smithi163_287034_object49824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 453.518947373, 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.214667942, 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.222 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49825 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692571+0000", 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692600+0000", 2023-10-09T04:13:00.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9017999999999999e-05 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692612+0000", 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.171e-05 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692620+0000", 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1240000000000005e-06 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692694+0000", 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4127000000000004e-05 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692812+0000", 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011761599999999999 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:13:00.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692939+0000", 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012688700000000001 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904425+0000", 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.211486216 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904454+0000", 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9196999999999998e-05 2023-10-09T04:13:00.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907121+0000", 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002666982 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907194+0000", 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3281999999999994e-05 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907241+0000", 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6106999999999997e-05 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117683 6.b 6:df64dfc0:::benchmark_data_smithi163_287034_object9420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 378.64969857400001, 2023-10-09T04:13:00.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21390703, 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117683 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561824+0000", 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.525e-06 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561832+0000", 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6060000000000003e-06 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561837+0000", 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5090000000000004e-06 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561900+0000", 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2890000000000003e-05 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.562018+0000", 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011803699999999999 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.562119+0000", 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101275 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565635+0000", 2023-10-09T04:13:00.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035162940000000001 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.568886+0000", 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032510759999999999 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775646+0000", 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.206759888 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775692+0000", 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5757999999999998e-05 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775728+0000", 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6587000000000001e-05 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.236 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:644 7.c 7:3fcf2acf:::benchmark_data_smithi163_346082_object643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.61985422000001, 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21846595499999999, 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.237 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 644 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591664+0000", 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591688+0000", 2023-10-09T04:13:00.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3495000000000001e-05 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591699+0000", 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1834000000000001e-05 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591707+0000", 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6029999999999999e-06 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591761+0000", 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4044e-05 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591884+0000", 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012263899999999999 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592028+0000", 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014450300000000001 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809992+0000", 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21796359500000001 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810008+0000", 2023-10-09T04:13:00.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6293000000000001e-05 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810044+0000", 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5468e-05 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810090+0000", 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6409999999999998e-05 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810132+0000", 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1647999999999998e-05 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:645 7.c 7:3c7dda99:::benchmark_data_smithi163_346082_object644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:13:00.244 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.61979922800001, 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21961103900000001, 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 645 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591720+0000", 2023-10-09T04:13:00.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591740+0000", 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9584000000000001e-05 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591748+0000", 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591755+0000", 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7190000000000001e-06 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809375+0000", 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.217619379 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809503+0000", 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012796899999999999 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809689+0000", 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000186577 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810749+0000", 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010602400000000001 2023-10-09T04:13:00.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810776+0000", 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6995000000000002e-05 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811241+0000", 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046449799999999998 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811284+0000", 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3325999999999998e-05 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811332+0000", 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7512999999999997e-05 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:647 7.2 e171/167 7:4b12a746:::benchmark_data_smithi163_346082_object646:head v 171'43, mlcod=171'39)", 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.61954631899999, 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21782781300000001, 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591973+0000", 2023-10-09T04:13:00.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591996+0000", 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2821000000000001e-05 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592006+0000", 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0288999999999999e-05 2023-10-09T04:13:00.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592016+0000", 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0186e-05 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592076+0000", 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9373000000000002e-05 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808728+0000", 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21665226300000001 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809757+0000", 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001029409 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809801+0000", 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4079000000000003e-05 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:644 7.c e171/167 ondisk, result = 0)", 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.618780276, 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.217269874, 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.257 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592739+0000", 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592743+0000", 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.442e-06 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808711+0000", 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21596727399999999 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808728+0000", 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7541999999999999e-05 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809978+0000", 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001249687 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809985+0000", 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.498e-06 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810010+0000", 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4077999999999999e-05 2023-10-09T04:13:00.261 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:644 7.c e171/167 ondisk, result = 0)", 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.618681829, 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21730428800000001, 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592842+0000", 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1080000000000003e-06 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809055+0000", 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.216213023 2023-10-09T04:13:00.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809069+0000", 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3861e-05 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810034+0000", 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000965032 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810039+0000", 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5489999999999997e-06 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810142+0000", 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010331999999999999 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:38946 7.c 7:3abe6352:::benchmark_data_smithi163_346082_object38945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 189.24214681300001, 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23044379800000001, 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38946 2023-10-09T04:13:00.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969372+0000", 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969391+0000", 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9732999999999999e-05 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969404+0000", 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2989999999999999e-05 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969412+0000", 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9889999999999992e-06 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969455+0000", 2023-10-09T04:13:00.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3009000000000001e-05 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969584+0000", 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012886500000000001 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969717+0000", 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013274599999999999 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972017+0000", 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023001100000000002 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199698+0000", 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22768131599999999 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199742+0000", 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.375e-05 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199779+0000", 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6940999999999997e-05 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199817+0000", 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7861000000000002e-05 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38943 7.16 e175/174 7:6e66cd39:::benchmark_data_smithi163_346082_object38942:head v 175'2310, mlcod=175'2307)", 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 189.24015948100001, 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22686308999999999, 2023-10-09T04:13:00.274 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971382+0000", 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1302e-05 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971388+0000", 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0599999999999996e-06 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971393+0000", 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3909999999999997e-06 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972116+0000", 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072317400000000004 2023-10-09T04:13:00.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972130+0000", 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3424999999999999e-05 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972810+0000", 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068082899999999998 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198224+0000", 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.225413118 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:38951 7.c 7:39c83fcb:::benchmark_data_smithi163_346082_object38950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 189.23923728099999, 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22769973700000001, 2023-10-09T04:13:00.279 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38951 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972281+0000", 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972307+0000", 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5286000000000002e-05 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972319+0000", 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2325e-05 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972341+0000", 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1837000000000001e-05 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972386+0000", 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5516000000000003e-05 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972524+0000", 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013795999999999999 2023-10-09T04:13:00.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972659+0000", 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013506 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199867+0000", 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22720789899999999 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199914+0000", 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6495999999999997e-05 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199923+0000", 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7970000000000006e-06 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199950+0000", 2023-10-09T04:13:00.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7152999999999999e-05 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199982+0000", 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2765000000000002e-05 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38945 7.a e175/167 7:5f8231b3:::benchmark_data_smithi163_346082_object38944:head v 175'2413, mlcod=175'2410)", 2023-10-09T04:13:00.286 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 189.23872472100001, 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.228137689, 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972816+0000", 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1577999999999999e-05 2023-10-09T04:13:00.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198218+0000", 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22540215399999999 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198227+0000", 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9449999999999999e-06 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198279+0000", 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1304999999999998e-05 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198298+0000", 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9219999999999999e-05 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200904+0000", 2023-10-09T04:13:00.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002605797 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200933+0000", 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9247e-05 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:38946 7.c e175/167 ondisk, result = 0)", 2023-10-09T04:13:00.291 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 189.238697307, 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22688220200000001, 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972822+0000", 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972826+0000", 2023-10-09T04:13:00.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1230000000000002e-06 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198187+0000", 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22536124799999999 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198204+0000", 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7017999999999999e-05 2023-10-09T04:13:00.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199684+0000", 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001479209 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199692+0000", 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1899999999999995e-06 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199705+0000", 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2989999999999999e-05 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38948 7.11 e175/174 7:88c099c1:::benchmark_data_smithi163_346082_object38947:head v 175'2351, mlcod=175'2349)", 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 189.238142154, 2023-10-09T04:13:00.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22730851799999999, 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973377+0000", 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973404+0000", 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7041000000000001e-05 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198189+0000", 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22478490800000001 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198205+0000", 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6181e-05 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198251+0000", 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6136e-05 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198278+0000", 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7257e-05 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200653+0000", 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023747289999999999 2023-10-09T04:13:00.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200686+0000", 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3191999999999997e-05 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80498 7.13 7:ce654277:::benchmark_data_smithi163_346082_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:13:00.301 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.893679222, 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1130738259999999, 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80498 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317838+0000", 2023-10-09T04:13:00.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317881+0000", 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3013000000000003e-05 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317895+0000", 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3732e-05 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317903+0000", 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5209999999999997e-06 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318091+0000", 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188829 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:13:00.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318101+0000", 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.55e-06 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323224+0000", 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051229439999999999 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323236+0000", 2023-10-09T04:13:00.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992e-05 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.423747+0000", 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.100511362 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.423904+0000", 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015625099999999999 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.424075+0000", 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000170977 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430599+0000", 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065249189999999997 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430661+0000", 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1789000000000001e-05 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430676+0000", 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4297999999999999e-05 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430753+0000", 2023-10-09T04:13:00.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7937999999999998e-05 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430805+0000", 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1829000000000001e-05 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430915+0000", 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109257 2023-10-09T04:13:00.310 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80508 7.13 7:cf9b83a2:::benchmark_data_smithi163_346082_object80507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.893603109, 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1132838570000001, 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.311 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80508 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317916+0000", 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317944+0000", 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7642000000000001e-05 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317952+0000", 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3070000000000003e-06 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317958+0000", 2023-10-09T04:13:00.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1290000000000001e-06 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318152+0000", 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019322999999999999 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318158+0000", 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7700000000000004e-06 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323259+0000", 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051002950000000003 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323278+0000", 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814999999999999e-05 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425771+0000", 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.102493095 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425915+0000", 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000144451 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.426078+0000", 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000162595 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430682+0000", 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046044320000000003 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430721+0000", 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9020000000000002e-05 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430993+0000", 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027199400000000001 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431048+0000", 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4808000000000001e-05 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431097+0000", 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9586000000000003e-05 2023-10-09T04:13:00.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431201+0000", 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103231 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80516 7.13 7:c9a4c09d:::benchmark_data_smithi163_346082_object80515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:13:00.320 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.89355635300001, 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.116727963, 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80516 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:13:00.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317963+0000", 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317983+0000", 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9938000000000001e-05 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317990+0000", 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7850000000000003e-06 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317993+0000", 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3680000000000002e-06 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318180+0000", 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018708599999999999 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318187+0000", 2023-10-09T04:13:00.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2589999999999997e-06 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323302+0000", 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051155979999999998 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323310+0000", 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0660000000000004e-06 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427478+0000", 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1041673599999999 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427613+0000", 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000135888 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427760+0000", 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014688799999999999 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430688+0000", 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029275270000000001 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431261+0000", 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00057298199999999998 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434445+0000", 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031837290000000002 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434514+0000", 2023-10-09T04:13:00.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8955999999999995e-05 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434567+0000", 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3599e-05 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434692+0000", 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012444600000000001 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.329 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80551 7.13 7:c9852ceb:::benchmark_data_smithi163_346082_object80550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.74104592500001, 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96918742400000002, 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.330 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80551 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470472+0000", 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470498+0000", 2023-10-09T04:13:00.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5672000000000001e-05 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470514+0000", 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5846999999999999e-05 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470522+0000", 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9629999999999998e-06 2023-10-09T04:13:00.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470576+0000", 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4360999999999997e-05 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470594+0000", 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7754999999999999e-05 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.428996+0000", 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95840238700000002 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429120+0000", 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012304300000000001 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:13:00.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429313+0000", 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019363099999999999 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431216+0000", 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019029240000000001 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434407+0000", 2023-10-09T04:13:00.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031906230000000001 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434794+0000", 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00038685899999999998 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439487+0000", 2023-10-09T04:13:00.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046938939999999997 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439541+0000", 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3309000000000001e-05 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439661+0000", 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120699 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.338 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80553 7.3 7:c0606c4f:::benchmark_data_smithi163_346082_object80552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.73785580200001, 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95719071, 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.339 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80553 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473663+0000", 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473684+0000", 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1585000000000001e-05 2023-10-09T04:13:00.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473699+0000", 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4456e-05 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473705+0000", 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6630000000000004e-06 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473767+0000", 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1607999999999997e-05 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473779+0000", 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2330999999999999e-05 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.419942+0000", 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94616239800000002 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.420106+0000", 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016377699999999999 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.420277+0000", 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017136500000000001 2023-10-09T04:13:00.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425713+0000", 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005435601 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425748+0000", 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5747999999999997e-05 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430649+0000", 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049003709999999997 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430704+0000", 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5140999999999999e-05 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430751+0000", 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7691999999999997e-05 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430855+0000", 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103417 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.347 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80558 7.13 7:c83af463:::benchmark_data_smithi163_346082_object80557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 141.731959929, 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96028118200000001, 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80558 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:13:00.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:13:00.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479559+0000", 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479582+0000", 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2878999999999999e-05 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479594+0000", 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1328000000000001e-05 2023-10-09T04:13:00.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479601+0000", 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3690000000000001e-06 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479661+0000", 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0053000000000003e-05 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479673+0000", 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1877e-05 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431306+0000", 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95163314200000004 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431474+0000", 2023-10-09T04:13:00.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016787400000000001 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431652+0000", 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017823200000000001 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434479+0000", 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028264900000000001 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434839+0000", 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00036005299999999999 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434892+0000", 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3001000000000003e-05 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439712+0000", 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048203079999999997 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439752+0000", 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9496999999999998e-05 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:13:00.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439841+0000", 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9649999999999997e-05 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:13:00.356 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:13:00.357 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:00.358 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:00.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:00.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:00.370 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:13:00.392 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:13:00.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:13:00.438 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:13:00.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:00.459 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:13:00.483 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:13:00.492 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:00.508 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:13:00.528 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.529 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:13:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[221067]: pgmap v601: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 543 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 177/26088 objects degraded (0.678%); 907/26088 objects misplaced (3.477%); 0 B/s, 60 objects/s recovering 2023-10-09T04:13:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[221067]: failed to encode map e224 with expected crc 2023-10-09T04:13:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[221067]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T04:13:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[221067]: failed to encode map e224 with expected crc 2023-10-09T04:13:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[221067]: failed to encode map e224 with expected crc 2023-10-09T04:13:00.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2965423555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[213841]: pgmap v601: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 543 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 177/26088 objects degraded (0.678%); 907/26088 objects misplaced (3.477%); 0 B/s, 60 objects/s recovering 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[213841]: failed to encode map e224 with expected crc 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[213841]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[213841]: failed to encode map e224 with expected crc 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[213841]: failed to encode map e224 with expected crc 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2965423555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:00.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:13:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:13:00] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-10-09T04:13:01.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:00 smithi181 ceph-mon[196979]: pgmap v601: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 543 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 177/26088 objects degraded (0.678%); 907/26088 objects misplaced (3.477%); 0 B/s, 60 objects/s recovering 2023-10-09T04:13:01.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:00 smithi181 ceph-mon[196979]: failed to encode map e224 with expected crc 2023-10-09T04:13:01.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:00 smithi181 ceph-mon[196979]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T04:13:01.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:00 smithi181 ceph-mon[196979]: failed to encode map e224 with expected crc 2023-10-09T04:13:01.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:00 smithi181 ceph-mon[196979]: failed to encode map e224 with expected crc 2023-10-09T04:13:01.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2965423555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:01.613 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:01.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:01.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:01.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:01.747 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[213841]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:01 smithi163 ceph-mon[221067]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:13:02.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T04:13:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:01 smithi181 ceph-mon[196979]: failed to encode map e225 with expected crc 2023-10-09T04:13:02.163 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:02.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:02.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:02.309 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:02.338 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:02.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:02.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:02.421 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:02.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:02.524 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:02.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:02.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:02.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:02.759 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: pgmap v604: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 456 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 697/21921 objects misplaced (3.180%); 241 KiB/s, 77 objects/s recovering 2023-10-09T04:13:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 177/26088 objects degraded (0.678%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:13:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[221067]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: pgmap v604: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 456 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 697/21921 objects misplaced (3.180%); 241 KiB/s, 77 objects/s recovering 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 177/26088 objects degraded (0.678%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:13:02.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T04:13:02.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:02 smithi163 ceph-mon[213841]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: pgmap v604: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 456 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 697/21921 objects misplaced (3.180%); 241 KiB/s, 77 objects/s recovering 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 177/26088 objects degraded (0.678%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:02 smithi181 ceph-mon[196979]: failed to encode map e226 with expected crc 2023-10-09T04:13:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:02.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:03.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:03.098 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:03.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:03.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:03.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:03.275 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:03.333 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:03.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:03.439 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:03.506 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:03.588 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:03.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:03.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:03 smithi181 ceph-mon[196979]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:03 smithi181 ceph-mon[196979]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:03 smithi181 ceph-mon[196979]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T04:13:03.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:03 smithi181 ceph-mon[196979]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:03.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[213841]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[213841]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[213841]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T04:13:03.745 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[213841]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[221067]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[221067]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[221067]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T04:13:03.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:03 smithi163 ceph-mon[221067]: failed to encode map e227 with expected crc 2023-10-09T04:13:03.754 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:03.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:03.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:03.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:04.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:04.064 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:04.107 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:04.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:04.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:04.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:04.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:13:04.278 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:04.692 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:04.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:04.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: pgmap v607: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 410 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 697/19740 objects misplaced (3.531%); 416 KiB/s, 50 objects/s recovering 2023-10-09T04:13:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T04:13:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:04.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: pgmap v607: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 410 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 697/19740 objects misplaced (3.531%); 416 KiB/s, 50 objects/s recovering 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: failed to encode map e228 with expected crc 2023-10-09T04:13:04.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:04.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:04.892 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: pgmap v607: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 95 active+clean; 410 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 697/19740 objects misplaced (3.531%); 416 KiB/s, 50 objects/s recovering 2023-10-09T04:13:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: failed to encode map e228 with expected crc 2023-10-09T04:13:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: failed to encode map e228 with expected crc 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: failed to encode map e228 with expected crc 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: failed to encode map e228 with expected crc 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: failed to encode map e228 with expected crc 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: failed to encode map e228 with expected crc 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:05.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:05.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:05.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:05.236 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:05.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:05.531 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:05.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:05.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:05.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:05.746 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:13:05.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:05.890 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3261236321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:13:05.890 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:13:05.891 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3261236321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:13:05.891 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:13:05.891 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:13:05.891 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":750,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":229,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":8,"osd_in_since":1696823548,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":98},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":99,"num_pools":5,"num_objects":5407,"data_bytes":353310648,"bytes_used":4022738944,"bytes_avail":763937546240,"bytes_total":767960285184,"misplaced_objects":199,"misplaced_total":16221,"misplaced_ratio":0.012268047592626842,"recovering_objects_per_sec":80,"recovering_bytes_per_sec":5266276,"recovering_keys_per_sec":0,"num_objects_recovered":391,"num_bytes_recovered":25559040,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":444},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":21,"modified":"2023-10-09T04:04:35.266858+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"61f8f0b7-6515-43d1-9684-78e0e9193886":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 7s)","progress":0.95959597826004028,"add_to_ceph_s":true}}} 2023-10-09T04:13:05.971 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:06.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3261236321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:13:06.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:13:06.803 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: pgmap v609: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 199/16221 objects misplaced (1.227%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T04:13:06.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:06.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: failed to encode map e229 with expected crc 2023-10-09T04:13:06.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3261236321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:13:06.863 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:13:06.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: osdmap e229: 8 total, 8 up, 8 in 2023-10-09T04:13:06.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: failed to encode map e229 with expected crc 2023-10-09T04:13:06.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: 7.6 scrub starts 2023-10-09T04:13:06.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4012809952' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:13:06.864 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:06 smithi181 ceph-mon[196979]: 7.6 scrub ok 2023-10-09T04:13:07.027 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:13:07.822 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:13:07.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: pgmap v609: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 199/16221 objects misplaced (1.227%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T04:13:07.823 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:07.824 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: failed to encode map e229 with expected crc 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3261236321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: osdmap e229: 8 total, 8 up, 8 in 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: failed to encode map e229 with expected crc 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: 7.6 scrub starts 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4012809952' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:13:07.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[221067]: 7.6 scrub ok 2023-10-09T04:13:07.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: pgmap v609: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 199/16221 objects misplaced (1.227%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T04:13:07.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:07.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: failed to encode map e229 with expected crc 2023-10-09T04:13:07.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3261236321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:13:07.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:13:07.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: osdmap e229: 8 total, 8 up, 8 in 2023-10-09T04:13:07.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: failed to encode map e229 with expected crc 2023-10-09T04:13:07.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: 7.6 scrub starts 2023-10-09T04:13:07.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4012809952' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:13:07.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:06 smithi163 ceph-mon[213841]: 7.6 scrub ok 2023-10-09T04:13:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:07.975 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:13:07.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:07.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:07.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: failed to encode map e230 with expected crc 2023-10-09T04:13:07.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T04:13:07.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: failed to encode map e230 with expected crc 2023-10-09T04:13:07.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: failed to encode map e230 with expected crc 2023-10-09T04:13:07.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: failed to encode map e230 with expected crc 2023-10-09T04:13:07.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: 7.11 scrub starts 2023-10-09T04:13:07.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: 7.11 scrub ok 2023-10-09T04:13:07.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: 7.8 scrub starts 2023-10-09T04:13:07.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: pgmap v612: 99 pgs: 99 active+clean; 307 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7.5 MiB/s, 119 objects/s recovering 2023-10-09T04:13:08.123 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: 7.11 scrub starts 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: 7.11 scrub ok 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: 7.8 scrub starts 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: pgmap v612: 99 pgs: 99 active+clean; 307 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7.5 MiB/s, 119 objects/s recovering 2023-10-09T04:13:08.124 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:13:08.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:08.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T04:13:08.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: failed to encode map e230 with expected crc 2023-10-09T04:13:08.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: 7.11 scrub starts 2023-10-09T04:13:08.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: 7.11 scrub ok 2023-10-09T04:13:08.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: 7.8 scrub starts 2023-10-09T04:13:08.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: pgmap v612: 99 pgs: 99 active+clean; 307 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 7.5 MiB/s, 119 objects/s recovering 2023-10-09T04:13:08.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:13:08.128 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:13:08.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:13:08.166 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:08.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:08.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:08.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:08.472 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:08.598 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:13:08.598 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:13:08.598 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:13:08.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:08.599 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:08.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:08.600 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:08.607 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:13:08.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:08.819 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:13:08.820 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:13:08.820 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:13:08.820 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:13:08.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.822 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.822 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.822 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:13:08.822 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:08.849 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:08.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:08.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: 7.8 scrub ok 2023-10-09T04:13:09.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:13:09.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T04:13:09.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: 7.7 scrub starts 2023-10-09T04:13:09.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: 7.7 scrub ok 2023-10-09T04:13:09.131 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2709594468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:09.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:09.133 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: 7.8 scrub ok 2023-10-09T04:13:09.133 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:13:09.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T04:13:09.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.134 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: 7.7 scrub starts 2023-10-09T04:13:09.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: 7.7 scrub ok 2023-10-09T04:13:09.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2709594468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:09.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: 7.8 scrub ok 2023-10-09T04:13:09.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:13:09.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T04:13:09.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: failed to encode map e231 with expected crc 2023-10-09T04:13:09.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: 7.7 scrub starts 2023-10-09T04:13:09.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: 7.7 scrub ok 2023-10-09T04:13:09.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2709594468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:09.160 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:09.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:09.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:09.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:09.501 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:09.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:09.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: failed to encode map e232 with expected crc 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: failed to encode map e232 with expected crc 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: failed to encode map e232 with expected crc 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: failed to encode map e232 with expected crc 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: pgmap v615: 99 pgs: 99 active+clean; 234 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:13:09.784 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:09 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:13:09.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:09.971 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:10.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:13:10.093 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.093 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.093 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T04:13:10.093 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.093 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.094 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: pgmap v615: 99 pgs: 99 active+clean; 234 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:13:10.094 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:13:10.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T04:13:10.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: failed to encode map e232 with expected crc 2023-10-09T04:13:10.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: pgmap v615: 99 pgs: 99 active+clean; 234 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:13:10.096 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:09 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:13:10.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:10.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:10.209 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:13:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:13:10.210 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:13:10.212 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:10.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:10.391 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:10.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:10.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:10.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:10.628 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:10.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:11.366 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:13:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:13:10] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2023-10-09T04:13:11.367 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[213841]: failed to encode map e233 with expected crc 2023-10-09T04:13:11.367 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[213841]: failed to encode map e233 with expected crc 2023-10-09T04:13:11.367 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:13:11.367 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[213841]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T04:13:11.368 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[213841]: failed to encode map e233 with expected crc 2023-10-09T04:13:11.368 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[213841]: failed to encode map e233 with expected crc 2023-10-09T04:13:11.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:11.368 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[221067]: failed to encode map e233 with expected crc 2023-10-09T04:13:11.369 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[221067]: failed to encode map e233 with expected crc 2023-10-09T04:13:11.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:13:11.438 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[221067]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T04:13:11.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[221067]: failed to encode map e233 with expected crc 2023-10-09T04:13:13.381 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:10 smithi163 ceph-mon[221067]: failed to encode map e233 with expected crc 2023-10-09T04:13:13.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:10 smithi181 ceph-mon[196979]: failed to encode map e233 with expected crc 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:10 smithi181 ceph-mon[196979]: failed to encode map e233 with expected crc 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:10 smithi181 ceph-mon[196979]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:10 smithi181 ceph-mon[196979]: failed to encode map e233 with expected crc 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:10 smithi181 ceph-mon[196979]: failed to encode map e233 with expected crc 2023-10-09T04:13:13.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:13:13.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:11 smithi181 ceph-mon[196979]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:13:13.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:11 smithi181 ceph-mon[196979]: failed to encode map e234 with expected crc 2023-10-09T04:13:13.387 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:11 smithi181 ceph-mon[196979]: failed to encode map e234 with expected crc 2023-10-09T04:13:13.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:11 smithi181 ceph-mon[196979]: failed to encode map e234 with expected crc 2023-10-09T04:13:13.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:11 smithi181 ceph-mon[196979]: pgmap v618: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 183 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:13:13.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T04:13:13.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: failed to encode map e235 with expected crc 2023-10-09T04:13:14.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: failed to encode map e235 with expected crc 2023-10-09T04:13:14.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:13:14.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: failed to encode map e235 with expected crc 2023-10-09T04:13:14.596 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: failed to encode map e235 with expected crc 2023-10-09T04:13:14.597 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:12 smithi181 ceph-mon[196979]: failed to encode map e235 with expected crc 2023-10-09T04:13:14.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:13:14.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:14.606 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[213841]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[213841]: failed to encode map e234 with expected crc 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[213841]: failed to encode map e234 with expected crc 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[213841]: failed to encode map e234 with expected crc 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[213841]: pgmap v618: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 183 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:13:14.620 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:13 smithi181 ceph-mon[196979]: pgmap v620: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 141 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2/6810 objects degraded (0.029%) 2023-10-09T04:13:14.621 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[221067]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:13:14.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[221067]: failed to encode map e234 with expected crc 2023-10-09T04:13:14.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[221067]: failed to encode map e234 with expected crc 2023-10-09T04:13:14.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[221067]: failed to encode map e234 with expected crc 2023-10-09T04:13:14.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:11 smithi163 ceph-mon[221067]: pgmap v618: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 183 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:13:14.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T04:13:14.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: failed to encode map e235 with expected crc 2023-10-09T04:13:14.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:13:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[213841]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-10-09T04:13:15.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:13:15.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:12 smithi163 ceph-mon[221067]: failed to encode map e235 with expected crc 2023-10-09T04:13:15.240 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:15.241 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:13 smithi163 ceph-mon[213841]: pgmap v620: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 141 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2/6810 objects degraded (0.029%) 2023-10-09T04:13:15.241 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:13 smithi163 ceph-mon[221067]: pgmap v620: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 141 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2/6810 objects degraded (0.029%) 2023-10-09T04:13:15.245 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:15.371 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:13:15.373 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:13:15.375 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 93267 objects 2023-10-09T04:13:15.375 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :135.849 2023-10-09T04:13:15.375 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:15 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:15.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:15 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:15.638 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:15 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:16.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:16.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:16.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:16.487 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:16.691 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:16 smithi181 ceph-mon[196979]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:16.691 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:16 smithi181 ceph-mon[196979]: pgmap v621: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 73 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/3546 objects degraded (0.056%) 2023-10-09T04:13:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:16 smithi163 ceph-mon[221067]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:16 smithi163 ceph-mon[221067]: pgmap v621: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 73 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/3546 objects degraded (0.056%) 2023-10-09T04:13:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:16 smithi163 ceph-mon[213841]: from='client.35409 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:16 smithi163 ceph-mon[213841]: pgmap v621: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 73 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/3546 objects degraded (0.056%) 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118957 6.16 6:6cc89a38:::benchmark_data_smithi163_287034_object10694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.580653+0000", 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 392.33480261199998, 2023-10-09T04:13:16.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91563185499999999, 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118957 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580653+0000", 2023-10-09T04:13:16.920 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580653+0000", 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580652+0000", 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580661+0000", 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4720000000000001e-06 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580679+0000", 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7669e-05 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580687+0000", 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6399999999999997e-06 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580744+0000", 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7002000000000001e-05 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.580754+0000", 2023-10-09T04:13:16.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0158e-05 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.484645+0000", 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90389140599999995 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.484758+0000", 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112625 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.484878+0000", 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011964 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491749+0000", 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068712770000000003 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.493783+0000", 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020337530000000001 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496193+0000", 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024100430000000002 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496242+0000", 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9422000000000002e-05 2023-10-09T04:13:16.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496285+0000", 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3022999999999998e-05 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118992 6.16 6:68802e52:::benchmark_data_smithi163_287034_object10729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.619382+0000", 2023-10-09T04:13:16.927 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 392.29607408700002, 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87704807399999996, 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118992 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619382+0000", 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619382+0000", 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619381+0000", 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619384+0000", 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.835e-06 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619390+0000", 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8989999999999998e-06 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619393+0000", 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3450000000000002e-06 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619431+0000", 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7994000000000002e-05 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.619438+0000", 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6610000000000003e-06 2023-10-09T04:13:16.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486449+0000", 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.867010634 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486559+0000", 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109966 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486662+0000", 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103287 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491805+0000", 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051429680000000004 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.493790+0000", 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019851890000000001 2023-10-09T04:13:16.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496340+0000", 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002549506 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496389+0000", 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9885000000000001e-05 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496430+0000", 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0365999999999999e-05 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118993 6.6 6:61f9ef42:::benchmark_data_smithi163_287034_object10730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.621222+0000", 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 392.29423338499998, 2023-10-09T04:13:16.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87532114500000002, 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118993 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621222+0000", 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621222+0000", 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621222+0000", 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621226+0000", 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3239999999999996e-06 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621234+0000", 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7300000000000005e-06 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621238+0000", 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.031e-06 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621266+0000", 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8045999999999999e-05 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.621271+0000", 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1919999999999996e-06 2023-10-09T04:13:16.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.482641+0000", 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86136939199999996 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.482774+0000", 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132955 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.482896+0000", 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122447 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486437+0000", 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035414050000000001 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.488120+0000", 2023-10-09T04:13:16.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0016823840000000001 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496444+0000", 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0083243420000000002 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496505+0000", 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0921999999999997e-05 2023-10-09T04:13:16.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.496544+0000", 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8587999999999997e-05 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:119007 6.16 6:6f753fe4:::benchmark_data_smithi163_287034_object10744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.735123+0000", 2023-10-09T04:13:16.943 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 392.18033285400003, 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.76611524099999995, 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 119007 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735123+0000", 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735123+0000", 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735122+0000", 2023-10-09T04:13:16.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735127+0000", 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1250000000000001e-06 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735135+0000", 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2309999999999996e-06 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735139+0000", 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6030000000000001e-06 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735189+0000", 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0587999999999997e-05 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735198+0000", 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9109999999999999e-06 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.488146+0000", 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75294745100000005 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.488263+0000", 2023-10-09T04:13:16.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011688000000000001 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.488363+0000", 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000100504 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.491857+0000", 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034935970000000002 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.493795+0000", 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019381660000000001 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.501160+0000", 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0073647519999999996 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.501202+0000", 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2434000000000001e-05 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.501238+0000", 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6099000000000001e-05 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:119008 6.16 6:69e07f52:::benchmark_data_smithi163_287034_object10745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.735377+0000", 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 392.180078842, 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.76598240299999998, 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 119008 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.952 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735377+0000", 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735377+0000", 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735377+0000", 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735379+0000", 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8990000000000001e-06 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735385+0000", 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.412e-06 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735389+0000", 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2209999999999999e-06 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735427+0000", 2023-10-09T04:13:16.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8143999999999999e-05 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.735433+0000", 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6409999999999999e-06 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489903+0000", 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75447040700000001 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.490033+0000", 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012979699999999999 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.490156+0000", 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123323 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.493621+0000", 2023-10-09T04:13:16.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034648140000000001 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.493800+0000", 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178572 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.501280+0000", 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0074798649999999996 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.501325+0000", 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5559999999999997e-05 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.501359+0000", 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4205000000000002e-05 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36610 7.1 7:86d4b555:::benchmark_data_smithi163_346082_object36609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.296917+0000", 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 208.61853903100001, 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97434309600000002, 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36610 2023-10-09T04:13:16.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296917+0000", 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296917+0000", 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296910+0000", 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999933 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296970+0000", 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0127999999999998e-05 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297009+0000", 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8614e-05 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297022+0000", 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3477e-05 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306475+0000", 2023-10-09T04:13:16.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0094528070000000006 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306491+0000", 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5642e-05 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.265950+0000", 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95945866800000001 2023-10-09T04:13:16.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.268174+0000", 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002224536 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271214+0000", 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003039618 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271260+0000", 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6233000000000003e-05 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36613 7.1 7:85783f21:::benchmark_data_smithi163_346082_object36612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.297031+0000", 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 208.61842528, 2023-10-09T04:13:16.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97415562300000003, 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36613 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297031+0000", 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297031+0000", 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297030+0000", 2023-10-09T04:13:16.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297063+0000", 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2888000000000001e-05 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297070+0000", 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.418e-06 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297075+0000", 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7690000000000004e-06 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306589+0000", 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0095134810000000007 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306596+0000", 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2200000000000003e-06 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.268225+0000", 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96162872300000002 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:13:16.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.269076+0000", 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00085153799999999999 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271114+0000", 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020374289999999999 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271186+0000", 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2655000000000001e-05 2023-10-09T04:13:16.972 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52729 7.13 7:cc295b9f:::benchmark_data_smithi163_346082_object52728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.293100+0000", 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.62235567600001, 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98155146500000001, 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.973 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52729 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293100+0000", 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293100+0000", 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293099+0000", 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293174+0000", 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5077e-05 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293189+0000", 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4742e-05 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293195+0000", 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1120000000000001e-06 2023-10-09T04:13:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293248+0000", 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2986999999999999e-05 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293258+0000", 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0259e-05 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.270342+0000", 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97708439199999997 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.270441+0000", 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8899000000000003e-05 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.270565+0000", 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123616 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273472+0000", 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029067540000000001 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273525+0000", 2023-10-09T04:13:16.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.342e-05 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274570+0000", 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010451099999999999 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274615+0000", 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4753999999999999e-05 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274652+0000", 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6816000000000001e-05 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52747 7.13 7:cda012a2:::benchmark_data_smithi163_346082_object52746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.302565+0000", 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.61289087599999, 2023-10-09T04:13:16.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97614182299999996, 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52747 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302565+0000", 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302565+0000", 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302564+0000", 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:16.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302622+0000", 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8745000000000001e-05 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302645+0000", 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.317e-05 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302653+0000", 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.289e-06 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302734+0000", 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0788e-05 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302746+0000", 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.29e-05 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.271839+0000", 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96909244900000002 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.271972+0000", 2023-10-09T04:13:16.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132862 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272106+0000", 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013400200000000001 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273559+0000", 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001452977 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274496+0000", 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00093728699999999997 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278619+0000", 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041224909999999998 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278667+0000", 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8822000000000001e-05 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278707+0000", 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9437000000000002e-05 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.989 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52798 7.13 7:cc44a9ee:::benchmark_data_smithi163_346082_object52797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.336634+0000", 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.578821699, 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94227905499999998, 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52798 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336634+0000", 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336634+0000", 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336633+0000", 2023-10-09T04:13:16.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336683+0000", 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0476000000000003e-05 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336716+0000", 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2693e-05 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336732+0000", 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6399999999999999e-05 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336797+0000", 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4950999999999995e-05 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336807+0000", 2023-10-09T04:13:16.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8889999999999997e-06 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272736+0000", 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93592933 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272820+0000", 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3343999999999994e-05 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272913+0000", 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3618999999999999e-05 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274531+0000", 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001617531 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:16.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274537+0000", 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7270000000000004e-06 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278834+0000", 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042977900000000001 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278879+0000", 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4243000000000003e-05 2023-10-09T04:13:16.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278913+0000", 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4530000000000003e-05 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52815 7.13 7:cd1aa158:::benchmark_data_smithi163_346082_object52814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.351193+0000", 2023-10-09T04:13:16.997 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.56426281399999, 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92784179700000002, 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52815 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:16.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351193+0000", 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351193+0000", 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351191+0000", 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:16.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351246+0000", 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4660999999999997e-05 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351259+0000", 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3348e-05 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351266+0000", 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3040000000000001e-06 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351324+0000", 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8427999999999997e-05 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351336+0000", 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1749000000000001e-05 2023-10-09T04:13:17.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273572+0000", 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92223578900000003 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273650+0000", 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8481999999999998e-05 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273775+0000", 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012533299999999999 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275290+0000", 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001514149 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275329+0000", 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9737000000000003e-05 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278961+0000", 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036314860000000002 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279004+0000", 2023-10-09T04:13:17.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3208999999999999e-05 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279035+0000", 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0839000000000001e-05 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52830 7.13 7:cb4153a8:::benchmark_data_smithi163_346082_object52829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.005 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.365239+0000", 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.55021649099999, 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91388604799999995, 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52830 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365239+0000", 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365239+0000", 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365238+0000", 2023-10-09T04:13:17.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365296+0000", 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8714000000000001e-05 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365314+0000", 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7504999999999999e-05 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365321+0000", 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2579999999999998e-06 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365387+0000", 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5335999999999999e-05 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365396+0000", 2023-10-09T04:13:17.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2860000000000005e-06 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274673+0000", 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90927686100000005 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274745+0000", 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2463000000000001e-05 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274831+0000", 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5567000000000003e-05 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276891+0000", 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020600050000000002 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276937+0000", 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6011999999999999e-05 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279076+0000", 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021394700000000001 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279105+0000", 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8549000000000001e-05 2023-10-09T04:13:17.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279125+0000", 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0636999999999999e-05 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52848 7.13 7:cd543811:::benchmark_data_smithi163_346082_object52847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.013 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.379437+0000", 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.536018552, 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89977305799999996, 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52848 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379437+0000", 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379437+0000", 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379436+0000", 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379497+0000", 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1274999999999999e-05 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379508+0000", 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0465999999999999e-05 2023-10-09T04:13:17.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379512+0000", 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000004e-06 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379570+0000", 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8325000000000001e-05 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379581+0000", 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0358e-05 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275369+0000", 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89578837700000002 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275474+0000", 2023-10-09T04:13:17.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010519299999999999 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275577+0000", 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010239299999999999 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276928+0000", 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001351093 2023-10-09T04:13:17.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276945+0000", 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7564e-05 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279156+0000", 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022106909999999999 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279181+0000", 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5131999999999999e-05 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279210+0000", 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9074999999999999e-05 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52857 7.13 7:cc4020e3:::benchmark_data_smithi163_346082_object52856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.385532+0000", 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.529923502, 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89567147499999999, 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52857 2023-10-09T04:13:17.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385532+0000", 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385532+0000", 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385531+0000", 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385594+0000", 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2288999999999999e-05 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385608+0000", 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4569000000000001e-05 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385615+0000", 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1899999999999998e-06 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385670+0000", 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4166000000000003e-05 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385679+0000", 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7739999999999999e-06 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276088+0000", 2023-10-09T04:13:17.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89040851399999998 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276210+0000", 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012211599999999999 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276353+0000", 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014332700000000001 2023-10-09T04:13:17.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278512+0000", 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002158751 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278565+0000", 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3368999999999997e-05 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281109+0000", 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025435779999999999 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281164+0000", 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4601000000000001e-05 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281204+0000", 2023-10-09T04:13:17.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0277000000000001e-05 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52865 7.13 7:cefc765f:::benchmark_data_smithi163_346082_object52864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.390532+0000", 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.52492405699999, 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89079931999999995, 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.030 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52865 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390532+0000", 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390532+0000", 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390531+0000", 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390597+0000", 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6179000000000005e-05 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390609+0000", 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2157999999999999e-05 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390615+0000", 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9349999999999999e-06 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390678+0000", 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2751999999999998e-05 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390687+0000", 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3130000000000004e-06 2023-10-09T04:13:17.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276971+0000", 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88628442699999999 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277087+0000", 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0001154 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277191+0000", 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104571 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278716+0000", 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015244970000000001 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278774+0000", 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8325999999999997e-05 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281248+0000", 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024733749999999999 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281292+0000", 2023-10-09T04:13:17.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4144000000000003e-05 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281331+0000", 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9363000000000002e-05 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52871 7.13 7:c955e08a:::benchmark_data_smithi163_346082_object52870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.038 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.395047+0000", 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.52040922699999, 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88652477799999996, 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52871 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395047+0000", 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395047+0000", 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395045+0000", 2023-10-09T04:13:17.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395097+0000", 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1892999999999999e-05 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395114+0000", 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7306e-05 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395123+0000", 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4970000000000001e-06 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395185+0000", 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2212999999999995e-05 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395194+0000", 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2210000000000003e-06 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277731+0000", 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88253674599999998 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277845+0000", 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011417300000000001 2023-10-09T04:13:17.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277994+0000", 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014894999999999999 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278925+0000", 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00093115099999999996 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279241+0000", 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031537999999999998 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281476+0000", 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022349290000000001 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281535+0000", 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9324999999999999e-05 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281571+0000", 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6480999999999999e-05 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52913 7.13 7:ce6a4229:::benchmark_data_smithi163_346082_object52912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.48621354799999, 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85417199600000004, 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52913 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.047 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429266+0000", 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4411999999999999e-05 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429273+0000", 2023-10-09T04:13:17.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5470000000000002e-06 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429276+0000", 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.898e-06 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429314+0000", 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8690999999999999e-05 2023-10-09T04:13:17.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429321+0000", 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5400000000000001e-06 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279261+0000", 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84994013899999998 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279332+0000", 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1373999999999998e-05 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279470+0000", 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013728500000000001 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281216+0000", 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001746446 2023-10-09T04:13:17.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281445+0000", 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022927900000000001 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283362+0000", 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019170459999999999 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283393+0000", 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0997000000000002e-05 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283414+0000", 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0826e-05 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52963 7.13 7:c8f4aee2:::benchmark_data_smithi163_346082_object52962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.472521+0000", 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.442934941, 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.81096846499999997, 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52963 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.055 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472521+0000", 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472521+0000", 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472520+0000", 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472555+0000", 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4545000000000002e-05 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472568+0000", 2023-10-09T04:13:17.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3464e-05 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472574+0000", 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9070000000000004e-06 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472636+0000", 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1921999999999994e-05 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472664+0000", 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7872999999999999e-05 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.280145+0000", 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80748120000000001 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.280241+0000", 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5638999999999997e-05 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.280364+0000", 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123497 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281414+0000", 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010499019999999999 2023-10-09T04:13:17.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281613+0000", 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019900399999999999 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283440+0000", 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018269479999999999 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283464+0000", 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3978999999999999e-05 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283489+0000", 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5471000000000001e-05 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52997 7.13 7:cdf64a9a:::benchmark_data_smithi163_346082_object52996:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.404575537, 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.77769543699999999, 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52997 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.063 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510911+0000", 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0945000000000003e-05 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510916+0000", 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5419999999999996e-06 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510919+0000", 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.27e-06 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510966+0000", 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6329999999999999e-05 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510973+0000", 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1080000000000004e-06 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281637+0000", 2023-10-09T04:13:17.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.77066409199999997 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281739+0000", 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010242300000000001 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281872+0000", 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013302100000000001 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283420+0000", 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001547732 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.285133+0000", 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017123399999999999 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288476+0000", 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033432010000000001 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288536+0000", 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0207999999999997e-05 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288576+0000", 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9786000000000002e-05 2023-10-09T04:13:17.070 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:53010 7.13 7:cd640b3a:::benchmark_data_smithi163_346082_object53009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.531489+0000", 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 188.38396654100001, 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75803050299999997, 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:17.071 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 53010 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531489+0000", 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531489+0000", 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531480+0000", 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999905 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531540+0000", 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0124999999999998e-05 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531564+0000", 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3927e-05 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:17.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531574+0000", 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9429999999999995e-06 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531699+0000", 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012563799999999999 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531722+0000", 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2609e-05 2023-10-09T04:13:17.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.282598+0000", 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75087631399999999 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.282676+0000", 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7862999999999996e-05 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.282756+0000", 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0043000000000006e-05 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.285163+0000", 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024068679999999999 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288608+0000", 2023-10-09T04:13:17.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003444754 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.289432+0000", 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000824009 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.289486+0000", 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4159000000000001e-05 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:17.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.289520+0000", 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3914000000000001e-05 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:17.079 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:17.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:13:17.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2/3546 objects degraded (0.056%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:13:17.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2/3546 objects degraded (0.056%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:13:17.925 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:18.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2/3546 objects degraded (0.056%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:13:18.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:18.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:18.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:18.582 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:18.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:18.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:18.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:18.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:18.754 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:18 smithi181 ceph-mon[196979]: pgmap v622: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 41 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2952/2013 objects degraded (146.647%) 2023-10-09T04:13:18.756 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:18 smithi163 ceph-mon[221067]: pgmap v622: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 41 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2952/2013 objects degraded (146.647%) 2023-10-09T04:13:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:18 smithi163 ceph-mon[213841]: pgmap v622: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 41 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2952/2013 objects degraded (146.647%) 2023-10-09T04:13:18.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:18.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:18.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:18.995 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:19.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:19.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:19.153 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:19.182 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:19.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:19.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:19.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:19.368 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:19.402 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:19.402 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:13:19.402 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:13:19.402 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:19.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:19.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:19.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:19.547 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:19.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:19.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:19.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:19.823 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:20.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:20.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:20.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:13:20.128 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:20.155 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:20.202 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:20.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:20.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:20.337 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:20.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:20.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:20.451 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:20.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:20.522 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:20.567 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:20.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:20 smithi181 ceph-mon[196979]: pgmap v623: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 6.1 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 2950/339 objects degraded (870.206%); 849 KiB/s, 83 objects/s recovering 2023-10-09T04:13:20.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:20.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:20 smithi163 ceph-mon[213841]: pgmap v623: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 6.1 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 2950/339 objects degraded (870.206%); 849 KiB/s, 83 objects/s recovering 2023-10-09T04:13:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:20 smithi163 ceph-mon[221067]: pgmap v623: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 6.1 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 2950/339 objects degraded (870.206%); 849 KiB/s, 83 objects/s recovering 2023-10-09T04:13:20.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:20.698 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:20.750 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:20.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:20.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:20.883 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:20.930 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:20.931 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:13:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:13:20] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:13:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:20.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:21.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:21.062 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:21.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:21.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:21.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:21.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:21.240 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:21.245 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T04:13:21.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T04:13:21.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:21.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:21.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:21.430 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:21.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:21.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:21.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:21.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:21.618 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:21.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:21.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:21.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:21.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:21.902 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:21.938 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:21.938 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:13:21.939 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:13:21.939 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:13:21.939 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:22.009 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":620,"stamp":"2023-10-09T04:13:13.519514+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":147691448,"num_objects":2270,"num_object_clones":0,"num_object_copies":6810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2270,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":155453,"num_write_kb":4822318,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13670,"num_bytes_recovered":835390132,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":186169,"ondisk_log_size":186169,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3295744,"kb_used_data":758656,"kb_used_omap":126,"kb_used_meta":2536385,"kb_avail":746665472,"statfs":{"total":767960285184,"available":764585443328,"internally_reserved":0,"allocated":776863744,"data_stored":770429933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2597258875},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,16],"upper_bound":2048},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-98107392,"num_objects":-1497,"num_object_clones":0,"num_object_copies":-4491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-9813,"num_write_kb":-368384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1017,"num_bytes_recovered":-76742656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.796160"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":230,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734265+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:13:08.734265+0000","last_peered":"2023-10-09T04:13:08.734265+0000","last_clean":"2023-10-09T04:13:08.734265+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:13:08.734265+0000","last_undegraded":"2023-10-09T04:13:08.734265+0000","last_fullsized":"2023-10-09T04:13:08.734265+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":167,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.824213+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:13:06.824213+0000","last_peered":"2023-10-09T04:13:06.824213+0000","last_clean":"2023-10-09T04:13:06.824213+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:13:06.824213+0000","last_undegraded":"2023-10-09T04:13:06.824213+0000","last_fullsized":"2023-10-09T04:13:06.824213+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":182,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.825804+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:13:06.825804+0000","last_peered":"2023-10-09T04:13:06.825804+0000","last_clean":"2023-10-09T04:13:06.825804+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:13:06.825804+0000","last_undegraded":"2023-10-09T04:13:06.825804+0000","last_fullsized":"2023-10-09T04:13:06.825804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":191,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.787050+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:13:10.787050+0000","last_peered":"2023-10-09T04:13:10.787050+0000","last_clean":"2023-10-09T04:13:10.787050+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:13:10.787050+0000","last_undegraded":"2023-10-09T04:13:10.787050+0000","last_fullsized":"2023-10-09T04:13:10.787050+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785133+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:13:10.785133+0000","last_peered":"2023-10-09T04:13:10.785133+0000","last_clean":"2023-10-09T04:13:10.785133+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:13:10.785133+0000","last_undegraded":"2023-10-09T04:13:10.785133+0000","last_fullsized":"2023-10-09T04:13:10.785133+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":166,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725249+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:13:08.725249+0000","last_peered":"2023-10-09T04:13:08.725249+0000","last_clean":"2023-10-09T04:13:08.725249+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:13:08.725249+0000","last_undegraded":"2023-10-09T04:13:08.725249+0000","last_fullsized":"2023-10-09T04:13:08.725249+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.13","version":"232'6855","reported_seq":10925,"reported_epoch":233,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:13:10.123697+0000","last_change":"2023-10-09T04:13:10.123836+0000","last_active":"2023-10-09T04:13:10.123697+0000","last_peered":"2023-10-09T04:13:10.123697+0000","last_clean":"2023-10-09T04:13:05.717873+0000","last_became_active":"2023-10-09T04:13:07.956121+0000","last_became_peered":"2023-10-09T04:13:07.956121+0000","last_unstale":"2023-10-09T04:13:10.123697+0000","last_undegraded":"2023-10-09T04:13:10.123697+0000","last_fullsized":"2023-10-09T04:13:07.721493+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6855,"log_dups_size":0,"ondisk_log_size":6855,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6815744,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5684,"num_write_kb":185216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1847,"num_bytes_recovered":112590848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[2,0],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"0,2","objects":104}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":190,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785874+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:13:10.785874+0000","last_peered":"2023-10-09T04:13:10.785874+0000","last_clean":"2023-10-09T04:13:10.785874+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:13:10.785874+0000","last_undegraded":"2023-10-09T04:13:10.785874+0000","last_fullsized":"2023-10-09T04:13:10.785874+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":213,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:09.841422+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:13:09.841422+0000","last_peered":"2023-10-09T04:13:09.841422+0000","last_clean":"2023-10-09T04:13:09.841422+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:13:09.841422+0000","last_undegraded":"2023-10-09T04:13:09.841422+0000","last_fullsized":"2023-10-09T04:13:09.841422+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"235'7069","reported_seq":14484,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.179139+0000","last_change":"2023-10-09T04:12:19.342281+0000","last_active":"2023-10-09T04:13:13.179139+0000","last_peered":"2023-10-09T04:13:13.179139+0000","last_clean":"2023-10-09T04:13:13.179139+0000","last_became_active":"2023-10-09T04:12:19.328982+0000","last_became_peered":"2023-10-09T04:12:19.328982+0000","last_unstale":"2023-10-09T04:13:13.179139+0000","last_undegraded":"2023-10-09T04:13:13.179139+0000","last_fullsized":"2023-10-09T04:13:13.179139+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5904,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2553,"num_bytes_recovered":167313408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725246+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:13:08.725246+0000","last_peered":"2023-10-09T04:13:08.725246+0000","last_clean":"2023-10-09T04:13:08.725246+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:13:08.725246+0000","last_undegraded":"2023-10-09T04:13:08.725246+0000","last_fullsized":"2023-10-09T04:13:08.725246+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":129,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733363+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:13:08.733363+0000","last_peered":"2023-10-09T04:13:08.733363+0000","last_clean":"2023-10-09T04:13:08.733363+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:13:08.733363+0000","last_undegraded":"2023-10-09T04:13:08.733363+0000","last_fullsized":"2023-10-09T04:13:08.733363+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.11","version":"233'6838","reported_seq":9770,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.405579+0000","last_change":"2023-10-09T04:13:06.865594+0000","last_active":"2023-10-09T04:13:10.405579+0000","last_peered":"2023-10-09T04:13:10.405579+0000","last_clean":"2023-10-09T04:13:10.405579+0000","last_became_active":"2023-10-09T04:10:18.269352+0000","last_became_peered":"2023-10-09T04:10:18.269352+0000","last_unstale":"2023-10-09T04:13:10.405579+0000","last_undegraded":"2023-10-09T04:13:10.405579+0000","last_fullsized":"2023-10-09T04:13:10.405579+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":5,"last_scrub":"229'6812","last_scrub_stamp":"2023-10-09T04:13:06.865533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:13:06.865533+0000","objects_scrubbed":110,"log_size":6838,"log_dups_size":0,"ondisk_log_size":6838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:34.935890+0000","scrub_duration":0.020894764,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5505024,"num_objects":84,"num_object_clones":0,"num_object_copies":252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":195,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733548+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:13:08.733548+0000","last_peered":"2023-10-09T04:13:08.733548+0000","last_clean":"2023-10-09T04:13:08.733548+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:13:08.733548+0000","last_undegraded":"2023-10-09T04:13:08.733548+0000","last_fullsized":"2023-10-09T04:13:08.733548+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":183,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.823951+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:13:06.823951+0000","last_peered":"2023-10-09T04:13:06.823951+0000","last_clean":"2023-10-09T04:13:06.823951+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:13:06.823951+0000","last_undegraded":"2023-10-09T04:13:06.823951+0000","last_fullsized":"2023-10-09T04:13:06.823951+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"232'7014","reported_seq":9969,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.830396+0000","last_change":"2023-10-09T04:10:34.014072+0000","last_active":"2023-10-09T04:13:08.830396+0000","last_peered":"2023-10-09T04:13:08.830396+0000","last_clean":"2023-10-09T04:13:08.830396+0000","last_became_active":"2023-10-09T04:10:04.892129+0000","last_became_peered":"2023-10-09T04:10:04.892129+0000","last_unstale":"2023-10-09T04:13:08.830396+0000","last_undegraded":"2023-10-09T04:13:08.830396+0000","last_fullsized":"2023-10-09T04:13:08.830396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4653056,"num_objects":71,"num_object_clones":0,"num_object_copies":213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5825,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1425,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":197,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.735760+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:13:08.735760+0000","last_peered":"2023-10-09T04:13:08.735760+0000","last_clean":"2023-10-09T04:13:08.735760+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:13:08.735760+0000","last_undegraded":"2023-10-09T04:13:08.735760+0000","last_fullsized":"2023-10-09T04:13:08.735760+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.735386+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:13:08.735386+0000","last_peered":"2023-10-09T04:13:08.735386+0000","last_clean":"2023-10-09T04:13:08.735386+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:13:08.735386+0000","last_undegraded":"2023-10-09T04:13:08.735386+0000","last_fullsized":"2023-10-09T04:13:08.735386+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"235'6941","reported_seq":7029,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.012282+0000","last_change":"2023-10-09T04:09:49.271262+0000","last_active":"2023-10-09T04:13:13.012282+0000","last_peered":"2023-10-09T04:13:13.012282+0000","last_clean":"2023-10-09T04:13:13.012282+0000","last_became_active":"2023-10-09T04:09:49.270155+0000","last_became_peered":"2023-10-09T04:09:49.270155+0000","last_unstale":"2023-10-09T04:13:13.012282+0000","last_undegraded":"2023-10-09T04:13:13.012282+0000","last_fullsized":"2023-10-09T04:13:13.012282+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6941,"log_dups_size":0,"ondisk_log_size":6941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3735552,"num_objects":57,"num_object_clones":0,"num_object_copies":171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":194,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733734+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:13:08.733734+0000","last_peered":"2023-10-09T04:13:08.733734+0000","last_clean":"2023-10-09T04:13:08.733734+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:13:08.733734+0000","last_undegraded":"2023-10-09T04:13:08.733734+0000","last_fullsized":"2023-10-09T04:13:08.733734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":167,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.824074+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:13:06.824074+0000","last_peered":"2023-10-09T04:13:06.824074+0000","last_clean":"2023-10-09T04:13:06.824074+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:13:06.824074+0000","last_undegraded":"2023-10-09T04:13:06.824074+0000","last_fullsized":"2023-10-09T04:13:06.824074+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"235'6745","reported_seq":6832,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.098733+0000","last_change":"2023-10-09T04:09:49.272246+0000","last_active":"2023-10-09T04:13:13.098733+0000","last_peered":"2023-10-09T04:13:13.098733+0000","last_clean":"2023-10-09T04:13:13.098733+0000","last_became_active":"2023-10-09T04:09:49.270950+0000","last_became_peered":"2023-10-09T04:09:49.270950+0000","last_unstale":"2023-10-09T04:13:13.098733+0000","last_undegraded":"2023-10-09T04:13:13.098733+0000","last_fullsized":"2023-10-09T04:13:13.098733+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6745,"log_dups_size":0,"ondisk_log_size":6745,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1146880,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5628,"num_write_kb":180640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":190,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785686+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:13:10.785686+0000","last_peered":"2023-10-09T04:13:10.785686+0000","last_clean":"2023-10-09T04:13:10.785686+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:13:10.785686+0000","last_undegraded":"2023-10-09T04:13:10.785686+0000","last_fullsized":"2023-10-09T04:13:10.785686+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725139+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:13:08.725139+0000","last_peered":"2023-10-09T04:13:08.725139+0000","last_clean":"2023-10-09T04:13:08.725139+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:13:08.725139+0000","last_undegraded":"2023-10-09T04:13:08.725139+0000","last_fullsized":"2023-10-09T04:13:08.725139+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"232'6933","reported_seq":10339,"reported_epoch":233,"state":"peering","last_fresh":"2023-10-09T04:13:09.739333+0000","last_change":"2023-10-09T04:13:09.739920+0000","last_active":"2023-10-09T04:13:09.739282+0000","last_peered":"2023-10-09T04:13:08.838681+0000","last_clean":"2023-10-09T04:13:08.838681+0000","last_became_active":"2023-10-09T04:13:02.677165+0000","last_became_peered":"2023-10-09T04:13:02.677165+0000","last_unstale":"2023-10-09T04:13:09.739333+0000","last_undegraded":"2023-10-09T04:13:09.739333+0000","last_fullsized":"2023-10-09T04:13:09.739333+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6933,"log_dups_size":0,"ondisk_log_size":6933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5603328,"num_objects":86,"num_object_clones":0,"num_object_copies":258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5818,"num_write_kb":188896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1637,"num_bytes_recovered":107282432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":175,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733274+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:13:08.733274+0000","last_peered":"2023-10-09T04:13:08.733274+0000","last_clean":"2023-10-09T04:13:08.733274+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:13:08.733274+0000","last_undegraded":"2023-10-09T04:13:08.733274+0000","last_fullsized":"2023-10-09T04:13:08.733274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":190,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.787116+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:13:10.787116+0000","last_peered":"2023-10-09T04:13:10.787116+0000","last_clean":"2023-10-09T04:13:10.787116+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:13:10.787116+0000","last_undegraded":"2023-10-09T04:13:10.787116+0000","last_fullsized":"2023-10-09T04:13:10.787116+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"235'6996","reported_seq":7656,"reported_epoch":235,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:13:13.191210+0000","last_change":"2023-10-09T04:13:13.190877+0000","last_active":"2023-10-09T04:13:13.191210+0000","last_peered":"2023-10-09T04:13:13.191210+0000","last_clean":"2023-10-09T04:13:07.260861+0000","last_became_active":"2023-10-09T04:13:10.093194+0000","last_became_peered":"2023-10-09T04:13:10.093194+0000","last_unstale":"2023-10-09T04:13:13.191210+0000","last_undegraded":"2023-10-09T04:13:13.191210+0000","last_fullsized":"2023-10-09T04:13:09.735390+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4849664,"num_objects":74,"num_object_clones":0,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5834,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":7602176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[0,2],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,2","objects":73},{"shards":"3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":126,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.784982+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:13:10.784982+0000","last_peered":"2023-10-09T04:13:10.784982+0000","last_clean":"2023-10-09T04:13:10.784982+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:13:10.784982+0000","last_undegraded":"2023-10-09T04:13:10.784982+0000","last_fullsized":"2023-10-09T04:13:10.784982+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":198,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:09.842844+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:13:09.842844+0000","last_peered":"2023-10-09T04:13:09.842844+0000","last_clean":"2023-10-09T04:13:09.842844+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:13:09.842844+0000","last_undegraded":"2023-10-09T04:13:09.842844+0000","last_fullsized":"2023-10-09T04:13:09.842844+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"231'11480","reported_seq":11554,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.674946+0000","last_change":"2023-10-09T04:09:20.476643+0000","last_active":"2023-10-09T04:13:08.674946+0000","last_peered":"2023-10-09T04:13:08.674946+0000","last_clean":"2023-10-09T04:13:08.674946+0000","last_became_active":"2023-10-09T04:09:20.476349+0000","last_became_peered":"2023-10-09T04:09:20.476349+0000","last_unstale":"2023-10-09T04:13:08.674946+0000","last_undegraded":"2023-10-09T04:13:08.674946+0000","last_fullsized":"2023-10-09T04:13:08.674946+0000","mapping_epoch":167,"log_start":"195'4500","ondisk_log_start":"195'4500","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:39.435927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12845056,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11480,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":193,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733828+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:13:08.733828+0000","last_peered":"2023-10-09T04:13:08.733828+0000","last_clean":"2023-10-09T04:13:08.733828+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:13:08.733828+0000","last_undegraded":"2023-10-09T04:13:08.733828+0000","last_fullsized":"2023-10-09T04:13:08.733828+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":174,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.724844+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:13:08.724844+0000","last_peered":"2023-10-09T04:13:08.724844+0000","last_clean":"2023-10-09T04:13:08.724844+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:13:08.724844+0000","last_undegraded":"2023-10-09T04:13:08.724844+0000","last_fullsized":"2023-10-09T04:13:08.724844+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"232'11386","reported_seq":11462,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.806452+0000","last_change":"2023-10-09T04:09:20.476821+0000","last_active":"2023-10-09T04:13:08.806452+0000","last_peered":"2023-10-09T04:13:08.806452+0000","last_clean":"2023-10-09T04:13:08.806452+0000","last_became_active":"2023-10-09T04:09:20.476582+0000","last_became_peered":"2023-10-09T04:09:20.476582+0000","last_unstale":"2023-10-09T04:13:08.806452+0000","last_undegraded":"2023-10-09T04:13:08.806452+0000","last_fullsized":"2023-10-09T04:13:08.806452+0000","mapping_epoch":167,"log_start":"172'1672","ondisk_log_start":"172'1672","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":9714,"log_dups_size":0,"ondisk_log_size":9714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.451583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11534336,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11386,"num_write_kb":369984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":194,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733517+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:13:08.733517+0000","last_peered":"2023-10-09T04:13:08.733517+0000","last_clean":"2023-10-09T04:13:08.733517+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:13:08.733517+0000","last_undegraded":"2023-10-09T04:13:08.733517+0000","last_fullsized":"2023-10-09T04:13:08.733517+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":208,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734353+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:13:08.734353+0000","last_peered":"2023-10-09T04:13:08.734353+0000","last_clean":"2023-10-09T04:13:08.734353+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:13:08.734353+0000","last_undegraded":"2023-10-09T04:13:08.734353+0000","last_fullsized":"2023-10-09T04:13:08.734353+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"235'7018","reported_seq":7213,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.089656+0000","last_change":"2023-10-09T04:09:51.630635+0000","last_active":"2023-10-09T04:13:13.089656+0000","last_peered":"2023-10-09T04:13:13.089656+0000","last_clean":"2023-10-09T04:13:13.089656+0000","last_became_active":"2023-10-09T04:09:49.252985+0000","last_became_peered":"2023-10-09T04:09:49.252985+0000","last_unstale":"2023-10-09T04:13:13.089656+0000","last_undegraded":"2023-10-09T04:13:13.089656+0000","last_fullsized":"2023-10-09T04:13:13.089656+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2424","last_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_deep_scrub":"175'2424","last_deep_scrub_stamp":"2023-10-09T04:09:51.630481+0000","last_clean_scrub_stamp":"2023-10-09T04:09:51.630481+0000","objects_scrubbed":1212,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:19.768518+0000","scrub_duration":1.797550314,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2621440,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4632,"num_write_kb":110720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":176,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733470+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:13:08.733470+0000","last_peered":"2023-10-09T04:13:08.733470+0000","last_clean":"2023-10-09T04:13:08.733470+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:13:08.733470+0000","last_undegraded":"2023-10-09T04:13:08.733470+0000","last_fullsized":"2023-10-09T04:13:08.733470+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":150,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.823679+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:13:06.823679+0000","last_peered":"2023-10-09T04:13:06.823679+0000","last_clean":"2023-10-09T04:13:06.823679+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:13:06.823679+0000","last_undegraded":"2023-10-09T04:13:06.823679+0000","last_fullsized":"2023-10-09T04:13:06.823679+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"232'6770","reported_seq":6872,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.773671+0000","last_change":"2023-10-09T04:13:07.501725+0000","last_active":"2023-10-09T04:13:08.773671+0000","last_peered":"2023-10-09T04:13:08.773671+0000","last_clean":"2023-10-09T04:13:08.773671+0000","last_became_active":"2023-10-09T04:09:49.251915+0000","last_became_peered":"2023-10-09T04:09:49.251915+0000","last_unstale":"2023-10-09T04:13:08.773671+0000","last_undegraded":"2023-10-09T04:13:08.773671+0000","last_fullsized":"2023-10-09T04:13:08.773671+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"230'6759","last_scrub_stamp":"2023-10-09T04:13:07.501510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:13:07.501510+0000","objects_scrubbed":92,"log_size":6770,"log_dups_size":0,"ondisk_log_size":6770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:25.603706+0000","scrub_duration":0.011838945999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5308416,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725072+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:13:08.725072+0000","last_peered":"2023-10-09T04:13:08.725072+0000","last_clean":"2023-10-09T04:13:08.725072+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:13:08.725072+0000","last_undegraded":"2023-10-09T04:13:08.725072+0000","last_fullsized":"2023-10-09T04:13:08.725072+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":190,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785904+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:13:10.785904+0000","last_peered":"2023-10-09T04:13:10.785904+0000","last_clean":"2023-10-09T04:13:10.785904+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:13:10.785904+0000","last_undegraded":"2023-10-09T04:13:10.785904+0000","last_fullsized":"2023-10-09T04:13:10.785904+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"235'11597","reported_seq":11675,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.188495+0000","last_change":"2023-10-09T04:09:20.477908+0000","last_active":"2023-10-09T04:13:13.188495+0000","last_peered":"2023-10-09T04:13:13.188495+0000","last_clean":"2023-10-09T04:13:13.188495+0000","last_became_active":"2023-10-09T04:09:20.477627+0000","last_became_peered":"2023-10-09T04:09:20.477627+0000","last_unstale":"2023-10-09T04:13:13.188495+0000","last_undegraded":"2023-10-09T04:13:13.188495+0000","last_fullsized":"2023-10-09T04:13:13.188495+0000","mapping_epoch":167,"log_start":"193'4224","ondisk_log_start":"193'4224","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:59.207541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4259840,"num_objects":65,"num_object_clones":0,"num_object_copies":195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11597,"num_write_kb":373184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":139,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.786217+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:13:10.786217+0000","last_peered":"2023-10-09T04:13:10.786217+0000","last_clean":"2023-10-09T04:13:10.786217+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:13:10.786217+0000","last_undegraded":"2023-10-09T04:13:10.786217+0000","last_fullsized":"2023-10-09T04:13:10.786217+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.735703+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:13:08.735703+0000","last_peered":"2023-10-09T04:13:08.735703+0000","last_clean":"2023-10-09T04:13:08.735703+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:13:08.735703+0000","last_undegraded":"2023-10-09T04:13:08.735703+0000","last_fullsized":"2023-10-09T04:13:08.735703+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"232'11425","reported_seq":15924,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.891403+0000","last_change":"2023-10-09T04:13:00.800740+0000","last_active":"2023-10-09T04:13:08.891403+0000","last_peered":"2023-10-09T04:13:08.891403+0000","last_clean":"2023-10-09T04:13:08.891403+0000","last_became_active":"2023-10-09T04:13:00.671385+0000","last_became_peered":"2023-10-09T04:13:00.671385+0000","last_unstale":"2023-10-09T04:13:08.891403+0000","last_undegraded":"2023-10-09T04:13:08.891403+0000","last_fullsized":"2023-10-09T04:13:08.891403+0000","mapping_epoch":224,"log_start":"172'1727","ondisk_log_start":"172'1727","created":167,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":9698,"log_dups_size":0,"ondisk_log_size":9698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:12.769316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12648448,"num_objects":193,"num_object_clones":0,"num_object_copies":579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11425,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2198,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":152,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734356+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:13:08.734356+0000","last_peered":"2023-10-09T04:13:08.734356+0000","last_clean":"2023-10-09T04:13:08.734356+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:13:08.734356+0000","last_undegraded":"2023-10-09T04:13:08.734356+0000","last_fullsized":"2023-10-09T04:13:08.734356+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":228,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734465+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:13:08.734465+0000","last_peered":"2023-10-09T04:13:08.734465+0000","last_clean":"2023-10-09T04:13:08.734465+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:13:08.734465+0000","last_undegraded":"2023-10-09T04:13:08.734465+0000","last_fullsized":"2023-10-09T04:13:08.734465+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"231'11431","reported_seq":11506,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.700314+0000","last_change":"2023-10-09T04:09:20.479628+0000","last_active":"2023-10-09T04:13:08.700314+0000","last_peered":"2023-10-09T04:13:08.700314+0000","last_clean":"2023-10-09T04:13:08.700314+0000","last_became_active":"2023-10-09T04:09:20.479178+0000","last_became_peered":"2023-10-09T04:09:20.479178+0000","last_unstale":"2023-10-09T04:13:08.700314+0000","last_undegraded":"2023-10-09T04:13:08.700314+0000","last_fullsized":"2023-10-09T04:13:08.700314+0000","mapping_epoch":167,"log_start":"195'4400","ondisk_log_start":"195'4400","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:42.732279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12320792,"num_objects":189,"num_object_clones":0,"num_object_copies":567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":189,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11431,"num_write_kb":371777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734188+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:13:08.734188+0000","last_peered":"2023-10-09T04:13:08.734188+0000","last_clean":"2023-10-09T04:13:08.734188+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:13:08.734188+0000","last_undegraded":"2023-10-09T04:13:08.734188+0000","last_fullsized":"2023-10-09T04:13:08.734188+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":184,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734851+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:13:08.734851+0000","last_peered":"2023-10-09T04:13:08.734851+0000","last_clean":"2023-10-09T04:13:08.734851+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:13:08.734851+0000","last_undegraded":"2023-10-09T04:13:08.734851+0000","last_fullsized":"2023-10-09T04:13:08.734851+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"234'11268","reported_seq":11345,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.772105+0000","last_change":"2023-10-09T04:09:20.477492+0000","last_active":"2023-10-09T04:13:10.772105+0000","last_peered":"2023-10-09T04:13:10.772105+0000","last_clean":"2023-10-09T04:13:10.772105+0000","last_became_active":"2023-10-09T04:09:20.477229+0000","last_became_peered":"2023-10-09T04:09:20.477229+0000","last_unstale":"2023-10-09T04:13:10.772105+0000","last_undegraded":"2023-10-09T04:13:10.772105+0000","last_fullsized":"2023-10-09T04:13:10.772105+0000","mapping_epoch":167,"log_start":"196'4800","ondisk_log_start":"196'4800","created":167,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:14.780410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11268,"num_write_kb":365760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:09.841220+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:13:09.841220+0000","last_peered":"2023-10-09T04:13:09.841220+0000","last_clean":"2023-10-09T04:13:09.841220+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:13:09.841220+0000","last_undegraded":"2023-10-09T04:13:09.841220+0000","last_fullsized":"2023-10-09T04:13:09.841220+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":183,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.823805+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:13:06.823805+0000","last_peered":"2023-10-09T04:13:06.823805+0000","last_clean":"2023-10-09T04:13:06.823805+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:13:06.823805+0000","last_undegraded":"2023-10-09T04:13:06.823805+0000","last_fullsized":"2023-10-09T04:13:06.823805+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"233'6911","reported_seq":7119,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.287917+0000","last_change":"2023-10-09T04:10:38.375108+0000","last_active":"2023-10-09T04:13:10.287917+0000","last_peered":"2023-10-09T04:13:10.287917+0000","last_clean":"2023-10-09T04:13:10.287917+0000","last_became_active":"2023-10-09T04:09:49.264241+0000","last_became_peered":"2023-10-09T04:09:49.264241+0000","last_unstale":"2023-10-09T04:13:10.287917+0000","last_undegraded":"2023-10-09T04:13:10.287917+0000","last_fullsized":"2023-10-09T04:13:10.287917+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"196'3653","last_scrub_stamp":"2023-10-09T04:10:38.375032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:10:38.375032+0000","objects_scrubbed":2511,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:54:26.443071+0000","scrub_duration":0.28747155200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5701632,"num_objects":87,"num_object_clones":0,"num_object_copies":261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3258,"num_write_kb":26688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":207,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.824207+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:13:06.824207+0000","last_peered":"2023-10-09T04:13:06.824207+0000","last_clean":"2023-10-09T04:13:06.824207+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:13:06.824207+0000","last_undegraded":"2023-10-09T04:13:06.824207+0000","last_fullsized":"2023-10-09T04:13:06.824207+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":209,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725165+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:13:08.725165+0000","last_peered":"2023-10-09T04:13:08.725165+0000","last_clean":"2023-10-09T04:13:08.725165+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:13:08.725165+0000","last_undegraded":"2023-10-09T04:13:08.725165+0000","last_fullsized":"2023-10-09T04:13:08.725165+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":169,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.823820+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:13:06.823820+0000","last_peered":"2023-10-09T04:13:06.823820+0000","last_clean":"2023-10-09T04:13:06.823820+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:13:06.823820+0000","last_undegraded":"2023-10-09T04:13:06.823820+0000","last_fullsized":"2023-10-09T04:13:06.823820+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"235'6978","reported_seq":7078,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.070830+0000","last_change":"2023-10-09T04:13:07.896983+0000","last_active":"2023-10-09T04:13:13.070830+0000","last_peered":"2023-10-09T04:13:13.070830+0000","last_clean":"2023-10-09T04:13:13.070830+0000","last_became_active":"2023-10-09T04:09:49.252296+0000","last_became_peered":"2023-10-09T04:09:49.252296+0000","last_unstale":"2023-10-09T04:13:13.070830+0000","last_undegraded":"2023-10-09T04:13:13.070830+0000","last_fullsized":"2023-10-09T04:13:13.070830+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"230'6932","last_scrub_stamp":"2023-10-09T04:13:07.896733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:13:07.896733+0000","objects_scrubbed":85,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:34.497242+0000","scrub_duration":0.013483225,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":196,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725049+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:13:08.725049+0000","last_peered":"2023-10-09T04:13:08.725049+0000","last_clean":"2023-10-09T04:13:08.725049+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:13:08.725049+0000","last_undegraded":"2023-10-09T04:13:08.725049+0000","last_fullsized":"2023-10-09T04:13:08.725049+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":192,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.735429+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:13:08.735429+0000","last_peered":"2023-10-09T04:13:08.735429+0000","last_clean":"2023-10-09T04:13:08.735429+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:13:08.735429+0000","last_undegraded":"2023-10-09T04:13:08.735429+0000","last_fullsized":"2023-10-09T04:13:08.735429+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":217,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.824144+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:13:06.824144+0000","last_peered":"2023-10-09T04:13:06.824144+0000","last_clean":"2023-10-09T04:13:06.824144+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:13:06.824144+0000","last_undegraded":"2023-10-09T04:13:06.824144+0000","last_fullsized":"2023-10-09T04:13:06.824144+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"235'7024","reported_seq":7112,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.070658+0000","last_change":"2023-10-09T04:09:49.258735+0000","last_active":"2023-10-09T04:13:13.070658+0000","last_peered":"2023-10-09T04:13:13.070658+0000","last_clean":"2023-10-09T04:13:13.070658+0000","last_became_active":"2023-10-09T04:09:49.257296+0000","last_became_peered":"2023-10-09T04:09:49.257296+0000","last_unstale":"2023-10-09T04:13:13.070658+0000","last_undegraded":"2023-10-09T04:13:13.070658+0000","last_fullsized":"2023-10-09T04:13:13.070658+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1441792,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5862,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":47,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.724733+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:13:08.724733+0000","last_peered":"2023-10-09T04:13:08.724733+0000","last_clean":"2023-10-09T04:13:08.724733+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:13:08.724733+0000","last_undegraded":"2023-10-09T04:13:08.724733+0000","last_fullsized":"2023-10-09T04:13:08.724733+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":201,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.823545+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:13:06.823545+0000","last_peered":"2023-10-09T04:13:06.823545+0000","last_clean":"2023-10-09T04:13:06.823545+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:13:06.823545+0000","last_undegraded":"2023-10-09T04:13:06.823545+0000","last_fullsized":"2023-10-09T04:13:06.823545+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":201,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.823871+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:13:06.823871+0000","last_peered":"2023-10-09T04:13:06.823871+0000","last_clean":"2023-10-09T04:13:06.823871+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:13:06.823871+0000","last_undegraded":"2023-10-09T04:13:06.823871+0000","last_fullsized":"2023-10-09T04:13:06.823871+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"235'6800","reported_seq":6901,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.102619+0000","last_change":"2023-10-09T04:13:05.894578+0000","last_active":"2023-10-09T04:13:13.102619+0000","last_peered":"2023-10-09T04:13:13.102619+0000","last_clean":"2023-10-09T04:13:13.102619+0000","last_became_active":"2023-10-09T04:09:49.257825+0000","last_became_peered":"2023-10-09T04:09:49.257825+0000","last_unstale":"2023-10-09T04:13:13.102619+0000","last_undegraded":"2023-10-09T04:13:13.102619+0000","last_fullsized":"2023-10-09T04:13:13.102619+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"229'6733","last_scrub_stamp":"2023-10-09T04:13:05.894356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:13:05.894356+0000","objects_scrubbed":103,"log_size":6800,"log_dups_size":0,"ondisk_log_size":6800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:10.284703+0000","scrub_duration":0.076933752999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2359296,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":67,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":216,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.724998+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:13:08.724998+0000","last_peered":"2023-10-09T04:13:08.724998+0000","last_clean":"2023-10-09T04:13:08.724998+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:13:08.724998+0000","last_undegraded":"2023-10-09T04:13:08.724998+0000","last_fullsized":"2023-10-09T04:13:08.724998+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":573,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733174+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:13:08.733174+0000","last_peered":"2023-10-09T04:13:08.733174+0000","last_clean":"2023-10-09T04:13:08.733174+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:13:08.733174+0000","last_undegraded":"2023-10-09T04:13:08.733174+0000","last_fullsized":"2023-10-09T04:13:08.733174+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":129,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.724991+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:13:08.724991+0000","last_peered":"2023-10-09T04:13:08.724991+0000","last_clean":"2023-10-09T04:13:08.724991+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:13:08.724991+0000","last_undegraded":"2023-10-09T04:13:08.724991+0000","last_fullsized":"2023-10-09T04:13:08.724991+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":112,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734671+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:13:08.734671+0000","last_peered":"2023-10-09T04:13:08.734671+0000","last_clean":"2023-10-09T04:13:08.734671+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:13:08.734671+0000","last_undegraded":"2023-10-09T04:13:08.734671+0000","last_fullsized":"2023-10-09T04:13:08.734671+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"233'6809","reported_seq":9188,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.598038+0000","last_change":"2023-10-09T04:12:37.913403+0000","last_active":"2023-10-09T04:13:10.598038+0000","last_peered":"2023-10-09T04:13:10.598038+0000","last_clean":"2023-10-09T04:13:10.598038+0000","last_became_active":"2023-10-09T04:10:53.272530+0000","last_became_peered":"2023-10-09T04:10:53.272530+0000","last_unstale":"2023-10-09T04:13:10.598038+0000","last_undegraded":"2023-10-09T04:13:10.598038+0000","last_fullsized":"2023-10-09T04:13:10.598038+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6809,"log_dups_size":0,"ondisk_log_size":6809,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5341184,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5693,"num_write_kb":184800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1122,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":213,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733166+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:13:08.733166+0000","last_peered":"2023-10-09T04:13:08.733166+0000","last_clean":"2023-10-09T04:13:08.733166+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:13:08.733166+0000","last_undegraded":"2023-10-09T04:13:08.733166+0000","last_fullsized":"2023-10-09T04:13:08.733166+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":191,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785393+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:13:10.785393+0000","last_peered":"2023-10-09T04:13:10.785393+0000","last_clean":"2023-10-09T04:13:10.785393+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:13:10.785393+0000","last_undegraded":"2023-10-09T04:13:10.785393+0000","last_fullsized":"2023-10-09T04:13:10.785393+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":224,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.786841+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:13:10.786841+0000","last_peered":"2023-10-09T04:13:10.786841+0000","last_clean":"2023-10-09T04:13:10.786841+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:13:10.786841+0000","last_undegraded":"2023-10-09T04:13:10.786841+0000","last_fullsized":"2023-10-09T04:13:10.786841+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"233'7065","reported_seq":7213,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.445073+0000","last_change":"2023-10-09T04:09:53.342919+0000","last_active":"2023-10-09T04:13:10.445073+0000","last_peered":"2023-10-09T04:13:10.445073+0000","last_clean":"2023-10-09T04:13:10.445073+0000","last_became_active":"2023-10-09T04:09:49.262905+0000","last_became_peered":"2023-10-09T04:09:49.262905+0000","last_unstale":"2023-10-09T04:13:10.445073+0000","last_undegraded":"2023-10-09T04:13:10.445073+0000","last_fullsized":"2023-10-09T04:13:10.445073+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2552","last_scrub_stamp":"2023-10-09T04:09:53.342838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:53.342838+0000","objects_scrubbed":1344,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:11.839527+0000","scrub_duration":0.32738803100000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6553600,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4518,"num_write_kb":104768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":196,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733501+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:13:08.733501+0000","last_peered":"2023-10-09T04:13:08.733501+0000","last_clean":"2023-10-09T04:13:08.733501+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:13:08.733501+0000","last_undegraded":"2023-10-09T04:13:08.733501+0000","last_fullsized":"2023-10-09T04:13:08.733501+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":201,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.824276+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:13:06.824276+0000","last_peered":"2023-10-09T04:13:06.824276+0000","last_clean":"2023-10-09T04:13:06.824276+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:13:06.824276+0000","last_undegraded":"2023-10-09T04:13:06.824276+0000","last_fullsized":"2023-10-09T04:13:06.824276+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":176,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785811+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:13:10.785811+0000","last_peered":"2023-10-09T04:13:10.785811+0000","last_clean":"2023-10-09T04:13:10.785811+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:13:10.785811+0000","last_undegraded":"2023-10-09T04:13:10.785811+0000","last_fullsized":"2023-10-09T04:13:10.785811+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"232'7085","reported_seq":10607,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.840775+0000","last_change":"2023-10-09T04:13:01.695288+0000","last_active":"2023-10-09T04:13:08.840775+0000","last_peered":"2023-10-09T04:13:08.840775+0000","last_clean":"2023-10-09T04:13:08.840775+0000","last_became_active":"2023-10-09T04:13:01.673219+0000","last_became_peered":"2023-10-09T04:13:01.673219+0000","last_unstale":"2023-10-09T04:13:08.840775+0000","last_undegraded":"2023-10-09T04:13:08.840775+0000","last_fullsized":"2023-10-09T04:13:08.840775+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7085,"log_dups_size":0,"ondisk_log_size":7085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4063232,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5914,"num_write_kb":191232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1699,"num_bytes_recovered":111345664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":137,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733620+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:13:08.733620+0000","last_peered":"2023-10-09T04:13:08.733620+0000","last_clean":"2023-10-09T04:13:08.733620+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:13:08.733620+0000","last_undegraded":"2023-10-09T04:13:08.733620+0000","last_fullsized":"2023-10-09T04:13:08.733620+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":167,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-10-09T04:13:06.825321+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:13:06.825321+0000","last_peered":"2023-10-09T04:13:06.825321+0000","last_clean":"2023-10-09T04:13:06.825321+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:13:06.825321+0000","last_undegraded":"2023-10-09T04:13:06.825321+0000","last_fullsized":"2023-10-09T04:13:06.825321+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":163,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733313+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:13:08.733313+0000","last_peered":"2023-10-09T04:13:08.733313+0000","last_clean":"2023-10-09T04:13:08.733313+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:13:08.733313+0000","last_undegraded":"2023-10-09T04:13:08.733313+0000","last_fullsized":"2023-10-09T04:13:08.733313+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"231'6898","reported_seq":9125,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.643201+0000","last_change":"2023-10-09T04:10:44.685973+0000","last_active":"2023-10-09T04:13:08.643201+0000","last_peered":"2023-10-09T04:13:08.643201+0000","last_clean":"2023-10-09T04:13:08.643201+0000","last_became_active":"2023-10-09T04:10:44.682373+0000","last_became_peered":"2023-10-09T04:10:44.682373+0000","last_unstale":"2023-10-09T04:13:08.643201+0000","last_undegraded":"2023-10-09T04:13:08.643201+0000","last_fullsized":"2023-10-09T04:13:08.643201+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":167,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"175'2488","last_scrub_stamp":"2023-10-09T04:09:52.498303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:52.498303+0000","objects_scrubbed":1318,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:51.283895+0000","scrub_duration":0.41788515500000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5832704,"num_objects":89,"num_object_clones":0,"num_object_copies":267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4421,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68026368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":161,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.733750+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:13:08.733750+0000","last_peered":"2023-10-09T04:13:08.733750+0000","last_clean":"2023-10-09T04:13:08.733750+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:13:08.733750+0000","last_undegraded":"2023-10-09T04:13:08.733750+0000","last_fullsized":"2023-10-09T04:13:08.733750+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":195,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.786926+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:13:10.786926+0000","last_peered":"2023-10-09T04:13:10.786926+0000","last_clean":"2023-10-09T04:13:10.786926+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:13:10.786926+0000","last_undegraded":"2023-10-09T04:13:10.786926+0000","last_fullsized":"2023-10-09T04:13:10.786926+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":174,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734273+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:13:08.734273+0000","last_peered":"2023-10-09T04:13:08.734273+0000","last_clean":"2023-10-09T04:13:08.734273+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:13:08.734273+0000","last_undegraded":"2023-10-09T04:13:08.734273+0000","last_fullsized":"2023-10-09T04:13:08.734273+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":179,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785985+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:13:10.785985+0000","last_peered":"2023-10-09T04:13:10.785985+0000","last_clean":"2023-10-09T04:13:10.785985+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:13:10.785985+0000","last_undegraded":"2023-10-09T04:13:10.785985+0000","last_fullsized":"2023-10-09T04:13:10.785985+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":47,"reported_epoch":233,"state":"active+clean","last_fresh":"2023-10-09T04:13:09.737307+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:13:09.737307+0000","last_peered":"2023-10-09T04:13:09.737307+0000","last_clean":"2023-10-09T04:13:09.737307+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:13:09.737307+0000","last_undegraded":"2023-10-09T04:13:09.737307+0000","last_fullsized":"2023-10-09T04:13:09.737307+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"232'6899","reported_seq":6987,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.763150+0000","last_change":"2023-10-09T04:10:30.081479+0000","last_active":"2023-10-09T04:13:08.763150+0000","last_peered":"2023-10-09T04:13:08.763150+0000","last_clean":"2023-10-09T04:13:08.763150+0000","last_became_active":"2023-10-09T04:09:49.267641+0000","last_became_peered":"2023-10-09T04:09:49.267641+0000","last_unstale":"2023-10-09T04:13:08.763150+0000","last_undegraded":"2023-10-09T04:13:08.763150+0000","last_fullsized":"2023-10-09T04:13:08.763150+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3866624,"num_objects":59,"num_object_clones":0,"num_object_copies":177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5759,"num_write_kb":186176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":194,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725157+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:13:08.725157+0000","last_peered":"2023-10-09T04:13:08.725157+0000","last_clean":"2023-10-09T04:13:08.725157+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:13:08.725157+0000","last_undegraded":"2023-10-09T04:13:08.725157+0000","last_fullsized":"2023-10-09T04:13:08.725157+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":208,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.735469+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:13:08.735469+0000","last_peered":"2023-10-09T04:13:08.735469+0000","last_clean":"2023-10-09T04:13:08.735469+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:13:08.735469+0000","last_undegraded":"2023-10-09T04:13:08.735469+0000","last_fullsized":"2023-10-09T04:13:08.735469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"235'7041","reported_seq":7126,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-10-09T04:13:13.057869+0000","last_change":"2023-10-09T04:09:49.274515+0000","last_active":"2023-10-09T04:13:13.057869+0000","last_peered":"2023-10-09T04:13:13.057869+0000","last_clean":"2023-10-09T04:13:13.057869+0000","last_became_active":"2023-10-09T04:09:49.273294+0000","last_became_peered":"2023-10-09T04:09:49.273294+0000","last_unstale":"2023-10-09T04:13:13.057869+0000","last_undegraded":"2023-10-09T04:13:13.057869+0000","last_fullsized":"2023-10-09T04:13:13.057869+0000","mapping_epoch":174,"log_start":"0'0","ondisk_log_start":"0'0","created":174,"last_epoch_clean":175,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:09:19.462808+0000","last_clean_scrub_stamp":"2023-10-09T04:09:19.462808+0000","objects_scrubbed":0,"log_size":7041,"log_dups_size":0,"ondisk_log_size":7041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5879,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":177,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.725234+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:13:08.725234+0000","last_peered":"2023-10-09T04:13:08.725234+0000","last_clean":"2023-10-09T04:13:08.725234+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:13:08.725234+0000","last_undegraded":"2023-10-09T04:13:08.725234+0000","last_fullsized":"2023-10-09T04:13:08.725234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":208,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734443+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:13:08.734443+0000","last_peered":"2023-10-09T04:13:08.734443+0000","last_clean":"2023-10-09T04:13:08.734443+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:13:08.734443+0000","last_undegraded":"2023-10-09T04:13:08.734443+0000","last_fullsized":"2023-10-09T04:13:08.734443+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":47,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.726987+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:13:08.726987+0000","last_peered":"2023-10-09T04:13:08.726987+0000","last_clean":"2023-10-09T04:13:08.726987+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:13:08.726987+0000","last_undegraded":"2023-10-09T04:13:08.726987+0000","last_fullsized":"2023-10-09T04:13:08.726987+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":47,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.724803+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:13:08.724803+0000","last_peered":"2023-10-09T04:13:08.724803+0000","last_clean":"2023-10-09T04:13:08.724803+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:13:08.724803+0000","last_undegraded":"2023-10-09T04:13:08.724803+0000","last_fullsized":"2023-10-09T04:13:08.724803+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":225,"reported_epoch":234,"state":"active+clean","last_fresh":"2023-10-09T04:13:10.785954+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:13:10.785954+0000","last_peered":"2023-10-09T04:13:10.785954+0000","last_clean":"2023-10-09T04:13:10.785954+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:13:10.785954+0000","last_undegraded":"2023-10-09T04:13:10.785954+0000","last_fullsized":"2023-10-09T04:13:10.785954+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":191,"reported_epoch":232,"state":"active+clean","last_fresh":"2023-10-09T04:13:08.734121+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:13:08.734121+0000","last_peered":"2023-10-09T04:13:08.734121+0000","last_clean":"2023-10-09T04:13:08.734121+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:13:08.734121+0000","last_undegraded":"2023-10-09T04:13:08.734121+0000","last_fullsized":"2023-10-09T04:13:08.734121+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":147095576,"num_objects":2246,"num_object_clones":0,"num_object_copies":6738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2246,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155153,"num_write_kb":4819873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13660,"num_bytes_recovered":833617920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":715010048,"data_stored":714997832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185953,"ondisk_log_size":185953,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201528,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":395236,"kb_used_data":85660,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93349916,"statfs":{"total":95995035648,"available":95590313984,"internally_reserved":0,"allocated":87715840,"data_stored":87028811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":4.7140000000000004,"5min":4.0140000000000002,"15min":1.863},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":62.140000000000001,"5min":126.752,"15min":126.752},"last":4.3710000000000004},{"interface":"front","average":{"1min":4.5869999999999997,"5min":4.194,"15min":1.8779999999999999},"min":{"1min":0.308,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":62.340000000000003,"5min":128.88399999999999,"15min":128.88399999999999},"last":1.1950000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.0430000000000001,"5min":6.6180000000000003,"15min":2.8260000000000001},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":101.471,"5min":245.28700000000001,"15min":245.28700000000001},"last":1.3520000000000001},{"interface":"front","average":{"1min":6.7069999999999999,"5min":6.3150000000000004,"15min":2.7799999999999998},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":98.527000000000001,"5min":244.626,"15min":244.626},"last":0.51100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.776,"15min":1.3200000000000001},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":4.351,"5min":13.887,"15min":58.615000000000002},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.97,"15min":1.77},"min":{"1min":0.38600000000000001,"5min":0.30399999999999999,"15min":0.27600000000000002},"max":{"1min":6.2320000000000002,"5min":13.872,"15min":99.372},"last":1.0569999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:49 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":1.7649999999999999,"15min":1.7370000000000001},"min":{"1min":0.34000000000000002,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":35.771000000000001,"5min":35.771000000000001,"15min":99.134},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.6019999999999999,"5min":1.9419999999999999,"15min":1.839},"min":{"1min":0.34599999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":35.848999999999997,"5min":35.848999999999997,"15min":99.117999999999995},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.0169999999999999,"15min":0.69799999999999995},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.98399999999999999,"5min":16.195,"15min":16.195},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97299999999999998,"15min":0.71799999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.343,"5min":12.442,"15min":12.442},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68700000000000006,"15min":0.61499999999999999},"min":{"1min":0.29299999999999998,"5min":0.27700000000000002,"15min":0.27400000000000002},"max":{"1min":4.4829999999999997,"5min":4.4829999999999997,"15min":6.5439999999999996},"last":1.3009999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76000000000000001,"15min":0.67800000000000005},"min":{"1min":0.33600000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83699999999999997,"5min":5.6779999999999999,"15min":5.6779999999999999},"last":1.165}]},{"osd":6,"last update":"Mon Oct 9 04:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.80300000000000005,"15min":0.69899999999999995},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.311},"max":{"1min":4.5199999999999996,"5min":8.3559999999999999,"15min":8.3559999999999999},"last":1.325},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.92600000000000005,"15min":0.75900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":2.399,"5min":5.6420000000000003,"15min":5.9039999999999999},"last":0.54700000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463165,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":382288,"kb_used_data":72712,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93362864,"statfs":{"total":95995035648,"available":95603572736,"internally_reserved":0,"allocated":74457088,"data_stored":73781906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":6.4000000000000004,"5min":4.4550000000000001,"15min":2.2370000000000001},"min":{"1min":0.222,"5min":0.217,"15min":0.217},"max":{"1min":77.453000000000003,"5min":77.453000000000003,"15min":78.983999999999995},"last":0.59199999999999997},{"interface":"front","average":{"1min":6.9790000000000001,"5min":4.9480000000000004,"15min":2.5129999999999999},"min":{"1min":0.28399999999999997,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":77.430999999999997,"5min":77.430999999999997,"15min":78.998000000000005},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:12:54 2023","interfaces":[{"interface":"back","average":{"1min":4.2729999999999997,"5min":3.2250000000000001,"15min":1.9450000000000001},"min":{"1min":0.25800000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":53.337000000000003,"5min":62.698,"15min":62.698},"last":0.57399999999999995},{"interface":"front","average":{"1min":4.0529999999999999,"5min":3.4670000000000001,"15min":2.0310000000000001},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":53.146000000000001,"5min":63.530999999999999,"15min":63.530999999999999},"last":0.60199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:12:22 2023","interfaces":[{"interface":"back","average":{"1min":3.456,"5min":3.7120000000000002,"15min":2.0350000000000001},"min":{"1min":0.33700000000000002,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":33.345999999999997,"5min":77.953000000000003,"15min":77.953000000000003},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.5680000000000001,"5min":3.7610000000000001,"15min":2.085},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":33.368000000000002,"5min":78.019000000000005,"15min":78.019000000000005},"last":0.47399999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:12:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5570000000000004,"5min":4.157,"15min":2.2970000000000002},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":51.082000000000001,"5min":57.509999999999998,"15min":57.509999999999998},"last":0.627},{"interface":"front","average":{"1min":4.556,"5min":3.9620000000000002,"15min":2.3119999999999998},"min":{"1min":0.32600000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":51.031999999999996,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.45100000000000001,"15min":0.48399999999999999},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.52500000000000002,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.37},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.57399999999999995,"15min":0.64200000000000002},"min":{"1min":0.26000000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.637,"5min":3.2970000000000002,"15min":7.7000000000000002},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 04:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.85299999999999998,"15min":0.85399999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.0089999999999999,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.93600000000000005,"15min":0.96299999999999997},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.998,"5min":23.006,"15min":54.853999999999999},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68600000000000005,"15min":0.628},"min":{"1min":0.317,"5min":0.313,"15min":0.313},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.79000000000000004,"15min":0.71199999999999997},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":3.3679999999999999,"5min":4.7679999999999998,"15min":4.7679999999999998},"last":0.51400000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089473,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":398624,"kb_used_data":148312,"kb_used_omap":9,"kb_used_meta":250294,"kb_avail":93346528,"statfs":{"total":95995035648,"available":95586844672,"internally_reserved":0,"allocated":151871488,"data_stored":150938885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":256302057},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":8.9399999999999995,"5min":4.9530000000000003,"15min":2.4140000000000001},"min":{"1min":0.318,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":68.048000000000002,"5min":85.977999999999994,"15min":85.977999999999994},"last":2.4449999999999998},{"interface":"front","average":{"1min":6.0739999999999998,"5min":4.7610000000000001,"15min":2.73},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":25.620999999999999,"5min":85.998999999999995,"15min":99.831999999999994},"last":2.722}]},{"osd":2,"last update":"Mon Oct 9 04:12:23 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":2.9359999999999999,"15min":1.7589999999999999},"min":{"1min":0.26100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":13.695,"5min":77.563000000000002,"15min":121.54600000000001},"last":2.3399999999999999},{"interface":"front","average":{"1min":4.9939999999999998,"5min":3.452,"15min":2.0110000000000001},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.186},"max":{"1min":15.298999999999999,"5min":77.504999999999995,"15min":99.899000000000001},"last":2.7530000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":9.3320000000000007,"5min":5.5259999999999998,"15min":2.7280000000000002},"min":{"1min":0.309,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":50.542999999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":2.3700000000000001},{"interface":"front","average":{"1min":9.9339999999999993,"5min":6.0990000000000002,"15min":3.2669999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":44.716000000000001,"5min":138.49799999999999,"15min":138.49799999999999},"last":2.77}]},{"osd":4,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.214,"15min":1.6879999999999999},"min":{"1min":0.29099999999999998,"5min":0.254,"15min":0.254},"max":{"1min":11.973000000000001,"5min":15.215999999999999,"15min":99.989999999999995},"last":2.7930000000000001},{"interface":"front","average":{"1min":4.8760000000000003,"5min":1.972,"15min":1.657},"min":{"1min":0.42999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":25.957999999999998,"5min":25.957999999999998,"15min":100.249},"last":0.44}]},{"osd":5,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":2.113,"15min":1.663},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.013999999999999,"5min":15.266,"15min":99.933999999999997},"last":2.8090000000000002},{"interface":"front","average":{"1min":4.8319999999999999,"5min":2.2040000000000002,"15min":1.768},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":100.34099999999999},"last":0.377}]},{"osd":6,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":4.7990000000000004,"5min":2.2719999999999998,"15min":1.8600000000000001},"min":{"1min":0.45300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":25.931000000000001,"5min":25.931000000000001,"15min":100.309},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.72,"15min":1.6779999999999999},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.446,"5min":13.446,"15min":100.143},"last":2.4199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":4.9329999999999998,"5min":2.3370000000000002,"15min":1.8480000000000001},"min":{"1min":0.53300000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":25.984999999999999,"5min":25.984999999999999,"15min":100.214},"last":0.47499999999999998},{"interface":"front","average":{"1min":3.0899999999999999,"5min":1.9970000000000001,"15min":1.698},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.317},"max":{"1min":13.471,"5min":13.471,"15min":100.113},"last":2.391}]}]},{"osd":0,"up_from":84,"seq":360777253000,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334252,"kb_used_data":59748,"kb_used_omap":28,"kb_used_meta":274147,"kb_avail":93410900,"statfs":{"total":95995035648,"available":95652761600,"internally_reserved":0,"allocated":61181952,"data_stored":60205507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":280727376},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,16],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":5.2050000000000001,"5min":3.0289999999999999,"15min":2.3050000000000002},"min":{"1min":0.20499999999999999,"5min":0.184,"15min":0.17000000000000001},"max":{"1min":37.764000000000003,"5min":44.287999999999997,"15min":118.48},"last":0.249},{"interface":"front","average":{"1min":5.718,"5min":3.5859999999999999,"15min":2.5649999999999999},"min":{"1min":0.28000000000000003,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":38.128999999999998,"5min":43.906999999999996,"15min":118.384},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:12:21 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":3.0339999999999998,"15min":2.2490000000000001},"min":{"1min":0.38,"5min":0.221,"15min":0.221},"max":{"1min":13.108000000000001,"5min":25.055,"15min":118.54000000000001},"last":0.436},{"interface":"front","average":{"1min":4.5979999999999999,"5min":3.2890000000000001,"15min":2.367},"min":{"1min":0.22900000000000001,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":12.372,"5min":44.436999999999998,"15min":118.727},"last":0.23499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.6109999999999998,"5min":5.0629999999999997,"15min":3.8820000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":18.196000000000002,"5min":156.488,"15min":156.488},"last":0.41999999999999998},{"interface":"front","average":{"1min":3.944,"5min":5.0300000000000002,"15min":3.923},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":12.465999999999999,"5min":156.50700000000001,"15min":156.50700000000001},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.0219999999999998,"15min":1.8080000000000001},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7270000000000003,"5min":10.023,"15min":118.633},"last":0.502},{"interface":"front","average":{"1min":2.714,"5min":2.4769999999999999,"15min":2.0379999999999998},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.492000000000001,"5min":44.475000000000001,"15min":118.913},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.6360000000000001,"15min":2.552},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":10.445,"5min":44.494,"15min":182.68899999999999},"last":0.375},{"interface":"front","average":{"1min":1.887,"5min":2.2210000000000001,"15min":2.387},"min":{"1min":0.41299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.4749999999999996,"5min":44.328000000000003,"15min":182.66300000000001},"last":0.34799999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":3.9180000000000001,"15min":2.3010000000000002},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.5090000000000003,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":3.649,"5min":3.9220000000000002,"15min":2.2679999999999998},"min":{"1min":0.44500000000000001,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":12.496,"5min":136.88999999999999,"15min":136.88999999999999},"last":0.58899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":3.3410000000000002,"15min":2.9169999999999998},"min":{"1min":0.38700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.5500000000000007,"5min":121.324,"15min":185.024},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.3239999999999998,"15min":2.508},"min":{"1min":0.46700000000000003,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":12.417,"5min":12.417,"15min":184.98599999999999},"last":0.621}]}]},{"osd":2,"up_from":94,"seq":403726925947,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":404056,"kb_used_data":74576,"kb_used_omap":81,"kb_used_meta":329390,"kb_avail":93341096,"statfs":{"total":95995035648,"available":95581282304,"internally_reserved":0,"allocated":76365824,"data_stored":75447336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":337295578},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2460000000000004,"5min":2.9359999999999999,"15min":1.4690000000000001},"min":{"1min":0.20200000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":22.321000000000002,"5min":22.321000000000002,"15min":22.321000000000002},"last":0.20499999999999999},{"interface":"front","average":{"1min":3.6800000000000002,"5min":3.581,"15min":1.738},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.6440000000000001,"5min":45.979999999999997,"15min":45.979999999999997},"last":0.44400000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7309999999999999,"5min":3.5910000000000002,"15min":1.9730000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":9.9429999999999996,"5min":53.695,"15min":68.009},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.7149999999999999,"5min":3.7090000000000001,"15min":2.1419999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.346,"5min":53.439,"15min":66.975999999999999},"last":0.40400000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":2.4380000000000002,"15min":1.5700000000000001},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":20.242999999999999,"5min":20.242999999999999,"15min":27.562000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":5.0439999999999996,"5min":3.2469999999999999,"15min":1.863},"min":{"1min":0.318,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":17.076000000000001,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.308}]},{"osd":4,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.2490000000000001,"15min":1.444},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5279999999999996,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":3.609,"5min":1.9419999999999999,"15min":1.234},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":20.134,"5min":20.134,"15min":20.134},"last":0.42699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.6789999999999998,"15min":1.4770000000000001},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":7.3399999999999999,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.501},{"interface":"front","average":{"1min":2.1779999999999999,"5min":2.6339999999999999,"15min":1.6020000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5039999999999996,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.33500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.6579999999999999,"5min":3.2829999999999999,"15min":1.671},"min":{"1min":0.376,"5min":0.311,"15min":0.311},"max":{"1min":20.184000000000001,"5min":114.14,"15min":114.14},"last":0.78900000000000003},{"interface":"front","average":{"1min":2.194,"5min":3.7719999999999998,"15min":1.8560000000000001},"min":{"1min":0.43099999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4199999999999999,"5min":114.123,"15min":114.123},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:12:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.556,"15min":1.8640000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":20.215,"5min":117.32599999999999,"15min":117.32599999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.6349999999999998,"5min":3.9209999999999998,"15min":1.982},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.4659999999999993,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.56000000000000005}]}]},{"osd":3,"up_from":99,"seq":425201762421,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678864,"kb_used_data":217992,"kb_used_omap":7,"kb_used_meta":460664,"kb_avail":93066288,"statfs":{"total":95995035648,"available":95299878912,"internally_reserved":0,"allocated":223223808,"data_stored":222318910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":471720552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":6.2770000000000001,"5min":5.2089999999999996,"15min":3.71},"min":{"1min":0.214,"5min":0.19400000000000001,"15min":0.185},"max":{"1min":24.077999999999999,"5min":100.512,"15min":106.22},"last":1.329},{"interface":"front","average":{"1min":7.8559999999999999,"5min":5.694,"15min":3.8660000000000001},"min":{"1min":0.22800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.091999999999999,"5min":101.733,"15min":106.986},"last":1.298}]},{"osd":1,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8450000000000002,"5min":3.819,"15min":2.484},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":19.262,"5min":57.055999999999997,"15min":106.121},"last":0.52100000000000002},{"interface":"front","average":{"1min":3.8180000000000001,"5min":4.1529999999999996,"15min":2.3330000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":16.646000000000001,"5min":57.088999999999999,"15min":65.015000000000001},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":2.9289999999999998,"15min":2.367},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":10.333,"5min":23.405999999999999,"15min":106.105},"last":1.5249999999999999},{"interface":"front","average":{"1min":4.3140000000000001,"5min":3.7370000000000001,"15min":2.782},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":13.201000000000001,"5min":98.765000000000001,"15min":106.70399999999999},"last":0.438}]},{"osd":4,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7400000000000002,"15min":2.0910000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.475,"5min":37.652000000000001,"15min":106.85899999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.4060000000000001,"5min":2.4580000000000002,"15min":1.4710000000000001},"min":{"1min":0.38700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.255000000000001,"5min":37.442,"15min":37.442},"last":0.47799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.0329999999999999,"15min":1.381},"min":{"1min":0.36899999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.285,"5min":10.285,"15min":12.819000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.8610000000000002,"5min":2.0510000000000002,"15min":1.6319999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.968,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.59099999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.9399999999999999,"15min":1.9910000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":9.9290000000000003,"5min":13.255000000000001,"15min":106.036},"last":0.628},{"interface":"front","average":{"1min":3.3290000000000002,"5min":2.371,"15min":1.96},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":11.425000000000001,"5min":15.622,"15min":106.741},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":2.3919999999999999,"15min":2.0030000000000001},"min":{"1min":0.34599999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.506,"5min":15.685,"15min":106.82599999999999},"last":0.498},{"interface":"front","average":{"1min":2.9609999999999999,"5min":2.214,"15min":1.859},"min":{"1min":0.42599999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":10.212999999999999,"5min":10.212999999999999,"15min":106.084},"last":0.55600000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986437,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":356248,"kb_used_data":54864,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93388904,"statfs":{"total":95995035648,"available":95630237696,"internally_reserved":0,"allocated":56180736,"data_stored":55505116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":4.7560000000000002,"5min":3.3079999999999998,"15min":1.631},"min":{"1min":0.222,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":38.831000000000003,"5min":63.128,"15min":63.128},"last":0.216},{"interface":"front","average":{"1min":4.7939999999999996,"5min":3.0590000000000002,"15min":1.5960000000000001},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":37.039999999999999,"5min":62.645000000000003,"15min":62.645000000000003},"last":0.27900000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.6989999999999998,"15min":1.5469999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.0899999999999999,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.8450000000000002,"15min":1.5900000000000001},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.288,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.5720000000000001,"15min":1.179},"min":{"1min":0.255,"5min":0.255,"15min":0.245},"max":{"1min":5.4470000000000001,"5min":8.8030000000000008,"15min":19.356000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.881,"5min":1.6299999999999999,"15min":1.1299999999999999},"min":{"1min":0.38700000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.4610000000000003,"5min":8.5920000000000005,"15min":19.324000000000002},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:12:57 2023","interfaces":[{"interface":"back","average":{"1min":8.1989999999999998,"5min":2.8919999999999999,"15min":1.611},"min":{"1min":0.32000000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":125.90300000000001,"5min":125.90300000000001,"15min":125.90300000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":8.6199999999999992,"5min":3.1699999999999999,"15min":1.76},"min":{"1min":0.39900000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":127.90000000000001,"5min":127.90000000000001,"15min":127.90000000000001},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55000000000000004,"15min":0.58099999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3050000000000002,"5min":3.4510000000000001,"15min":8.5259999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.956,"5min":5.9569999999999999,"15min":8.3149999999999995},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 04:12:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.753,"15min":0.751},"min":{"1min":0.30199999999999999,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":3.1920000000000002,"5min":5.6849999999999996,"15min":8.3989999999999991},"last":0.379},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.76400000000000001,"15min":0.68600000000000005},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":3.21,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.40200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:12:19 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.70299999999999996,"15min":0.70699999999999996},"min":{"1min":0.311,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":2.3599999999999999,"5min":3.5190000000000001,"15min":9.0050000000000008},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.79100000000000004,"15min":0.77200000000000002},"min":{"1min":0.36299999999999999,"5min":0.34100000000000003,"15min":0.29199999999999998},"max":{"1min":2.214,"5min":5.9939999999999998,"15min":8.5440000000000005},"last":0.47099999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724801,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346176,"kb_used_data":44792,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93398976,"statfs":{"total":95995035648,"available":95640551424,"internally_reserved":0,"allocated":45867008,"data_stored":45203462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":24.518000000000001,"5min":7.5119999999999996,"15min":3.1709999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":151.23599999999999,"5min":151.23599999999999,"15min":151.23599999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":25.135000000000002,"5min":7.8940000000000001,"15min":3.3359999999999999},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":151.566,"5min":151.566,"15min":151.566},"last":0.46700000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:12:50 2023","interfaces":[{"interface":"back","average":{"1min":7.6600000000000001,"5min":3.6869999999999998,"15min":2.177},"min":{"1min":0.26100000000000001,"5min":0.219,"15min":0.219},"max":{"1min":58.408000000000001,"5min":59.588999999999999,"15min":91.212999999999994},"last":0.41899999999999998},{"interface":"front","average":{"1min":7.4909999999999997,"5min":3.8450000000000002,"15min":2.3740000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":58.393999999999998,"5min":59.533000000000001,"15min":91.254000000000005},"last":0.53900000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:12:27 2023","interfaces":[{"interface":"back","average":{"1min":5.7990000000000004,"5min":3.677,"15min":1.8720000000000001},"min":{"1min":0.29699999999999999,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":67.685000000000002,"5min":67.685000000000002,"15min":67.685000000000002},"last":2.895},{"interface":"front","average":{"1min":6.0250000000000004,"5min":3.96,"15min":1.8839999999999999},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":67.596000000000004,"5min":67.596000000000004,"15min":67.596000000000004},"last":2.964}]},{"osd":3,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":1.756,"15min":1.885},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":26.645,"5min":26.645,"15min":91.834999999999994},"last":0.503},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.9390000000000001,"15min":1.9279999999999999},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.309},"max":{"1min":26.536000000000001,"5min":26.536000000000001,"15min":91.802000000000007},"last":0.60199999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.75600000000000001,"15min":0.65300000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.64600000000000002,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.373},{"interface":"front","average":{"1min":0.371,"5min":0.78200000000000003,"15min":0.70799999999999996},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.51700000000000002,"5min":5.29,"15min":6.7709999999999999},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":2.0409999999999999,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":1.407,"5min":123.239,"15min":123.239},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.0139999999999998,"15min":1.1319999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0289999999999999,"5min":123.014,"15min":123.014},"last":0.76300000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.877,"15min":0.77500000000000002},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":0.64100000000000001,"5min":9.4930000000000003,"15min":9.4930000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.747,"15min":0.79800000000000004},"min":{"1min":0.35599999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.73499999999999999,"5min":8.3740000000000006,"15min":8.5329999999999995},"last":0.56999999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":51056640,"data_stored":51052568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":139595776,"data_stored":139591704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":74514432,"data_stored":74514432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":209780736,"data_stored":209780736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":52953088,"data_stored":52953088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":40042496,"data_stored":40042496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":69537792,"data_stored":69533720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":77529088,"data_stored":77529088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:13:22.023 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:22.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:22.060 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:22.122 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:22.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:22.162 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:22.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:22.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:22.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:22.341 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:22.575 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:22.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:22.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:13:22.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:22.727 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:22.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:22.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:22.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:22 smithi163 ceph-mon[221067]: pgmap v624: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 2950/285 objects degraded (1035.088%); 742 KiB/s, 139 objects/s recovering 2023-10-09T04:13:22.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:13:22.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:22 smithi163 ceph-mon[213841]: pgmap v624: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 2950/285 objects degraded (1035.088%); 742 KiB/s, 139 objects/s recovering 2023-10-09T04:13:22.788 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:13:22.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:22.916 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:22 smithi181 ceph-mon[196979]: pgmap v624: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 294 op/s; 2950/285 objects degraded (1035.088%); 742 KiB/s, 139 objects/s recovering 2023-10-09T04:13:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:13:23.025 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:23.604 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:13:24.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2351271604' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:24.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:24.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2351271604' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:24.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:24.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2351271604' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:24.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:24.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:24.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:24.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:24.214 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:24.545 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:24.545 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:13:24.545 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:13:24.545 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:13:24.546 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.546 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118877 6.a 6:5646758f:::benchmark_data_smithi163_287034_object10614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:13:24.546 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:13:24.546 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 401.53139212000002, 2023-10-09T04:13:24.546 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24332414399999999, 2023-10-09T04:13:24.546 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.547 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.547 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.547 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.547 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.547 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118877 2023-10-09T04:13:24.547 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.548 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.549 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.549 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:13:24.549 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.549 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.549 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.549 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010794+0000", 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010802+0000", 2023-10-09T04:13:24.550 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3140000000000004e-06 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010818+0000", 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5945000000000001e-05 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.551 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010825+0000", 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1659999999999997e-06 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010868+0000", 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2886999999999998e-05 2023-10-09T04:13:24.552 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148662+0000", 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.137794378 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148810+0000", 2023-10-09T04:13:24.553 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147527 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.152530+0000", 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003719703 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.554 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.153038+0000", 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050862599999999998 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254050+0000", 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.101012064 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.555 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254090+0000", 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9533999999999998e-05 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254120+0000", 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9903000000000002e-05 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.556 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.557 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.557 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.557 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151856 6.d 6:ba33a9cc:::benchmark_data_smithi163_287034_object43593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:13:24.557 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:13:24.557 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 356.91522341799998, 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23517902700000001, 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.558 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151856 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.559 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626966+0000", 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.877e-06 2023-10-09T04:13:24.560 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626972+0000", 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7089999999999999e-06 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626975+0000", 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1319999999999998e-06 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.561 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627008+0000", 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3343999999999998e-05 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627097+0000", 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8706000000000001e-05 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.562 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627179+0000", 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2310000000000003e-05 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.631260+0000", 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040813139999999999 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794861+0000", 2023-10-09T04:13:24.563 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16360079 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862048+0000", 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067187075999999998 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862105+0000", 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6997000000000003e-05 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.564 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862143+0000", 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8050999999999999e-05 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151866 6.d 6:b4107b04:::benchmark_data_smithi163_287034_object43603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 356.905266627, 2023-10-09T04:13:24.565 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22535977800000001, 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151866 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.566 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636920+0000", 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.567 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636926+0000", 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636939+0000", 2023-10-09T04:13:24.568 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2904e-05 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636944+0000", 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8879999999999996e-06 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636983+0000", 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8402000000000003e-05 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.569 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637122+0000", 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000139307 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637246+0000", 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012406599999999999 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.570 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.641603+0000", 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043568410000000002 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794991+0000", 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15338833599999999 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.571 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862183+0000", 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067192247999999996 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862248+0000", 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4317000000000001e-05 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862281+0000", 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3144e-05 2023-10-09T04:13:24.572 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169261 6.d 6:b3c8e3b4:::benchmark_data_smithi163_287034_object60998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.531345886, 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22831853499999999, 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.573 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169261 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.574 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010841+0000", 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.575 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010846+0000", 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1510000000000003e-06 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010858+0000", 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1446e-05 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010864+0000", 2023-10-09T04:13:24.576 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3620000000000002e-06 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010914+0000", 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9552000000000003e-05 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011016+0000", 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102262 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.577 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011120+0000", 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104622 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020057+0000", 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089365069999999998 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.578 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179106+0000", 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15904936 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239056+0000", 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.059950160000000002 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.579 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239115+0000", 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8446000000000003e-05 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239160+0000", 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5503999999999997e-05 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.580 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169262 6.d 6:b9ed22f3:::benchmark_data_smithi163_287034_object60999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:13:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:13:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 333.52632898799999, 2023-10-09T04:13:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23726620200000001, 2023-10-09T04:13:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169262 2023-10-09T04:13:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:13:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:13:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015858+0000", 2023-10-09T04:13:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015863+0000", 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.87e-06 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015871+0000", 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7810000000000005e-06 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015874+0000", 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6440000000000002e-06 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015992+0000", 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117414 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016078+0000", 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6527999999999996e-05 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016159+0000", 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0648000000000004e-05 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020242+0000", 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040829209999999998 2023-10-09T04:13:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179222+0000", 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15898009699999999 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253062+0000", 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.073840684000000004 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.589 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253100+0000", 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7886000000000003e-05 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253125+0000", 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4675999999999999e-05 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205752 6.a 6:5755c8f4:::benchmark_data_smithi163_287034_object97489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 278.66995712599999, 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22375711200000001, 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205752 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872230+0000", 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872233+0000", 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8949999999999998e-06 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872238+0000", 2023-10-09T04:13:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2449999999999998e-06 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872241+0000", 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8600000000000001e-06 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872286+0000", 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5157999999999997e-05 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872387+0000", 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010013699999999999 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872470+0000", 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3599999999999999e-05 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881422+0000", 2023-10-09T04:13:24.595 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089518930000000007 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881460+0000", 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7812000000000003e-05 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095901+0000", 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.214440778 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.596 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095945+0000", 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4332000000000002e-05 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095988+0000", 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2778000000000003e-05 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.597 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205754 6.d 6:b347cae0:::benchmark_data_smithi163_287034_object97491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 278.66993537500002, 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22360279999999999, 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.598 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205754 2023-10-09T04:13:24.599 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.599 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.599 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.599 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.599 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872252+0000", 2023-10-09T04:13:24.600 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872255+0000", 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.796e-06 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872261+0000", 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.117e-06 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.601 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872265+0000", 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6839999999999998e-06 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877378+0000", 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051136020000000001 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.602 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877474+0000", 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5499000000000002e-05 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877596+0000", 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012222700000000001 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.882034+0000", 2023-10-09T04:13:24.603 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044384189999999999 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.890599+0000", 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085642140000000005 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891540+0000", 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00094105299999999996 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.604 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.891569+0000", 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9382000000000001e-05 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095855+0000", 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.204286197 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.605 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205761 6.a 6:5be756cf:::benchmark_data_smithi163_287034_object97498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 278.66477445300001, 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.218706593, 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:24.606 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205761 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877413+0000", 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.607 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877412+0000", 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877421+0000", 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4700000000000002e-06 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.608 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877430+0000", 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8929999999999994e-06 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877434+0000", 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9559999999999996e-06 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877558+0000", 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012344299999999999 2023-10-09T04:13:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877639+0000", 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1212999999999996e-05 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.877720+0000", 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0533999999999998e-05 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881593+0000", 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003873479 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881717+0000", 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123604 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096041+0000", 2023-10-09T04:13:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21432412100000001 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096082+0000", 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1437999999999999e-05 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.096120+0000", 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7599999999999999e-05 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36612 7.0 7:006250d9:::benchmark_data_smithi163_346082_object36611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 216.25217369000001, 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97418880500000005, 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36612 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290012+0000", 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290059+0000", 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6390000000000001e-05 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290079+0000", 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0370999999999999e-05 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290088+0000", 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.183e-06 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291399+0000", 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013106369999999999 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291468+0000", 2023-10-09T04:13:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9116999999999996e-05 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264113+0000", 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97264483000000002 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264203+0000", 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0143999999999996e-05 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:50550 7.0 7:02154344:::benchmark_data_smithi163_346082_object50549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 198.26324688, 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25363291799999998, 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 50550 2023-10-09T04:13:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:13:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278940+0000", 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278990+0000", 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0781000000000001e-05 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279008+0000", 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7193999999999998e-05 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279016+0000", 2023-10-09T04:13:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2830000000000001e-06 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279063+0000", 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7596000000000003e-05 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279201+0000", 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137634 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.624 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279310+0000", 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109231 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.280343+0000", 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001032989 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.280822+0000", 2023-10-09T04:13:24.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047909999999999999 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532453+0000", 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25163097600000001 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532526+0000", 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2867000000000006e-05 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532574+0000", 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7599000000000003e-05 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:50558 7.0 7:042b1ccf:::benchmark_data_smithi163_346082_object50557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 198.257608089, 2023-10-09T04:13:24.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24840627900000001, 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 50558 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284578+0000", 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284632+0000", 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4066e-05 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284645+0000", 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2251e-05 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284650+0000", 2023-10-09T04:13:24.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2100000000000001e-06 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284690+0000", 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9845000000000002e-05 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284837+0000", 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014754199999999999 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284947+0000", 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109949 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.285939+0000", 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00099183299999999991 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.286366+0000", 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042700700000000003 2023-10-09T04:13:24.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532907+0000", 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24654063800000001 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532954+0000", 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6845999999999998e-05 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.633 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532986+0000", 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2450000000000003e-05 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80455 7.0 7:03584563:::benchmark_data_smithi163_346082_object80454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.40664381900001, 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31383416800000002, 2023-10-09T04:13:24.634 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80455 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.635 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135542+0000", 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135577+0000", 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5281000000000003e-05 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135599+0000", 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2073e-05 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135607+0000", 2023-10-09T04:13:24.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.174e-06 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135662+0000", 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4206999999999998e-05 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135784+0000", 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121916 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135908+0000", 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124819 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.137658+0000", 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017491220000000001 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.139918+0000", 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022599949999999999 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449290+0000", 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30937226000000001 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449346+0000", 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6529e-05 2023-10-09T04:13:24.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449378+0000", 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.18e-05 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80493 7.0 7:04f35a7b:::benchmark_data_smithi163_346082_object80492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.36118784799999, 2023-10-09T04:13:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27444662600000003, 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80493 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.180999+0000", 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181023+0000", 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4460999999999998e-05 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181036+0000", 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.277e-05 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181043+0000", 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6320000000000002e-06 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181091+0000", 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8309000000000001e-05 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181190+0000", 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9109000000000003e-05 2023-10-09T04:13:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181286+0000", 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5846000000000004e-05 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.182553+0000", 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001267191 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.182801+0000", 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024814299999999999 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455353+0000", 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27255158600000001 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455406+0000", 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3074e-05 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455447+0000", 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0441000000000001e-05 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80494 7.e 7:79ff37ea:::benchmark_data_smithi163_346082_object80493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.092560209, 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.00624114, 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80494 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449627+0000", 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449685+0000", 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8378000000000002e-05 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449697+0000", 2023-10-09T04:13:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2427e-05 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449703+0000", 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449751+0000", 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8476999999999997e-05 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449761+0000", 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0378e-05 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441447+0000", 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99168547399999996 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441622+0000", 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017500600000000001 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441827+0000", 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020516999999999999 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451473+0000", 2023-10-09T04:13:24.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096461649999999999 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452805+0000", 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013312770000000001 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454609+0000", 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018042959999999999 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454649+0000", 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0256999999999998e-05 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.455869+0000", 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001219649 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80509 7.e 7:77548b6a:::benchmark_data_smithi163_346082_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.073722242, 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0024772179999999, 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80509 2023-10-09T04:13:24.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468464+0000", 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468512+0000", 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615000000000001e-05 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468527+0000", 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4778e-05 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468534+0000", 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2049999999999996e-06 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468595+0000", 2023-10-09T04:13:24.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0865000000000001e-05 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468603+0000", 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2439999999999996e-06 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445422+0000", 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97681920499999997 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445635+0000", 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000213382 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445836+0000", 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020043 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451507+0000", 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056710220000000004 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452890+0000", 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013838699999999999 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469631+0000", 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01674059 2023-10-09T04:13:24.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469678+0000", 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6551999999999997e-05 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.470943+0000", 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00126515 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.664 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80511 7.e 7:74ae5f43:::benchmark_data_smithi163_346082_object80510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.07215993899999, 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002226042, 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.665 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80511 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470027+0000", 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470082+0000", 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5106999999999999e-05 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470093+0000", 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0738e-05 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470098+0000", 2023-10-09T04:13:24.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5820000000000003e-06 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470142+0000", 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4261000000000002e-05 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470148+0000", 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0170000000000002e-06 2023-10-09T04:13:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.447968+0000", 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97782021900000005 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448099+0000", 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013048700000000001 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448204+0000", 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105266 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451513+0000", 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033096240000000002 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.456169+0000", 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046552060000000003 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471004+0000", 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014834926999999999 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471062+0000", 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8873999999999999e-05 2023-10-09T04:13:24.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472254+0000", 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011913939999999999 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80513 7.e 7:768b3fe4:::benchmark_data_smithi163_346082_object80512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:13:24.673 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.07023422899999, 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0014613109999999, 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80513 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471952+0000", 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:24.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472006+0000", 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3981999999999999e-05 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472018+0000", 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1924e-05 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472023+0000", 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6999999999999999e-06 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472070+0000", 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7218000000000001e-05 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472078+0000", 2023-10-09T04:13:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8830000000000006e-06 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449809+0000", 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97773131000000002 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449917+0000", 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107543 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.450027+0000", 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010959399999999999 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451621+0000", 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015948360000000001 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.466881+0000", 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015260029 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472336+0000", 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054546960000000002 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472393+0000", 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6829000000000001e-05 2023-10-09T04:13:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.473415+0000", 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001021852 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:130222 7.e 7:72845360:::benchmark_data_smithi163_346082_object36952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:13:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.175651368999993, 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.231189378, 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 130222 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:13:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366539+0000", 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0939999999999999e-06 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366546+0000", 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5060000000000001e-06 2023-10-09T04:13:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366550+0000", 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2880000000000003e-06 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.370042+0000", 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034921779999999999 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.370190+0000", 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000148338 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.370348+0000", 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015750399999999999 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.380035+0000", 2023-10-09T04:13:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096872339999999994 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.387136+0000", 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0071007370000000002 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.596137+0000", 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20900122300000001 2023-10-09T04:13:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.596197+0000", 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0238999999999997e-05 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.597726+0000", 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015284649999999999 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:130225 7.e 7:70682ac4:::benchmark_data_smithi163_346082_object36955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:48.366579+0000", 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.175609218000005, 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.232392707, 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 130225 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366579+0000", 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366579+0000", 2023-10-09T04:13:24.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366578+0000", 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366581+0000", 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.32e-06 2023-10-09T04:13:24.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366588+0000", 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6059999999999999e-06 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366592+0000", 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1740000000000002e-06 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.373524+0000", 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069322630000000001 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.373652+0000", 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127868 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.373767+0000", 2023-10-09T04:13:24.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011438 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.381882+0000", 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081150849999999993 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.387171+0000", 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052897389999999999 2023-10-09T04:13:24.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.597799+0000", 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21062740799999999 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.597852+0000", 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3316999999999998e-05 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.598971+0000", 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011191230000000001 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:130230 7.e 7:795e8c02:::benchmark_data_smithi163_346082_object36960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.169805707999998, 2023-10-09T04:13:24.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22737015799999999, 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 130230 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:13:24.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372384+0000", 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4710000000000001e-06 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372396+0000", 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1887e-05 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:24.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372400+0000", 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1729999999999997e-06 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.376814+0000", 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004414187 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.376922+0000", 2023-10-09T04:13:24.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010727500000000001 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.377006+0000", 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4342e-05 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.381892+0000", 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004886189 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.387197+0000", 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053050759999999997 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.599035+0000", 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.211838046 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:24.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.599075+0000", 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9289000000000003e-05 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.599752+0000", 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00067761399999999999 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:24.703 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:25.018 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:13:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:24 smithi181 ceph-mon[196979]: pgmap v625: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 2950/285 objects degraded (1035.088%); 632 KiB/s, 156 objects/s recovering 2023-10-09T04:13:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:24 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:13:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:24 smithi181 ceph-mon[196979]: failed to encode map e236 with expected crc 2023-10-09T04:13:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:24 smithi181 ceph-mon[196979]: failed to encode map e236 with expected crc 2023-10-09T04:13:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:13:25.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:24 smithi181 ceph-mon[196979]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:13:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[221067]: pgmap v625: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 2950/285 objects degraded (1035.088%); 632 KiB/s, 156 objects/s recovering 2023-10-09T04:13:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:13:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[221067]: failed to encode map e236 with expected crc 2023-10-09T04:13:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[221067]: failed to encode map e236 with expected crc 2023-10-09T04:13:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[221067]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[213841]: pgmap v625: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 2950/285 objects degraded (1035.088%); 632 KiB/s, 156 objects/s recovering 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[213841]: failed to encode map e236 with expected crc 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[213841]: failed to encode map e236 with expected crc 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:13:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:24 smithi163 ceph-mon[213841]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:13:25.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:13:25.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:25.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:25.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:25.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:25.682 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:25.878 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:25 smithi181 ceph-mon[196979]: failed to encode map e236 with expected crc 2023-10-09T04:13:26.144 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:25 smithi163 ceph-mon[213841]: failed to encode map e236 with expected crc 2023-10-09T04:13:26.145 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:25 smithi163 ceph-mon[221067]: failed to encode map e236 with expected crc 2023-10-09T04:13:26.992 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:26 smithi181 ceph-mon[196979]: pgmap v627: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 2950/285 objects degraded (1035.088%); 0 B/s, 243 objects/s recovering 2023-10-09T04:13:27.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:26 smithi163 ceph-mon[221067]: pgmap v627: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 2950/285 objects degraded (1035.088%); 0 B/s, 243 objects/s recovering 2023-10-09T04:13:27.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:26 smithi163 ceph-mon[213841]: pgmap v627: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 2950/285 objects degraded (1035.088%); 0 B/s, 243 objects/s recovering 2023-10-09T04:13:27.216 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:27.217 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:13:27.217 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:13:27.217 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:27.342 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:27.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:13:28.204 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:28.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:28.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:28.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:28.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:28.581 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:28.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:28.663 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[213841]: failed to encode map e237 with expected crc 2023-10-09T04:13:28.663 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[213841]: failed to encode map e237 with expected crc 2023-10-09T04:13:28.663 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[213841]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:13:28.663 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[213841]: pgmap v629: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 2950/285 objects degraded (1035.088%); 0 B/s, 267 objects/s recovering 2023-10-09T04:13:28.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[221067]: failed to encode map e237 with expected crc 2023-10-09T04:13:28.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[221067]: failed to encode map e237 with expected crc 2023-10-09T04:13:28.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[221067]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:13:28.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:28 smithi163 ceph-mon[221067]: pgmap v629: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 2950/285 objects degraded (1035.088%); 0 B/s, 267 objects/s recovering 2023-10-09T04:13:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:28.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:28.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:28 smithi181 ceph-mon[196979]: failed to encode map e237 with expected crc 2023-10-09T04:13:28.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:28 smithi181 ceph-mon[196979]: failed to encode map e237 with expected crc 2023-10-09T04:13:28.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:28 smithi181 ceph-mon[196979]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:13:28.744 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:28 smithi181 ceph-mon[196979]: pgmap v629: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 2 op/s; 2950/285 objects degraded (1035.088%); 0 B/s, 267 objects/s recovering 2023-10-09T04:13:28.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:28.795 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:28.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:28.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:28.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:28.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:29.022 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:29.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:29.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:29.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:29.347 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:29.491 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:29.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:29.596 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:29.622 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[221067]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[221067]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[221067]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[221067]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[213841]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[213841]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[213841]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.623 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:29 smithi163 ceph-mon[213841]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.625 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:29.687 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:29.687 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:13:29.687 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:13:29.687 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:13:29.687 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:29.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:29 smithi181 ceph-mon[196979]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:29 smithi181 ceph-mon[196979]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:13:29.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:29 smithi181 ceph-mon[196979]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:29 smithi181 ceph-mon[196979]: failed to encode map e238 with expected crc 2023-10-09T04:13:29.979 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:29.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:30.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:30.112 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:30.361 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:13:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:30 smithi163 ceph-mon[221067]: failed to encode map e238 with expected crc 2023-10-09T04:13:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:30 smithi163 ceph-mon[221067]: pgmap v631: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:30.636 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:13:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:13:30] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:13:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:30 smithi163 ceph-mon[213841]: failed to encode map e238 with expected crc 2023-10-09T04:13:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:30 smithi163 ceph-mon[213841]: pgmap v631: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:30.716 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:30 smithi181 ceph-mon[196979]: failed to encode map e238 with expected crc 2023-10-09T04:13:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:30 smithi181 ceph-mon[196979]: pgmap v631: 99 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:31.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:31 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:13:31.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:31 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:13:31.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:31 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:13:32.267 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:32.267 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:13:32.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:13:32.268 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:13:32.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.268 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118953 6.c 6:3024d817:::benchmark_data_smithi163_287034_object10690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:32.268 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:13:32.268 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 407.72921428400002, 2023-10-09T04:13:32.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96349797500000001, 2023-10-09T04:13:32.269 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.269 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.269 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.269 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.269 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.270 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118953 2023-10-09T04:13:32.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.270 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:13:32.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:13:32.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535426+0000", 2023-10-09T04:13:32.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:32.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535432+0000", 2023-10-09T04:13:32.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0039999999999996e-06 2023-10-09T04:13:32.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535441+0000", 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0080000000000008e-06 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535448+0000", 2023-10-09T04:13:32.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.737e-06 2023-10-09T04:13:32.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535513+0000", 2023-10-09T04:13:32.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5147000000000005e-05 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535523+0000", 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0067e-05 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.481396+0000", 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94587314700000003 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.483600+0000", 2023-10-09T04:13:32.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022032530000000001 2023-10-09T04:13:32.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.483670+0000", 2023-10-09T04:13:32.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0501999999999997e-05 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.497950+0000", 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014279490000000001 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498056+0000", 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106544 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498844+0000", 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00078815099999999996 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498889+0000", 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4614999999999999e-05 2023-10-09T04:13:32.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498925+0000", 2023-10-09T04:13:32.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6409000000000003e-05 2023-10-09T04:13:32.282 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.282 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118965 6.c 6:37b70bb5:::benchmark_data_smithi163_287034_object10702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 407.6790057, 2023-10-09T04:13:32.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91336464399999995, 2023-10-09T04:13:32.284 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.284 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.284 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.284 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.284 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.284 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118965 2023-10-09T04:13:32.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.285 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:13:32.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:13:32.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585634+0000", 2023-10-09T04:13:32.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:32.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585641+0000", 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0720000000000002e-06 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585655+0000", 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3332e-05 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585661+0000", 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4740000000000002e-06 2023-10-09T04:13:32.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585709+0000", 2023-10-09T04:13:32.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7892999999999997e-05 2023-10-09T04:13:32.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585715+0000", 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4180000000000002e-06 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486249+0000", 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90053330399999998 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486335+0000", 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6322000000000005e-05 2023-10-09T04:13:32.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486411+0000", 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5829999999999997e-05 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498155+0000", 2023-10-09T04:13:32.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011743709999999999 2023-10-09T04:13:32.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498809+0000", 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00065421399999999997 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498938+0000", 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012885899999999999 2023-10-09T04:13:32.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498971+0000", 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3025999999999999e-05 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499001+0000", 2023-10-09T04:13:32.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9971000000000002e-05 2023-10-09T04:13:32.297 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.297 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.297 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.298 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118966 6.c 6:3491ce90:::benchmark_data_smithi163_287034_object10703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:32.298 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:13:32.298 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 407.67896698300001, 2023-10-09T04:13:32.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91791875999999994, 2023-10-09T04:13:32.298 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118966 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585674+0000", 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585677+0000", 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6410000000000002e-06 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585683+0000", 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1410000000000002e-06 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585687+0000", 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9369999999999997e-06 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585735+0000", 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7969000000000001e-05 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585741+0000", 2023-10-09T04:13:32.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7339999999999996e-06 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489228+0000", 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90348698199999999 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489334+0000", 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106357 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489437+0000", 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103119 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498838+0000", 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00940123 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499006+0000", 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016752499999999999 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503505+0000", 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044992570000000004 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503553+0000", 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8050000000000002e-05 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503594+0000", 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0227e-05 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.307 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118982 6.c 6:3e1a0c35:::benchmark_data_smithi163_287034_object10719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 407.65912659399999, 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90241945000000001, 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118982 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605514+0000", 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605521+0000", 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8539999999999999e-06 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605532+0000", 2023-10-09T04:13:32.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.114e-05 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605537+0000", 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.118e-06 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605601+0000", 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4351999999999996e-05 2023-10-09T04:13:32.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605612+0000", 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0536e-05 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492104+0000", 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88649252599999995 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492184+0000", 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9084000000000004e-05 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492265+0000", 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1630000000000003e-05 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499011+0000", 2023-10-09T04:13:32.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067454790000000004 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503639+0000", 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046281070000000002 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507860+0000", 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042216689999999999 2023-10-09T04:13:32.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507899+0000", 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.841e-05 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507935+0000", 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5747000000000002e-05 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.315 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:119013 6.c 6:3a6a5865:::benchmark_data_smithi163_287034_object10750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 407.51607891399999, 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75951261400000003, 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.316 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 119013 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:13:32.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748561+0000", 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748576+0000", 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4914e-05 2023-10-09T04:13:32.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748593+0000", 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6529999999999999e-05 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748600+0000", 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6789999999999999e-06 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748669+0000", 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9541000000000004e-05 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748682+0000", 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3251000000000001e-05 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494591+0000", 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.74590919700000002 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494688+0000", 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6761000000000004e-05 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494788+0000", 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9431000000000004e-05 2023-10-09T04:13:32.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499015+0000", 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042273709999999997 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503667+0000", 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046520449999999996 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507982+0000", 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043144380000000003 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.508033+0000", 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1585000000000002e-05 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.508075+0000", 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2299999999999998e-05 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:186822 6.c 6:363d9db9:::benchmark_data_smithi163_287034_object78559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:18.841517+0000", 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.42312447099999, 2023-10-09T04:13:32.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14537045900000001, 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 186822 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841517+0000", 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841517+0000", 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841516+0000", 2023-10-09T04:13:32.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841523+0000", 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7900000000000002e-06 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841540+0000", 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7207e-05 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.841546+0000", 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6420000000000001e-06 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.845953+0000", 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044065739999999999 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.846093+0000", 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000140035 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.846259+0000", 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000165754 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851091+0000", 2023-10-09T04:13:32.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048319640000000002 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851120+0000", 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9507999999999999e-05 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.986799+0000", 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13567926299999999 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.986847+0000", 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7463999999999998e-05 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.986888+0000", 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0781000000000001e-05 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.331 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:186830 6.c 6:3c7a66bf:::benchmark_data_smithi163_287034_object78567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:18.851800+0000", 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.41284157899997, 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.143798601, 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.332 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 186830 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851800+0000", 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851800+0000", 2023-10-09T04:13:32.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851799+0000", 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851806+0000", 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6780000000000003e-06 2023-10-09T04:13:32.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851821+0000", 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5142e-05 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.851827+0000", 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7479999999999996e-06 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.855956+0000", 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041287590000000001 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.856117+0000", 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016046300000000001 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.856266+0000", 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014936300000000001 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.859956+0000", 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036902979999999998 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.860463+0000", 2023-10-09T04:13:32.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050628999999999995 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.995496+0000", 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13503367099999999 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.995549+0000", 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3134999999999999e-05 2023-10-09T04:13:32.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:18.995599+0000", 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9357000000000002e-05 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:201176 6.c 6:3908047e:::benchmark_data_smithi163_287034_object92913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T04:13:32.339 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:39.354240+0000", 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 292.91040193100002, 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14421740799999999, 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 201176 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354240+0000", 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354240+0000", 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354239+0000", 2023-10-09T04:13:32.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354244+0000", 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1449999999999999e-06 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354263+0000", 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9398e-05 2023-10-09T04:13:32.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354271+0000", 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5669999999999997e-06 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354326+0000", 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4938000000000002e-05 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354459+0000", 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013323600000000001 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.354600+0000", 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014062600000000001 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.357949+0000", 2023-10-09T04:13:32.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033488530000000002 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.358248+0000", 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029936399999999999 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.498386+0000", 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14013837900000001 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.498423+0000", 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6891999999999998e-05 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:39.498457+0000", 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3994e-05 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:216453 6.c 6:331e527e:::benchmark_data_smithi163_287034_object108190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:02.434388+0000", 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 269.83025354, 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13862418500000001, 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 216453 2023-10-09T04:13:32.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434388+0000", 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434388+0000", 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434386+0000", 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434395+0000", 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3599999999999996e-06 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434408+0000", 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3511e-05 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434415+0000", 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1920000000000002e-06 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434470+0000", 2023-10-09T04:13:32.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.537e-05 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434594+0000", 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012374599999999999 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.434704+0000", 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011008799999999999 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.438729+0000", 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004025392 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.447073+0000", 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0083440919999999991 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.572925+0000", 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12585126299999999 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.572969+0000", 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4978999999999997e-05 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573012+0000", 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2885e-05 2023-10-09T04:13:32.353 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:216459 6.c 6:30c38a9c:::benchmark_data_smithi163_287034_object108196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:02.440534+0000", 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 269.82410723100003, 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.132590931, 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.354 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 216459 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440534+0000", 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440534+0000", 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440533+0000", 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440542+0000", 2023-10-09T04:13:32.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9080000000000001e-06 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440555+0000", 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2544999999999999e-05 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440561+0000", 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8140000000000003e-06 2023-10-09T04:13:32.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440609+0000", 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8059000000000002e-05 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440732+0000", 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012246199999999999 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.440850+0000", 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011856799999999999 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.447574+0000", 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067238269999999999 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.451382+0000", 2023-10-09T04:13:32.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0038073410000000001 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573053+0000", 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.121671258 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573092+0000", 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8645e-05 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:02.573125+0000", 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3896999999999997e-05 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101632 7.b e209/167 7:d4bf3a27:::benchmark_data_smithi163_346082_object8362:head v 209'6353, mlcod=209'6351)", 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.900432+0000", 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.36420992800001, 2023-10-09T04:13:32.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16590674799999999, 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900432+0000", 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900432+0000", 2023-10-09T04:13:32.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900431+0000", 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900441+0000", 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8530000000000004e-06 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900451+0000", 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7960000000000007e-06 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900455+0000", 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3839999999999999e-06 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900504+0000", 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8786999999999999e-05 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900525+0000", 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0769000000000001e-05 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.066301+0000", 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.165776852 2023-10-09T04:13:32.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.066339+0000", 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7185999999999999e-05 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101631 7.d e209/167 7:b5aefcb9:::benchmark_data_smithi163_346082_object8361:head v 209'6360, mlcod=209'6358)", 2023-10-09T04:13:32.366 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.900468+0000", 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.364173988, 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15987315299999999, 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900468+0000", 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900468+0000", 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900467+0000", 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900471+0000", 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9639999999999997e-06 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900477+0000", 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6010000000000002e-06 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900485+0000", 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9140000000000008e-06 2023-10-09T04:13:32.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900563+0000", 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8036999999999998e-05 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900581+0000", 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8187999999999999e-05 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060301+0000", 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.159719371 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060341+0000", 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0363e-05 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101637 7.b e209/167 7:dd967b0a:::benchmark_data_smithi163_346082_object8367:head v 209'6354, mlcod=209'6351)", 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.900496+0000", 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.36414569199999, 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.165904316, 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900496+0000", 2023-10-09T04:13:32.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900496+0000", 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900496+0000", 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900499+0000", 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4319999999999999e-06 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900505+0000", 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4369999999999998e-06 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.900508+0000", 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3969999999999998e-06 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060335+0000", 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.159826986 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060360+0000", 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4932000000000001e-05 2023-10-09T04:13:32.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.066376+0000", 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060162210000000004 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.066400+0000", 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4124000000000001e-05 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.376 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101645 7.c e209/167 7:31886dfb:::benchmark_data_smithi163_346082_object8375:head v 209'6174, mlcod=209'6172)", 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.903622+0000", 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.36101994000001, 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15670747099999999, 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903622+0000", 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903622+0000", 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903621+0000", 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903629+0000", 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4340000000000003e-06 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903638+0000", 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3190000000000001e-06 2023-10-09T04:13:32.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903643+0000", 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4199999999999998e-06 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903692+0000", 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8498999999999998e-05 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.903710+0000", 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8681999999999998e-05 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060285+0000", 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15657406900000001 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060329+0000", 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4731000000000003e-05 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101646 7.11 e209/192 7:8c0d6217:::benchmark_data_smithi163_346082_object8376:head v 209'4291, mlcod=209'4289)", 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.905511+0000", 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.359130636, 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15814866699999999, 2023-10-09T04:13:32.382 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905511+0000", 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905511+0000", 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905510+0000", 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905518+0000", 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2169999999999997e-06 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905530+0000", 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.206e-05 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905535+0000", 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6710000000000004e-06 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905581+0000", 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6057999999999998e-05 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.905598+0000", 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6577000000000001e-05 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.063624+0000", 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.158026579 2023-10-09T04:13:32.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.063660+0000", 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5299000000000002e-05 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101650 7.c e209/167 7:323ad320:::benchmark_data_smithi163_346082_object8380:head v 209'6175, mlcod=209'6172)", 2023-10-09T04:13:32.387 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.906368+0000", 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.358273467, 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16563910300000001, 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.906368+0000", 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.906368+0000", 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.906368+0000", 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.906375+0000", 2023-10-09T04:13:32.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7190000000000001e-06 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060272+0000", 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15389663000000001 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060287+0000", 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4912999999999999e-05 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060378+0000", 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1461999999999999e-05 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060398+0000", 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9942999999999999e-05 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.071972+0000", 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011573597 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.072007+0000", 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5472000000000002e-05 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101658 7.b e209/167 7:de22303b:::benchmark_data_smithi163_346082_object8388:head v 209'6355, mlcod=209'6351)", 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.911650+0000", 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.35299152100001, 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15489455899999999, 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.911650+0000", 2023-10-09T04:13:32.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.911650+0000", 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.911650+0000", 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:13:32.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.911655+0000", 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8089999999999998e-06 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060283+0000", 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.148627761 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060293+0000", 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5880000000000004e-06 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.063522+0000", 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032297810000000001 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.063539+0000", 2023-10-09T04:13:32.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6099e-05 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.066520+0000", 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002981544 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.066545+0000", 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4644999999999999e-05 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.397 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:101657 7.12 e209/198 7:48acea92:::benchmark_data_smithi163_346082_object8387:head v 209'4420, mlcod=172'1794)", 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:09.919777+0000", 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 142.34486500200001, 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14921076599999999, 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.919777+0000", 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.919777+0000", 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.919775+0000", 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:09.919783+0000", 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9999999999999996e-06 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060322+0000", 2023-10-09T04:13:32.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14053874199999999 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060328+0000", 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0599999999999996e-06 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.060375+0000", 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6631000000000001e-05 2023-10-09T04:13:32.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.064532+0000", 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041571000000000004 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.068936+0000", 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004403644 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:10.068987+0000", 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1944999999999998e-05 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35115.0:104298 7.d e210/167 7:b677cbd9:::benchmark_data_smithi163_346082_object11028:head v 210'6519, mlcod=210'6516)", 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:13.061979+0000", 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 139.202662855, 2023-10-09T04:13:32.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14153690599999999, 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.061979+0000", 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.061979+0000", 2023-10-09T04:13:32.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.061977+0000", 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.061990+0000", 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2405999999999999e-05 2023-10-09T04:13:32.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.062010+0000", 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0143e-05 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.191061+0000", 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12905085499999999 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.191123+0000", 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2058999999999997e-05 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.191143+0000", 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9993999999999999e-05 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:13:32.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.203474+0000", 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012330827000000001 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:13.203516+0000", 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2058999999999998e-05 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDPGPush(7.12 210/198 [PushOp(7:4f1afc79:::benchmark_data_smithi163_346082_object30331:head, version: 172'1909, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:4f1afc79:::benchmark_data_smithi163_346082_object30331:head@172'1909, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:16.100065+0000", 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 136.16457711000001, 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13818196499999999, 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100065+0000", 2023-10-09T04:13:32.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100065+0000", 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100063+0000", 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:13:32.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100128+0000", 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4929000000000001e-05 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100162+0000", 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3300000000000003e-05 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100180+0000", 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8379000000000001e-05 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100275+0000", 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4617999999999994e-05 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.100280+0000", 2023-10-09T04:13:32.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.434e-06 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:16.238247+0000", 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13796656800000001 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:13:32.413 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:32 smithi181 ceph-mon[196979]: pgmap v632: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:32 smithi163 ceph-mon[221067]: pgmap v632: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:32.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:32 smithi163 ceph-mon[213841]: pgmap v632: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:32.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:13:33.291 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:34.790 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:13:34.790 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:13:34.790 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:13:34.790 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:13:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:34 smithi163 ceph-mon[221067]: pgmap v633: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:34.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:34 smithi163 ceph-mon[213841]: pgmap v633: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:34.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:34.964 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:34.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:34.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:34 smithi181 ceph-mon[196979]: pgmap v633: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:34.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:34.968 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:35.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:35.100 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:35.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:35.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:35.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:35.274 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:35.320 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:35.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:35.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:35.425 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:35.463 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:35.468 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:35.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:35.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:35.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:35.674 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:35.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:13:35.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:35.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:35.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:35.829 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:35.857 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:35.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:35.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:36.012 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:36.038 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:13:36.041 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:36.076 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:13:36.102 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:36.112 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:13:36.133 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:13:36.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:36.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:36.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:36.158 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:13:36.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:13:36.211 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:13:36.230 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:13:36.231 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.232 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:13:36.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:36.280 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:36.332 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:36.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:36.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:36.463 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:36.623 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_1' removed 2023-10-09T04:13:36.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:36.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:36.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:36.845 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: pgmap v634: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/220317540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/360361989' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:36.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: pgmap v634: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: failed to encode map e239 with expected crc 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/220317540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/360361989' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:36.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:36 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:36.850 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: pgmap v634: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.0 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2950/285 objects degraded (1035.088%) 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: failed to encode map e239 with expected crc 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: failed to encode map e239 with expected crc 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: failed to encode map e239 with expected crc 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: failed to encode map e239 with expected crc 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/220317540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/360361989' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:37.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:36 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:37.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:37.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:37.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:13:37.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:37.282 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:37.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:37.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:37.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:37.458 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:37.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:37.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:37.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:37.636 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:37.732 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: 7.4 scrub starts 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[221067]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.733 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:13:37.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:13:37.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: 7.4 scrub starts 2023-10-09T04:13:37.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.734 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:37 smithi163 ceph-mon[213841]: failed to encode map e240 with expected crc 2023-10-09T04:13:37.735 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:13:37.735 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:13:37.736 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T04:13:37.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T04:13:37.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:37.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:37.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:37.915 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: 7.4 scrub starts 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:37 smithi181 ceph-mon[196979]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.044 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:38.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:38.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:38.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:38.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:38.247 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:38.364 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:13:38.366 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:13:38.425 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:38.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:38.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:38.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:38.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:38.587 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:38.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:38.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:38.775 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:13:38.775 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:13:38.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:38 smithi181 ceph-mon[196979]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:38 smithi181 ceph-mon[196979]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:38.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:38 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 2 pgs degraded) 2023-10-09T04:13:38.799 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:38 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:13:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[221067]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[221067]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 2 pgs degraded) 2023-10-09T04:13:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:13:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[213841]: failed to encode map e240 with expected crc 2023-10-09T04:13:38.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[213841]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:38.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2950/285 objects degraded (1035.088%), 2 pgs degraded) 2023-10-09T04:13:38.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:38 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:13:38.835 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:38.864 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:39.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:39.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:39.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:39.137 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:39.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:39.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:39.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:13:39.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:39.515 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:39.676 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":637,"stamp":"2023-10-09T04:13:37.525967+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1772212,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2739404,"kb_used_data":136396,"kb_used_omap":126,"kb_used_meta":2602817,"kb_avail":747221812,"statfs":{"total":767960285184,"available":765155135488,"internally_reserved":0,"allocated":139669504,"data_stored":133120772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2665285243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-4620288,"num_objects":-71,"num_object_clones":0,"num_object_copies":-213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2950,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-71,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-157435,"num_write_kb":-4819873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-15272,"num_bytes_recovered":-833617920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002015"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":235,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349104+0000","last_change":"2023-10-09T04:10:38.371514+0000","last_active":"2023-10-09T04:13:27.349104+0000","last_peered":"2023-10-09T04:13:27.349104+0000","last_clean":"2023-10-09T04:13:27.349104+0000","last_became_active":"2023-10-09T04:10:38.371110+0000","last_became_peered":"2023-10-09T04:10:38.371110+0000","last_unstale":"2023-10-09T04:13:27.349104+0000","last_undegraded":"2023-10-09T04:13:27.349104+0000","last_fullsized":"2023-10-09T04:13:27.349104+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:55.246347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":174,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418286+0000","last_change":"2023-10-09T04:06:45.469980+0000","last_active":"2023-10-09T04:13:27.418286+0000","last_peered":"2023-10-09T04:13:27.418286+0000","last_clean":"2023-10-09T04:13:27.418286+0000","last_became_active":"2023-10-09T04:06:45.469676+0000","last_became_peered":"2023-10-09T04:06:45.469676+0000","last_unstale":"2023-10-09T04:13:27.418286+0000","last_undegraded":"2023-10-09T04:13:27.418286+0000","last_fullsized":"2023-10-09T04:13:27.418286+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:57.623174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":189,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.417945+0000","last_change":"2023-10-09T04:03:37.220333+0000","last_active":"2023-10-09T04:13:27.417945+0000","last_peered":"2023-10-09T04:13:27.417945+0000","last_clean":"2023-10-09T04:13:27.417945+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:13:27.417945+0000","last_undegraded":"2023-10-09T04:13:27.417945+0000","last_fullsized":"2023-10-09T04:13:27.417945+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:02.015229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":194,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346134+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:13:27.346134+0000","last_peered":"2023-10-09T04:13:27.346134+0000","last_clean":"2023-10-09T04:13:27.346134+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:13:27.346134+0000","last_undegraded":"2023-10-09T04:13:27.346134+0000","last_fullsized":"2023-10-09T04:13:27.346134+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":192,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.345180+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:13:27.345180+0000","last_peered":"2023-10-09T04:13:27.345180+0000","last_clean":"2023-10-09T04:13:27.345180+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:13:27.345180+0000","last_undegraded":"2023-10-09T04:13:27.345180+0000","last_fullsized":"2023-10-09T04:13:27.345180+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":172,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369567+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:13:28.369567+0000","last_peered":"2023-10-09T04:13:28.369567+0000","last_clean":"2023-10-09T04:13:28.369567+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:13:28.369567+0000","last_undegraded":"2023-10-09T04:13:28.369567+0000","last_fullsized":"2023-10-09T04:13:28.369567+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346207+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:13:27.346207+0000","last_peered":"2023-10-09T04:13:27.346207+0000","last_clean":"2023-10-09T04:13:27.346207+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:13:27.346207+0000","last_undegraded":"2023-10-09T04:13:27.346207+0000","last_fullsized":"2023-10-09T04:13:27.346207+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":218,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.358182+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:13:28.358182+0000","last_peered":"2023-10-09T04:13:28.358182+0000","last_clean":"2023-10-09T04:13:28.358182+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:13:28.358182+0000","last_undegraded":"2023-10-09T04:13:28.358182+0000","last_fullsized":"2023-10-09T04:13:28.358182+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":198,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369351+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:13:28.369351+0000","last_peered":"2023-10-09T04:13:28.369351+0000","last_clean":"2023-10-09T04:13:28.369351+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:13:28.369351+0000","last_undegraded":"2023-10-09T04:13:28.369351+0000","last_fullsized":"2023-10-09T04:13:28.369351+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364865+0000","last_change":"2023-10-09T04:06:45.471272+0000","last_active":"2023-10-09T04:13:28.364865+0000","last_peered":"2023-10-09T04:13:28.364865+0000","last_clean":"2023-10-09T04:13:28.364865+0000","last_became_active":"2023-10-09T04:06:45.470931+0000","last_became_peered":"2023-10-09T04:06:45.470931+0000","last_unstale":"2023-10-09T04:13:28.364865+0000","last_undegraded":"2023-10-09T04:13:28.364865+0000","last_fullsized":"2023-10-09T04:13:28.364865+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:08.009878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.365156+0000","last_change":"2023-10-09T04:03:37.220454+0000","last_active":"2023-10-09T04:13:28.365156+0000","last_peered":"2023-10-09T04:13:28.365156+0000","last_clean":"2023-10-09T04:13:28.365156+0000","last_became_active":"2023-10-09T04:03:37.220113+0000","last_became_peered":"2023-10-09T04:03:37.220113+0000","last_unstale":"2023-10-09T04:13:28.365156+0000","last_undegraded":"2023-10-09T04:13:28.365156+0000","last_fullsized":"2023-10-09T04:13:28.365156+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.578445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":190,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418222+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:13:27.418222+0000","last_peered":"2023-10-09T04:13:27.418222+0000","last_clean":"2023-10-09T04:13:27.418222+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:13:27.418222+0000","last_undegraded":"2023-10-09T04:13:27.418222+0000","last_fullsized":"2023-10-09T04:13:27.418222+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":202,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349559+0000","last_change":"2023-10-09T03:54:13.222014+0000","last_active":"2023-10-09T04:13:27.349559+0000","last_peered":"2023-10-09T04:13:27.349559+0000","last_clean":"2023-10-09T04:13:27.349559+0000","last_became_active":"2023-10-09T03:54:13.221441+0000","last_became_peered":"2023-10-09T03:54:13.221441+0000","last_unstale":"2023-10-09T04:13:27.349559+0000","last_undegraded":"2023-10-09T04:13:27.349559+0000","last_fullsized":"2023-10-09T04:13:27.349559+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:20.209052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":196,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349689+0000","last_change":"2023-10-09T04:02:35.092467+0000","last_active":"2023-10-09T04:13:27.349689+0000","last_peered":"2023-10-09T04:13:27.349689+0000","last_clean":"2023-10-09T04:13:27.349689+0000","last_became_active":"2023-10-09T04:02:35.092040+0000","last_became_peered":"2023-10-09T04:02:35.092040+0000","last_unstale":"2023-10-09T04:13:27.349689+0000","last_undegraded":"2023-10-09T04:13:27.349689+0000","last_fullsized":"2023-10-09T04:13:27.349689+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:01.683137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":200,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364882+0000","last_change":"2023-10-09T04:03:06.346254+0000","last_active":"2023-10-09T04:13:28.364882+0000","last_peered":"2023-10-09T04:13:28.364882+0000","last_clean":"2023-10-09T04:13:28.364882+0000","last_became_active":"2023-10-09T04:03:06.345886+0000","last_became_peered":"2023-10-09T04:03:06.345886+0000","last_unstale":"2023-10-09T04:13:28.364882+0000","last_undegraded":"2023-10-09T04:13:28.364882+0000","last_fullsized":"2023-10-09T04:13:28.364882+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:03:54.610163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":174,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418160+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:13:27.418160+0000","last_peered":"2023-10-09T04:13:27.418160+0000","last_clean":"2023-10-09T04:13:27.418160+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:13:27.418160+0000","last_undegraded":"2023-10-09T04:13:27.418160+0000","last_fullsized":"2023-10-09T04:13:27.418160+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":193,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.345338+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:13:27.345338+0000","last_peered":"2023-10-09T04:13:27.345338+0000","last_clean":"2023-10-09T04:13:27.345338+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:13:27.345338+0000","last_undegraded":"2023-10-09T04:13:27.345338+0000","last_fullsized":"2023-10-09T04:13:27.345338+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":197,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369119+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:13:28.369119+0000","last_peered":"2023-10-09T04:13:28.369119+0000","last_clean":"2023-10-09T04:13:28.369119+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:13:28.369119+0000","last_undegraded":"2023-10-09T04:13:28.369119+0000","last_fullsized":"2023-10-09T04:13:28.369119+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":181,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364830+0000","last_change":"2023-10-09T03:54:13.224309+0000","last_active":"2023-10-09T04:13:28.364830+0000","last_peered":"2023-10-09T04:13:28.364830+0000","last_clean":"2023-10-09T04:13:28.364830+0000","last_became_active":"2023-10-09T03:54:13.224010+0000","last_became_peered":"2023-10-09T03:54:13.224010+0000","last_unstale":"2023-10-09T04:13:28.364830+0000","last_undegraded":"2023-10-09T04:13:28.364830+0000","last_fullsized":"2023-10-09T04:13:28.364830+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:23.607537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":193,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346214+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:13:27.346214+0000","last_peered":"2023-10-09T04:13:27.346214+0000","last_clean":"2023-10-09T04:13:27.346214+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:13:27.346214+0000","last_undegraded":"2023-10-09T04:13:27.346214+0000","last_fullsized":"2023-10-09T04:13:27.346214+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":129,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.345090+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:13:27.345090+0000","last_peered":"2023-10-09T04:13:27.345090+0000","last_clean":"2023-10-09T04:13:27.345090+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:13:27.345090+0000","last_undegraded":"2023-10-09T04:13:27.345090+0000","last_fullsized":"2023-10-09T04:13:27.345090+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":204,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.359042+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:13:28.359042+0000","last_peered":"2023-10-09T04:13:28.359042+0000","last_clean":"2023-10-09T04:13:28.359042+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:13:28.359042+0000","last_undegraded":"2023-10-09T04:13:28.359042+0000","last_fullsized":"2023-10-09T04:13:28.359042+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":199,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.365203+0000","last_change":"2023-10-09T04:02:03.589753+0000","last_active":"2023-10-09T04:13:28.365203+0000","last_peered":"2023-10-09T04:13:28.365203+0000","last_clean":"2023-10-09T04:13:28.365203+0000","last_became_active":"2023-10-09T04:02:03.588878+0000","last_became_peered":"2023-10-09T04:02:03.588878+0000","last_unstale":"2023-10-09T04:13:28.365203+0000","last_undegraded":"2023-10-09T04:13:28.365203+0000","last_fullsized":"2023-10-09T04:13:28.365203+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.338851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":180,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.368914+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:13:28.368914+0000","last_peered":"2023-10-09T04:13:28.368914+0000","last_clean":"2023-10-09T04:13:28.368914+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:13:28.368914+0000","last_undegraded":"2023-10-09T04:13:28.368914+0000","last_fullsized":"2023-10-09T04:13:28.368914+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":200,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.365057+0000","last_change":"2023-10-09T04:02:35.092272+0000","last_active":"2023-10-09T04:13:28.365057+0000","last_peered":"2023-10-09T04:13:28.365057+0000","last_clean":"2023-10-09T04:13:28.365057+0000","last_became_active":"2023-10-09T04:02:35.091923+0000","last_became_peered":"2023-10-09T04:02:35.091923+0000","last_unstale":"2023-10-09T04:13:28.365057+0000","last_undegraded":"2023-10-09T04:13:28.365057+0000","last_fullsized":"2023-10-09T04:13:28.365057+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:18.375153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":213,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349161+0000","last_change":"2023-10-09T04:03:37.225348+0000","last_active":"2023-10-09T04:13:27.349161+0000","last_peered":"2023-10-09T04:13:27.349161+0000","last_clean":"2023-10-09T04:13:27.349161+0000","last_became_active":"2023-10-09T04:03:37.225036+0000","last_became_peered":"2023-10-09T04:03:37.225036+0000","last_unstale":"2023-10-09T04:13:27.349161+0000","last_undegraded":"2023-10-09T04:13:27.349161+0000","last_fullsized":"2023-10-09T04:13:27.349161+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:55.978464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":182,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364998+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:13:28.364998+0000","last_peered":"2023-10-09T04:13:28.364998+0000","last_clean":"2023-10-09T04:13:28.364998+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:13:28.364998+0000","last_undegraded":"2023-10-09T04:13:28.364998+0000","last_fullsized":"2023-10-09T04:13:28.364998+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.718107+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:13:28.718107+0000","last_peered":"2023-10-09T04:13:28.718107+0000","last_clean":"2023-10-09T04:13:28.718107+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:13:28.718107+0000","last_undegraded":"2023-10-09T04:13:28.718107+0000","last_fullsized":"2023-10-09T04:13:28.718107+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":198,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369332+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:13:28.369332+0000","last_peered":"2023-10-09T04:13:28.369332+0000","last_clean":"2023-10-09T04:13:28.369332+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:13:28.369332+0000","last_undegraded":"2023-10-09T04:13:28.369332+0000","last_fullsized":"2023-10-09T04:13:28.369332+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":194,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346394+0000","last_change":"2023-10-09T04:02:35.093224+0000","last_active":"2023-10-09T04:13:27.346394+0000","last_peered":"2023-10-09T04:13:27.346394+0000","last_clean":"2023-10-09T04:13:27.346394+0000","last_became_active":"2023-10-09T04:02:35.092925+0000","last_became_peered":"2023-10-09T04:02:35.092925+0000","last_unstale":"2023-10-09T04:13:27.346394+0000","last_undegraded":"2023-10-09T04:13:27.346394+0000","last_fullsized":"2023-10-09T04:13:27.346394+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:21.090406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":143,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.360233+0000","last_change":"2023-10-09T04:06:45.341236+0000","last_active":"2023-10-09T04:13:28.360233+0000","last_peered":"2023-10-09T04:13:28.360233+0000","last_clean":"2023-10-09T04:13:28.360233+0000","last_became_active":"2023-10-09T04:06:45.340748+0000","last_became_peered":"2023-10-09T04:06:45.340748+0000","last_unstale":"2023-10-09T04:13:28.360233+0000","last_undegraded":"2023-10-09T04:13:28.360233+0000","last_fullsized":"2023-10-09T04:13:28.360233+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:08.632250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":196,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349469+0000","last_change":"2023-10-09T04:03:37.225364+0000","last_active":"2023-10-09T04:13:27.349469+0000","last_peered":"2023-10-09T04:13:27.349469+0000","last_clean":"2023-10-09T04:13:27.349469+0000","last_became_active":"2023-10-09T04:03:37.225032+0000","last_became_peered":"2023-10-09T04:03:37.225032+0000","last_unstale":"2023-10-09T04:13:27.349469+0000","last_undegraded":"2023-10-09T04:13:27.349469+0000","last_fullsized":"2023-10-09T04:13:27.349469+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:47.501524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":157,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349148+0000","last_change":"2023-10-09T04:06:57.495883+0000","last_active":"2023-10-09T04:13:27.349148+0000","last_peered":"2023-10-09T04:13:27.349148+0000","last_clean":"2023-10-09T04:13:27.349148+0000","last_became_active":"2023-10-09T04:06:57.495412+0000","last_became_peered":"2023-10-09T04:06:57.495412+0000","last_unstale":"2023-10-09T04:13:27.349148+0000","last_undegraded":"2023-10-09T04:13:27.349148+0000","last_fullsized":"2023-10-09T04:13:27.349148+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:22.220637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":233,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349609+0000","last_change":"2023-10-09T04:10:38.340387+0000","last_active":"2023-10-09T04:13:27.349609+0000","last_peered":"2023-10-09T04:13:27.349609+0000","last_clean":"2023-10-09T04:13:27.349609+0000","last_became_active":"2023-10-09T04:10:38.340048+0000","last_became_peered":"2023-10-09T04:10:38.340048+0000","last_unstale":"2023-10-09T04:13:27.349609+0000","last_undegraded":"2023-10-09T04:13:27.349609+0000","last_fullsized":"2023-10-09T04:13:27.349609+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:01.107441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349020+0000","last_change":"2023-10-09T04:03:06.346026+0000","last_active":"2023-10-09T04:13:27.349020+0000","last_peered":"2023-10-09T04:13:27.349020+0000","last_clean":"2023-10-09T04:13:27.349020+0000","last_became_active":"2023-10-09T04:03:06.345708+0000","last_became_peered":"2023-10-09T04:03:06.345708+0000","last_unstale":"2023-10-09T04:13:27.349020+0000","last_undegraded":"2023-10-09T04:13:27.349020+0000","last_fullsized":"2023-10-09T04:13:27.349020+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:08.860029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349342+0000","last_change":"2023-10-09T03:55:18.575220+0000","last_active":"2023-10-09T04:13:27.349342+0000","last_peered":"2023-10-09T04:13:27.349342+0000","last_clean":"2023-10-09T04:13:27.349342+0000","last_became_active":"2023-10-09T03:55:18.574851+0000","last_became_peered":"2023-10-09T03:55:18.574851+0000","last_unstale":"2023-10-09T04:13:27.349342+0000","last_undegraded":"2023-10-09T04:13:27.349342+0000","last_fullsized":"2023-10-09T04:13:27.349342+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:12.647527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":205,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.358103+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:13:28.358103+0000","last_peered":"2023-10-09T04:13:28.358103+0000","last_clean":"2023-10-09T04:13:28.358103+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:13:28.358103+0000","last_undegraded":"2023-10-09T04:13:28.358103+0000","last_fullsized":"2023-10-09T04:13:28.358103+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418391+0000","last_change":"2023-10-09T04:03:37.219140+0000","last_active":"2023-10-09T04:13:27.418391+0000","last_peered":"2023-10-09T04:13:27.418391+0000","last_clean":"2023-10-09T04:13:27.418391+0000","last_became_active":"2023-10-09T04:03:37.218842+0000","last_became_peered":"2023-10-09T04:03:37.218842+0000","last_unstale":"2023-10-09T04:13:27.418391+0000","last_undegraded":"2023-10-09T04:13:27.418391+0000","last_fullsized":"2023-10-09T04:13:27.418391+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:37.935266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":214,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418279+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:13:27.418279+0000","last_peered":"2023-10-09T04:13:27.418279+0000","last_clean":"2023-10-09T04:13:27.418279+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:13:27.418279+0000","last_undegraded":"2023-10-09T04:13:27.418279+0000","last_fullsized":"2023-10-09T04:13:27.418279+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":215,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369296+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:13:28.369296+0000","last_peered":"2023-10-09T04:13:28.369296+0000","last_clean":"2023-10-09T04:13:28.369296+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:13:28.369296+0000","last_undegraded":"2023-10-09T04:13:28.369296+0000","last_fullsized":"2023-10-09T04:13:28.369296+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":176,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418037+0000","last_change":"2023-10-09T04:02:35.092660+0000","last_active":"2023-10-09T04:13:27.418037+0000","last_peered":"2023-10-09T04:13:27.418037+0000","last_clean":"2023-10-09T04:13:27.418037+0000","last_became_active":"2023-10-09T04:02:35.092383+0000","last_became_peered":"2023-10-09T04:02:35.092383+0000","last_unstale":"2023-10-09T04:13:27.418037+0000","last_undegraded":"2023-10-09T04:13:27.418037+0000","last_fullsized":"2023-10-09T04:13:27.418037+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:40.923985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":202,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369218+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:13:28.369218+0000","last_peered":"2023-10-09T04:13:28.369218+0000","last_clean":"2023-10-09T04:13:28.369218+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:13:28.369218+0000","last_undegraded":"2023-10-09T04:13:28.369218+0000","last_fullsized":"2023-10-09T04:13:28.369218+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":197,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349691+0000","last_change":"2023-10-09T04:02:03.589238+0000","last_active":"2023-10-09T04:13:27.349691+0000","last_peered":"2023-10-09T04:13:27.349691+0000","last_clean":"2023-10-09T04:13:27.349691+0000","last_became_active":"2023-10-09T04:02:03.588876+0000","last_became_peered":"2023-10-09T04:02:03.588876+0000","last_unstale":"2023-10-09T04:13:27.349691+0000","last_undegraded":"2023-10-09T04:13:27.349691+0000","last_fullsized":"2023-10-09T04:13:27.349691+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:36.637432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":224,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418229+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:13:27.418229+0000","last_peered":"2023-10-09T04:13:27.418229+0000","last_clean":"2023-10-09T04:13:27.418229+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:13:27.418229+0000","last_undegraded":"2023-10-09T04:13:27.418229+0000","last_fullsized":"2023-10-09T04:13:27.418229+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":52,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.343387+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:13:27.343387+0000","last_peered":"2023-10-09T04:13:27.343387+0000","last_clean":"2023-10-09T04:13:27.343387+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:13:27.343387+0000","last_undegraded":"2023-10-09T04:13:27.343387+0000","last_fullsized":"2023-10-09T04:13:27.343387+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":208,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.417760+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:13:27.417760+0000","last_peered":"2023-10-09T04:13:27.417760+0000","last_clean":"2023-10-09T04:13:27.417760+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:13:27.417760+0000","last_undegraded":"2023-10-09T04:13:27.417760+0000","last_fullsized":"2023-10-09T04:13:27.417760+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418394+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:13:27.418394+0000","last_peered":"2023-10-09T04:13:27.418394+0000","last_clean":"2023-10-09T04:13:27.418394+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:13:27.418394+0000","last_undegraded":"2023-10-09T04:13:27.418394+0000","last_fullsized":"2023-10-09T04:13:27.418394+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":222,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369370+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:13:28.369370+0000","last_peered":"2023-10-09T04:13:28.369370+0000","last_clean":"2023-10-09T04:13:28.369370+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:13:28.369370+0000","last_undegraded":"2023-10-09T04:13:28.369370+0000","last_fullsized":"2023-10-09T04:13:28.369370+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":579,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364780+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:13:28.364780+0000","last_peered":"2023-10-09T04:13:28.364780+0000","last_clean":"2023-10-09T04:13:28.364780+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:13:28.364780+0000","last_undegraded":"2023-10-09T04:13:28.364780+0000","last_fullsized":"2023-10-09T04:13:28.364780+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369140+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:13:28.369140+0000","last_peered":"2023-10-09T04:13:28.369140+0000","last_clean":"2023-10-09T04:13:28.369140+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:13:28.369140+0000","last_undegraded":"2023-10-09T04:13:28.369140+0000","last_fullsized":"2023-10-09T04:13:28.369140+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":117,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349471+0000","last_change":"2023-10-09T04:06:57.494412+0000","last_active":"2023-10-09T04:13:27.349471+0000","last_peered":"2023-10-09T04:13:27.349471+0000","last_clean":"2023-10-09T04:13:27.349471+0000","last_became_active":"2023-10-09T04:06:57.494208+0000","last_became_peered":"2023-10-09T04:06:57.494208+0000","last_unstale":"2023-10-09T04:13:27.349471+0000","last_undegraded":"2023-10-09T04:13:27.349471+0000","last_fullsized":"2023-10-09T04:13:27.349471+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:02:03.705061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":219,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364692+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:13:28.364692+0000","last_peered":"2023-10-09T04:13:28.364692+0000","last_clean":"2023-10-09T04:13:28.364692+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:13:28.364692+0000","last_undegraded":"2023-10-09T04:13:28.364692+0000","last_fullsized":"2023-10-09T04:13:28.364692+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":194,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.345275+0000","last_change":"2023-10-09T04:03:37.219362+0000","last_active":"2023-10-09T04:13:27.345275+0000","last_peered":"2023-10-09T04:13:27.345275+0000","last_clean":"2023-10-09T04:13:27.345275+0000","last_became_active":"2023-10-09T04:03:37.219069+0000","last_became_peered":"2023-10-09T04:03:37.219069+0000","last_unstale":"2023-10-09T04:13:27.345275+0000","last_undegraded":"2023-10-09T04:13:27.345275+0000","last_fullsized":"2023-10-09T04:13:27.345275+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:51.020036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":227,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.345684+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:13:27.345684+0000","last_peered":"2023-10-09T04:13:27.345684+0000","last_clean":"2023-10-09T04:13:27.345684+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:13:27.345684+0000","last_undegraded":"2023-10-09T04:13:27.345684+0000","last_fullsized":"2023-10-09T04:13:27.345684+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":202,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.365068+0000","last_change":"2023-10-09T04:02:03.586195+0000","last_active":"2023-10-09T04:13:28.365068+0000","last_peered":"2023-10-09T04:13:28.365068+0000","last_clean":"2023-10-09T04:13:28.365068+0000","last_became_active":"2023-10-09T04:02:03.585848+0000","last_became_peered":"2023-10-09T04:02:03.585848+0000","last_unstale":"2023-10-09T04:13:28.365068+0000","last_undegraded":"2023-10-09T04:13:28.365068+0000","last_fullsized":"2023-10-09T04:13:28.365068+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:03.954241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":208,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418337+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:13:27.418337+0000","last_peered":"2023-10-09T04:13:27.418337+0000","last_clean":"2023-10-09T04:13:27.418337+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:13:27.418337+0000","last_undegraded":"2023-10-09T04:13:27.418337+0000","last_fullsized":"2023-10-09T04:13:27.418337+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":179,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346263+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:13:27.346263+0000","last_peered":"2023-10-09T04:13:27.346263+0000","last_clean":"2023-10-09T04:13:27.346263+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:13:27.346263+0000","last_undegraded":"2023-10-09T04:13:27.346263+0000","last_fullsized":"2023-10-09T04:13:27.346263+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":143,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.365112+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:13:28.365112+0000","last_peered":"2023-10-09T04:13:28.365112+0000","last_clean":"2023-10-09T04:13:28.365112+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:13:28.365112+0000","last_undegraded":"2023-10-09T04:13:28.365112+0000","last_fullsized":"2023-10-09T04:13:28.365112+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":175,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.418185+0000","last_change":"2023-10-09T04:02:35.092718+0000","last_active":"2023-10-09T04:13:27.418185+0000","last_peered":"2023-10-09T04:13:27.418185+0000","last_clean":"2023-10-09T04:13:27.418185+0000","last_became_active":"2023-10-09T04:02:35.092398+0000","last_became_peered":"2023-10-09T04:02:35.092398+0000","last_unstale":"2023-10-09T04:13:27.418185+0000","last_undegraded":"2023-10-09T04:13:27.418185+0000","last_fullsized":"2023-10-09T04:13:27.418185+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:33.103847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.364934+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:13:28.364934+0000","last_peered":"2023-10-09T04:13:28.364934+0000","last_clean":"2023-10-09T04:13:28.364934+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:13:28.364934+0000","last_undegraded":"2023-10-09T04:13:28.364934+0000","last_fullsized":"2023-10-09T04:13:28.364934+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":167,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.365166+0000","last_change":"2023-10-09T04:06:57.498098+0000","last_active":"2023-10-09T04:13:28.365166+0000","last_peered":"2023-10-09T04:13:28.365166+0000","last_clean":"2023-10-09T04:13:28.365166+0000","last_became_active":"2023-10-09T04:06:57.497794+0000","last_became_peered":"2023-10-09T04:06:57.497794+0000","last_unstale":"2023-10-09T04:13:28.365166+0000","last_undegraded":"2023-10-09T04:13:28.365166+0000","last_fullsized":"2023-10-09T04:13:28.365166+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:29.358760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":198,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.345701+0000","last_change":"2023-10-09T04:02:35.092588+0000","last_active":"2023-10-09T04:13:27.345701+0000","last_peered":"2023-10-09T04:13:27.345701+0000","last_clean":"2023-10-09T04:13:27.345701+0000","last_became_active":"2023-10-09T04:02:35.092201+0000","last_became_peered":"2023-10-09T04:02:35.092201+0000","last_unstale":"2023-10-09T04:13:27.345701+0000","last_undegraded":"2023-10-09T04:13:27.345701+0000","last_fullsized":"2023-10-09T04:13:27.345701+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:31.250535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":179,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349030+0000","last_change":"2023-10-09T03:54:13.239885+0000","last_active":"2023-10-09T04:13:27.349030+0000","last_peered":"2023-10-09T04:13:27.349030+0000","last_clean":"2023-10-09T04:13:27.349030+0000","last_became_active":"2023-10-09T03:54:13.239585+0000","last_became_peered":"2023-10-09T03:54:13.239585+0000","last_unstale":"2023-10-09T04:13:27.349030+0000","last_undegraded":"2023-10-09T04:13:27.349030+0000","last_fullsized":"2023-10-09T04:13:27.349030+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:57.421909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":182,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346439+0000","last_change":"2023-10-09T04:02:03.589304+0000","last_active":"2023-10-09T04:13:27.346439+0000","last_peered":"2023-10-09T04:13:27.346439+0000","last_clean":"2023-10-09T04:13:27.346439+0000","last_became_active":"2023-10-09T04:02:03.589000+0000","last_became_peered":"2023-10-09T04:02:03.589000+0000","last_unstale":"2023-10-09T04:13:27.346439+0000","last_undegraded":"2023-10-09T04:13:27.346439+0000","last_fullsized":"2023-10-09T04:13:27.346439+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:01.705712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":52,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.343686+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:13:27.343686+0000","last_peered":"2023-10-09T04:13:27.343686+0000","last_clean":"2023-10-09T04:13:27.343686+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:13:27.343686+0000","last_undegraded":"2023-10-09T04:13:27.343686+0000","last_fullsized":"2023-10-09T04:13:27.343686+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":200,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369385+0000","last_change":"2023-10-09T04:02:35.091612+0000","last_active":"2023-10-09T04:13:28.369385+0000","last_peered":"2023-10-09T04:13:28.369385+0000","last_clean":"2023-10-09T04:13:28.369385+0000","last_became_active":"2023-10-09T04:02:35.091232+0000","last_became_peered":"2023-10-09T04:02:35.091232+0000","last_unstale":"2023-10-09T04:13:28.369385+0000","last_undegraded":"2023-10-09T04:13:28.369385+0000","last_fullsized":"2023-10-09T04:13:28.369385+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:34.594533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":213,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349743+0000","last_change":"2023-10-09T04:03:37.225542+0000","last_active":"2023-10-09T04:13:27.349743+0000","last_peered":"2023-10-09T04:13:27.349743+0000","last_clean":"2023-10-09T04:13:27.349743+0000","last_became_active":"2023-10-09T04:03:37.225137+0000","last_became_peered":"2023-10-09T04:03:37.225137+0000","last_unstale":"2023-10-09T04:13:27.349743+0000","last_undegraded":"2023-10-09T04:13:27.349743+0000","last_fullsized":"2023-10-09T04:13:27.349743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:07.332786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":183,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-10-09T04:13:28.369234+0000","last_change":"2023-10-09T03:54:13.225304+0000","last_active":"2023-10-09T04:13:28.369234+0000","last_peered":"2023-10-09T04:13:28.369234+0000","last_clean":"2023-10-09T04:13:28.369234+0000","last_became_active":"2023-10-09T03:54:13.224752+0000","last_became_peered":"2023-10-09T03:54:13.224752+0000","last_unstale":"2023-10-09T04:13:28.369234+0000","last_undegraded":"2023-10-09T04:13:28.369234+0000","last_fullsized":"2023-10-09T04:13:28.369234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:05.175551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":213,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.349257+0000","last_change":"2023-10-09T04:03:37.224382+0000","last_active":"2023-10-09T04:13:27.349257+0000","last_peered":"2023-10-09T04:13:27.349257+0000","last_clean":"2023-10-09T04:13:27.349257+0000","last_became_active":"2023-10-09T04:03:37.224088+0000","last_became_peered":"2023-10-09T04:03:37.224088+0000","last_unstale":"2023-10-09T04:13:27.349257+0000","last_undegraded":"2023-10-09T04:13:27.349257+0000","last_fullsized":"2023-10-09T04:13:27.349257+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:41.684349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":52,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.343847+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:13:27.343847+0000","last_peered":"2023-10-09T04:13:27.343847+0000","last_clean":"2023-10-09T04:13:27.343847+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:13:27.343847+0000","last_undegraded":"2023-10-09T04:13:27.343847+0000","last_fullsized":"2023-10-09T04:13:27.343847+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":52,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.343418+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:13:27.343418+0000","last_peered":"2023-10-09T04:13:27.343418+0000","last_clean":"2023-10-09T04:13:27.343418+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:13:27.343418+0000","last_undegraded":"2023-10-09T04:13:27.343418+0000","last_fullsized":"2023-10-09T04:13:27.343418+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":228,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.346231+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:13:27.346231+0000","last_peered":"2023-10-09T04:13:27.346231+0000","last_clean":"2023-10-09T04:13:27.346231+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:13:27.346231+0000","last_undegraded":"2023-10-09T04:13:27.346231+0000","last_fullsized":"2023-10-09T04:13:27.346231+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":196,"reported_epoch":237,"state":"active+clean","last_fresh":"2023-10-09T04:13:27.348987+0000","last_change":"2023-10-09T04:02:35.094075+0000","last_active":"2023-10-09T04:13:27.348987+0000","last_peered":"2023-10-09T04:13:27.348987+0000","last_clean":"2023-10-09T04:13:27.348987+0000","last_became_active":"2023-10-09T04:02:35.093735+0000","last_became_peered":"2023-10-09T04:02:35.093735+0000","last_unstale":"2023-10-09T04:13:27.348987+0000","last_undegraded":"2023-10-09T04:13:27.348987+0000","last_fullsized":"2023-10-09T04:13:27.348987+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:52:41.562554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201533,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308376,"kb_used_data":2896,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93436776,"statfs":{"total":95995035648,"available":95679258624,"internally_reserved":0,"allocated":2965504,"data_stored":2266254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":6.3760000000000003,"5min":4.6859999999999999,"15min":2.254},"min":{"1min":0.23400000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":88.144999999999996,"5min":126.752,"15min":126.752},"last":0.24399999999999999},{"interface":"front","average":{"1min":6.7599999999999998,"5min":4.9119999999999999,"15min":2.294},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":89.284999999999997,"5min":128.88399999999999,"15min":128.88399999999999},"last":0.27400000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.0430000000000001,"5min":6.6180000000000003,"15min":2.8260000000000001},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":101.471,"5min":245.28700000000001,"15min":245.28700000000001},"last":0.307},{"interface":"front","average":{"1min":6.7069999999999999,"5min":6.3150000000000004,"15min":2.7799999999999998},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":98.527000000000001,"5min":244.626,"15min":244.626},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":5.8410000000000002,"5min":2.2240000000000002,"15min":1.6719999999999999},"min":{"1min":0.27800000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":76.774000000000001,"5min":76.774000000000001,"15min":76.774000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":6.0250000000000004,"5min":2.407,"15min":2.1309999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.27600000000000002},"max":{"1min":76.715999999999994,"5min":76.715999999999994,"15min":99.372},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:12:49 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":1.7649999999999999,"15min":1.7370000000000001},"min":{"1min":0.34000000000000002,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":35.771000000000001,"5min":35.771000000000001,"15min":99.134},"last":0.45100000000000001},{"interface":"front","average":{"1min":3.6019999999999999,"5min":1.9419999999999999,"15min":1.839},"min":{"1min":0.34599999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":35.848999999999997,"5min":35.848999999999997,"15min":99.117999999999995},"last":0.45600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.80400000000000005,"15min":0.69999999999999996},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.8220000000000001,"5min":16.195,"15min":16.195},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.755,"5min":0.89400000000000002,"15min":0.73899999999999999},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.3929999999999998,"5min":12.442,"15min":12.442},"last":0.441}]},{"osd":5,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.629,"15min":0.626},"min":{"1min":0.30399999999999999,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":3.4020000000000001,"5min":4.4829999999999997,"15min":6.5439999999999996},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64300000000000002,"15min":0.68799999999999994},"min":{"1min":0.29499999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.7250000000000001,"5min":4.9359999999999999,"15min":5.6779999999999999},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.72199999999999998,"15min":0.71499999999999997},"min":{"1min":0.36199999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":3.431,"5min":4.5199999999999996,"15min":8.3559999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.73399999999999999,"15min":0.75700000000000001},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":1.456,"5min":4.9089999999999998,"15min":5.9039999999999999},"last":0.42799999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463170,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300168,"kb_used_data":2880,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93444984,"statfs":{"total":95995035648,"available":95687663616,"internally_reserved":0,"allocated":2949120,"data_stored":2265785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:13:23 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":3.9079999999999999,"15min":2.2410000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.0540000000000003,"5min":77.453000000000003,"15min":78.983999999999995},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.8799999999999999,"5min":4.3630000000000004,"15min":2.6030000000000002},"min":{"1min":0.252,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":10.742000000000001,"5min":77.430999999999997,"15min":78.998000000000005},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:54 2023","interfaces":[{"interface":"back","average":{"1min":4.2729999999999997,"5min":3.2250000000000001,"15min":1.9450000000000001},"min":{"1min":0.25800000000000001,"5min":0.25600000000000001,"15min":0.23300000000000001},"max":{"1min":53.337000000000003,"5min":62.698,"15min":62.698},"last":0.36399999999999999},{"interface":"front","average":{"1min":4.0529999999999999,"5min":3.4670000000000001,"15min":2.0310000000000001},"min":{"1min":0.27000000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":53.146000000000001,"5min":63.530999999999999,"15min":63.530999999999999},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:13:23 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":3.5550000000000002,"15min":2.1120000000000001},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.218},"max":{"1min":4.9690000000000003,"5min":77.953000000000003,"15min":77.953000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.738,"5min":3.6030000000000002,"15min":2.1600000000000001},"min":{"1min":0.433,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":5.0049999999999999,"5min":78.019000000000005,"15min":78.019000000000005},"last":0.443}]},{"osd":3,"last update":"Mon Oct 9 04:12:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5570000000000004,"5min":4.157,"15min":2.2970000000000002},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":51.082000000000001,"5min":57.509999999999998,"15min":57.509999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":4.556,"5min":3.9620000000000002,"15min":2.3119999999999998},"min":{"1min":0.32600000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":51.031999999999996,"5min":57.453000000000003,"15min":57.453000000000003},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.45100000000000001,"15min":0.48399999999999999},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.52500000000000002,"5min":2.3889999999999998,"15min":2.6909999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.57399999999999995,"15min":0.64200000000000002},"min":{"1min":0.26000000000000001,"5min":0.252,"15min":0.252},"max":{"1min":1.637,"5min":3.2970000000000002,"15min":7.7000000000000002},"last":0.32600000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.85299999999999998,"15min":0.85399999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.0089999999999999,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.374},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.93600000000000005,"15min":0.96299999999999997},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.998,"5min":23.006,"15min":54.853999999999999},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.67000000000000004,"15min":0.628},"min":{"1min":0.34000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.2310000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.73599999999999999,"15min":0.71399999999999997},"min":{"1min":0.33600000000000002,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":1.6930000000000001,"5min":3.8730000000000002,"15min":4.7679999999999998},"last":0.28899999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089478,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285804,"kb_used_data":31396,"kb_used_omap":9,"kb_used_meta":254390,"kb_avail":93459348,"statfs":{"total":95995035648,"available":95702372352,"internally_reserved":0,"allocated":32149504,"data_stored":31197502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":260496361},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":8.9399999999999995,"5min":4.9530000000000003,"15min":2.4140000000000001},"min":{"1min":0.318,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":68.048000000000002,"5min":85.977999999999994,"15min":85.977999999999994},"last":0.38300000000000001},{"interface":"front","average":{"1min":6.0739999999999998,"5min":4.7610000000000001,"15min":2.73},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":25.620999999999999,"5min":85.998999999999995,"15min":99.831999999999994},"last":0.25600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.3260000000000001,"5min":3.0640000000000001,"15min":1.958},"min":{"1min":0.23899999999999999,"5min":0.191,"15min":0.191},"max":{"1min":25.577000000000002,"5min":77.563000000000002,"15min":121.54600000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.4409999999999998,"5min":3.3399999999999999,"15min":2.149},"min":{"1min":0.215,"5min":0.215,"15min":0.186},"max":{"1min":12.039,"5min":77.504999999999995,"15min":99.899000000000001},"last":0.22800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":9.3320000000000007,"5min":5.5259999999999998,"15min":2.7280000000000002},"min":{"1min":0.309,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":50.542999999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":9.9339999999999993,"5min":6.0990000000000002,"15min":3.2669999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":44.716000000000001,"5min":138.49799999999999,"15min":138.49799999999999},"last":0.28000000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.214,"15min":1.6879999999999999},"min":{"1min":0.29099999999999998,"5min":0.254,"15min":0.254},"max":{"1min":11.973000000000001,"5min":15.215999999999999,"15min":99.989999999999995},"last":0.378},{"interface":"front","average":{"1min":4.8760000000000003,"5min":1.972,"15min":1.657},"min":{"1min":0.42999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":25.957999999999998,"5min":25.957999999999998,"15min":100.249},"last":0.503}]},{"osd":5,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":2.113,"15min":1.663},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":12.013999999999999,"5min":15.266,"15min":99.933999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":4.8319999999999999,"5min":2.2040000000000002,"15min":1.768},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.307},"max":{"1min":25.890999999999998,"5min":25.890999999999998,"15min":100.34099999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":4.7990000000000004,"5min":2.2719999999999998,"15min":1.8600000000000001},"min":{"1min":0.45300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":25.931000000000001,"5min":25.931000000000001,"15min":100.309},"last":0.54800000000000004},{"interface":"front","average":{"1min":3.1549999999999998,"5min":1.72,"15min":1.6779999999999999},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.446,"5min":13.446,"15min":100.143},"last":0.63500000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":4.9329999999999998,"5min":2.3370000000000002,"15min":1.8480000000000001},"min":{"1min":0.53300000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":25.984999999999999,"5min":25.984999999999999,"15min":100.214},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.0899999999999999,"5min":1.9970000000000001,"15min":1.698},"min":{"1min":0.42599999999999999,"5min":0.374,"15min":0.317},"max":{"1min":13.471,"5min":13.471,"15min":100.113},"last":0.59699999999999998}]}]},{"osd":0,"up_from":84,"seq":360777253004,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310664,"kb_used_data":8576,"kb_used_omap":28,"kb_used_meta":302051,"kb_avail":93434488,"statfs":{"total":95995035648,"available":95676915712,"internally_reserved":0,"allocated":8781824,"data_stored":7801615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":309301072},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":5.2050000000000001,"5min":3.0289999999999999,"15min":2.3050000000000002},"min":{"1min":0.20499999999999999,"5min":0.184,"15min":0.17000000000000001},"max":{"1min":37.764000000000003,"5min":44.287999999999997,"15min":118.48},"last":1.4019999999999999},{"interface":"front","average":{"1min":5.718,"5min":3.5859999999999999,"15min":2.5649999999999999},"min":{"1min":0.28000000000000003,"5min":0.24299999999999999,"15min":0.214},"max":{"1min":38.128999999999998,"5min":43.906999999999996,"15min":118.384},"last":0.91600000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:13:26 2023","interfaces":[{"interface":"back","average":{"1min":2.7170000000000001,"5min":3.2210000000000001,"15min":2.4020000000000001},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.221},"max":{"1min":12.531000000000001,"5min":25.055,"15min":118.54000000000001},"last":0.318},{"interface":"front","average":{"1min":3.3220000000000001,"5min":2.8820000000000001,"15min":2.5609999999999999},"min":{"1min":0.22900000000000001,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":12.821,"5min":16.515000000000001,"15min":118.727},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.6109999999999998,"5min":5.0629999999999997,"15min":3.8820000000000001},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":18.196000000000002,"5min":156.488,"15min":156.488},"last":0.45400000000000001},{"interface":"front","average":{"1min":3.944,"5min":5.0300000000000002,"15min":3.923},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":12.465999999999999,"5min":156.50700000000001,"15min":156.50700000000001},"last":2.9900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.0219999999999998,"15min":1.8080000000000001},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7270000000000003,"5min":10.023,"15min":118.633},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.714,"5min":2.4769999999999999,"15min":2.0379999999999998},"min":{"1min":0.39800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.492000000000001,"5min":44.475000000000001,"15min":118.913},"last":0.55800000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.6360000000000001,"15min":2.552},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":10.445,"5min":44.494,"15min":182.68899999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.887,"5min":2.2210000000000001,"15min":2.387},"min":{"1min":0.41299999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.4749999999999996,"5min":44.328000000000003,"15min":182.66300000000001},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":3.9180000000000001,"15min":2.3010000000000002},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.5090000000000003,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":3.649,"5min":3.9220000000000002,"15min":2.2679999999999998},"min":{"1min":0.44500000000000001,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":12.496,"5min":136.88999999999999,"15min":136.88999999999999},"last":0.46899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":3.3410000000000002,"15min":2.9169999999999998},"min":{"1min":0.38700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.5500000000000007,"5min":121.324,"15min":185.024},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.3239999999999998,"15min":2.508},"min":{"1min":0.46700000000000003,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":12.417,"5min":12.417,"15min":184.98599999999999},"last":0.434}]}]},{"osd":2,"up_from":94,"seq":403726925952,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362400,"kb_used_data":2584,"kb_used_omap":81,"kb_used_meta":359726,"kb_avail":93382752,"statfs":{"total":95995035648,"available":95623938048,"internally_reserved":0,"allocated":2646016,"data_stored":1704053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":368359642},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:13:22 2023","interfaces":[{"interface":"back","average":{"1min":5.242,"5min":3.2269999999999999,"15min":1.796},"min":{"1min":0.20499999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":46.750999999999998,"5min":46.750999999999998,"15min":46.750999999999998},"last":0.251},{"interface":"front","average":{"1min":5.3179999999999996,"5min":3.5990000000000002,"15min":2.0659999999999998},"min":{"1min":0.22800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":48.350999999999999,"5min":48.350999999999999,"15min":48.350999999999999},"last":0.26500000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:13:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.6890000000000001,"15min":2.1120000000000001},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":12.483000000000001,"5min":46.049999999999997,"15min":68.009},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":2.6349999999999998,"15min":2.1890000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.4660000000000002,"5min":46.012999999999998,"15min":66.975999999999999},"last":0.29199999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.7050000000000001,"5min":2.4380000000000002,"15min":1.5700000000000001},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":20.242999999999999,"5min":20.242999999999999,"15min":27.562000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":5.0439999999999996,"5min":3.2469999999999999,"15min":1.863},"min":{"1min":0.318,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":17.076000000000001,"5min":46.286999999999999,"15min":46.286999999999999},"last":0.47199999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:13:22 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.982,"15min":1.4570000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.5129999999999999,"5min":46.075000000000003,"15min":46.075000000000003},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.345,"5min":1.756,"15min":1.284},"min":{"1min":0.38700000000000001,"5min":0.28899999999999998,"15min":0.27200000000000002},"max":{"1min":11.871,"5min":20.134,"15min":20.134},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:13:22 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":2.4329999999999998,"15min":1.548},"min":{"1min":0.33400000000000002,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":8.9450000000000003,"5min":46.265000000000001,"15min":46.265000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.76300000000000001,"5min":2.2850000000000001,"15min":1.6100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.5510000000000002,"5min":46.106000000000002,"15min":46.106000000000002},"last":0.36399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:13:22 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":3.101,"15min":1.718},"min":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.311},"max":{"1min":11.766999999999999,"5min":114.14,"15min":114.14},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.5629999999999999,"5min":3.4700000000000002,"15min":1.915},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.9700000000000006,"5min":114.123,"15min":114.123},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:13:22 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":3.2240000000000002,"15min":1.9079999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34300000000000003},"max":{"1min":11.808,"5min":117.32599999999999,"15min":117.32599999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.581,"5min":3.5830000000000002,"15min":2.04},"min":{"1min":0.378,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.9920000000000009,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.55200000000000005}]}]},{"osd":3,"up_from":99,"seq":425201762426,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":544912,"kb_used_data":76040,"kb_used_omap":7,"kb_used_meta":468856,"kb_avail":93200240,"statfs":{"total":95995035648,"available":95437045760,"internally_reserved":0,"allocated":77864960,"data_stored":76936599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":480109160},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":6.2770000000000001,"5min":5.2089999999999996,"15min":3.71},"min":{"1min":0.214,"5min":0.19400000000000001,"15min":0.185},"max":{"1min":24.077999999999999,"5min":100.512,"15min":106.22},"last":0.48999999999999999},{"interface":"front","average":{"1min":7.8559999999999999,"5min":5.694,"15min":3.8660000000000001},"min":{"1min":0.22800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":26.091999999999999,"5min":101.733,"15min":106.986},"last":0.505}]},{"osd":1,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.8450000000000002,"5min":3.819,"15min":2.484},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":19.262,"5min":57.055999999999997,"15min":106.121},"last":0.23799999999999999},{"interface":"front","average":{"1min":3.8180000000000001,"5min":4.1529999999999996,"15min":2.3330000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":16.646000000000001,"5min":57.088999999999999,"15min":65.015000000000001},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":2.9289999999999998,"15min":2.367},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":10.333,"5min":23.405999999999999,"15min":106.105},"last":0.27800000000000002},{"interface":"front","average":{"1min":4.3140000000000001,"5min":3.7370000000000001,"15min":2.782},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":13.201000000000001,"5min":98.765000000000001,"15min":106.70399999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7400000000000002,"15min":2.0910000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.475,"5min":37.652000000000001,"15min":106.85899999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.4060000000000001,"5min":2.4580000000000002,"15min":1.4710000000000001},"min":{"1min":0.38700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.255000000000001,"5min":37.442,"15min":37.442},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.0329999999999999,"15min":1.381},"min":{"1min":0.36899999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.285,"5min":10.285,"15min":12.819000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.8610000000000002,"5min":2.0510000000000002,"15min":1.6319999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.968,"5min":13.228999999999999,"15min":13.228999999999999},"last":0.378}]},{"osd":6,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.9399999999999999,"15min":1.9910000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":9.9290000000000003,"5min":13.255000000000001,"15min":106.036},"last":0.41899999999999998},{"interface":"front","average":{"1min":3.3290000000000002,"5min":2.371,"15min":1.96},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":11.425000000000001,"5min":15.622,"15min":106.741},"last":0.46899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:12:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":2.3919999999999999,"15min":2.0030000000000001},"min":{"1min":0.34599999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.506,"5min":15.685,"15min":106.82599999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.9609999999999999,"5min":2.214,"15min":1.859},"min":{"1min":0.42599999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":10.212999999999999,"5min":10.212999999999999,"15min":106.084},"last":0.46100000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986442,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315212,"kb_used_data":9732,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93429940,"statfs":{"total":95995035648,"available":95672258560,"internally_reserved":0,"allocated":9965568,"data_stored":9277727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:13:21 2023","interfaces":[{"interface":"back","average":{"1min":7.944,"5min":4.4859999999999998,"15min":2.1259999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":125.538,"5min":125.538,"15min":125.538},"last":0.58099999999999996},{"interface":"front","average":{"1min":8.0399999999999991,"5min":4.4109999999999996,"15min":2.0939999999999999},"min":{"1min":0.27900000000000003,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":125.498,"5min":125.498,"15min":125.498},"last":0.64300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.6989999999999998,"15min":1.5469999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.0899999999999999,"5min":79.075999999999993,"15min":79.075999999999993},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.8450000000000002,"15min":1.5900000000000001},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":14.288,"5min":75.314999999999998,"15min":75.314999999999998},"last":0.63300000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:13:28 2023","interfaces":[{"interface":"back","average":{"1min":8.1509999999999998,"5min":2.7639999999999998,"15min":1.6830000000000001},"min":{"1min":0.34899999999999998,"5min":0.255,"15min":0.245},"max":{"1min":123.598,"5min":123.598,"15min":123.598},"last":0.67200000000000004},{"interface":"front","average":{"1min":8.1720000000000006,"5min":2.7549999999999999,"15min":1.637},"min":{"1min":0.35499999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":123.619,"5min":123.619,"15min":123.619},"last":0.66400000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:12:57 2023","interfaces":[{"interface":"back","average":{"1min":8.1989999999999998,"5min":2.8919999999999999,"15min":1.611},"min":{"1min":0.32000000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":125.90300000000001,"5min":125.90300000000001,"15min":125.90300000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":8.6199999999999992,"5min":3.1699999999999999,"15min":1.76},"min":{"1min":0.39900000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":127.90000000000001,"5min":127.90000000000001,"15min":127.90000000000001},"last":0.60799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:12:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55000000000000004,"15min":0.58099999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.3050000000000002,"5min":3.4510000000000001,"15min":8.5259999999999998},"last":0.372},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":1.956,"5min":5.9569999999999999,"15min":8.3149999999999995},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:12:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.753,"15min":0.751},"min":{"1min":0.30199999999999999,"5min":0.29399999999999998,"15min":0.26600000000000001},"max":{"1min":3.1920000000000002,"5min":5.6849999999999996,"15min":8.3989999999999991},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.76400000000000001,"15min":0.68600000000000005},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":3.21,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.496}]},{"osd":7,"last update":"Mon Oct 9 04:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.626,"15min":0.70799999999999996},"min":{"1min":0.34100000000000003,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":1.121,"5min":3.5190000000000001,"15min":9.0050000000000008},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.65900000000000003,"15min":0.77200000000000002},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.29199999999999998},"max":{"1min":1.0860000000000001,"5min":3.202,"15min":8.5440000000000005},"last":0.55800000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724806,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311868,"kb_used_data":2292,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433284,"statfs":{"total":95995035648,"available":95675682816,"internally_reserved":0,"allocated":2347008,"data_stored":1671237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":6.9539999999999997,"15min":3.2709999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":21.120999999999999,"5min":151.23599999999999,"15min":151.23599999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":3.2130000000000001,"5min":7.5700000000000003,"15min":3.5129999999999999},"min":{"1min":0.24399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.265999999999998,"5min":151.566,"15min":151.566},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:12:50 2023","interfaces":[{"interface":"back","average":{"1min":7.6600000000000001,"5min":3.6869999999999998,"15min":2.177},"min":{"1min":0.26100000000000001,"5min":0.219,"15min":0.219},"max":{"1min":58.408000000000001,"5min":59.588999999999999,"15min":91.212999999999994},"last":0.33400000000000002},{"interface":"front","average":{"1min":7.4909999999999997,"5min":3.8450000000000002,"15min":2.3740000000000001},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":58.393999999999998,"5min":59.533000000000001,"15min":91.254000000000005},"last":1.329}]},{"osd":2,"last update":"Mon Oct 9 04:13:27 2023","interfaces":[{"interface":"back","average":{"1min":4.7039999999999997,"5min":2.8359999999999999,"15min":2.1469999999999998},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":66.682000000000002,"5min":67.685000000000002,"15min":67.685000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":4.4770000000000003,"5min":3.1059999999999999,"15min":2.1469999999999998},"min":{"1min":0.27400000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":66.605000000000004,"5min":67.596000000000004,"15min":67.596000000000004},"last":0.52600000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":1.756,"15min":1.885},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":26.645,"5min":26.645,"15min":91.834999999999994},"last":0.497},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.9390000000000001,"15min":1.9279999999999999},"min":{"1min":0.35499999999999998,"5min":0.309,"15min":0.309},"max":{"1min":26.536000000000001,"5min":26.536000000000001,"15min":91.802000000000007},"last":0.42499999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.60499999999999998,"15min":0.65700000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":1.899,"5min":5.7430000000000003,"15min":5.7430000000000003},"last":0.437},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.58499999999999996,"15min":0.70499999999999996},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.1619999999999999,"5min":4.2869999999999999,"15min":6.7709999999999999},"last":0.55900000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":2.0409999999999999,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":1.407,"5min":123.239,"15min":123.239},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.0139999999999998,"15min":1.1319999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.0289999999999999,"5min":123.014,"15min":123.014},"last":0.59699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.64300000000000002,"15min":0.78200000000000003},"min":{"1min":0.311,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":3.25,"5min":5.0179999999999998,"15min":9.4930000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.60699999999999998,"15min":0.80600000000000005},"min":{"1min":0.33600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.2629999999999999,"5min":4.1449999999999996,"15min":8.5329999999999995},"last":0.56999999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:13:39.681 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:13:39.681 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:13:39.682 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:13:39.682 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.831612 2023-10-09T04:13:39.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0.8316124304242898 2023-10-09T04:13:39.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:39 smithi163 ceph-mon[213841]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:39.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:39 smithi163 ceph-mon[221067]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:39.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:39 smithi181 ceph-mon[196979]: from='client.35430 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:13:39.775 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:13:39.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:13:39.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:39.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:39.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:39.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:39.995 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:40.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:40.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:40.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:40.306 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:40.341 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:40.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:40.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:40.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:40.524 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:40.526 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:40.699 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:40.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:40.707 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:40.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:40.831 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:40.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:40 smithi163 ceph-mon[221067]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:40.879 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:40 smithi163 ceph-mon[213841]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:40.880 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:13:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:13:40] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-10-09T04:13:40.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:40.893 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:40 smithi181 ceph-mon[196979]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:40.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:40.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:41.024 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:41.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:41.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:41.199 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:13:41.200 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:13:41.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:41.251 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:41.398 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:41.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:41.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:41.502 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:41.534 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:41.626 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_2' created 2023-10-09T04:13:41.630 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:13:41.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3866439851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:13:41.787 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/176301293' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:13:41.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3866439851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:13:41.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/176301293' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:13:41.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:13:42.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3866439851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:13:42.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/176301293' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:13:42.282 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:42.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:42.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:42.412 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:42.556 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:42.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:42.660 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:42.690 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:42.736 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:42.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[213841]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[213841]: failed to encode map e241 with expected crc 2023-10-09T04:13:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3866439851' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:13:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/176301293' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:13:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[213841]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:13:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[221067]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[221067]: failed to encode map e241 with expected crc 2023-10-09T04:13:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3866439851' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:13:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/176301293' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:13:42.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:42 smithi163 ceph-mon[221067]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:13:42.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:42.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:42 smithi181 ceph-mon[196979]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:42.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:42 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:42.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:42 smithi181 ceph-mon[196979]: failed to encode map e241 with expected crc 2023-10-09T04:13:42.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3866439851' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:13:42.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/176301293' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:13:42.868 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:42 smithi181 ceph-mon[196979]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:13:42.870 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:42.913 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:42.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:42.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T04:13:43.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:43.058 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:43.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:43.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:43.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:43.381 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:43.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:43.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:43.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:43.558 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:43.561 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:13:43.582 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:13:43.611 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:13:43.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:43.640 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:13:43.663 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:13:43.665 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:43.684 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:13:43.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:13:43.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:43.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:43.727 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:43.733 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:13:43.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.736 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.736 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:13:43.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[213841]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[213841]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:13:43.798 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[213841]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[213841]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[213841]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/650768000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[221067]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[221067]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[221067]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[221067]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.799 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[221067]: failed to encode map e242 with expected crc 2023-10-09T04:13:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/650768000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:43.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:43.853 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:43 smithi181 ceph-mon[196979]: failed to encode map e242 with expected crc 2023-10-09T04:13:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:43 smithi181 ceph-mon[196979]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:13:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:43 smithi181 ceph-mon[196979]: failed to encode map e242 with expected crc 2023-10-09T04:13:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:43 smithi181 ceph-mon[196979]: failed to encode map e242 with expected crc 2023-10-09T04:13:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:43 smithi181 ceph-mon[196979]: failed to encode map e242 with expected crc 2023-10-09T04:13:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/650768000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:44.640 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.7 primary-affinity to 0.831612 (8545002) 2023-10-09T04:13:44.752 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:44.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:44.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:44.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[221067]: pgmap v642: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:44.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[221067]: failed to encode map e243 with expected crc 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[221067]: failed to encode map e243 with expected crc 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[221067]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2560055188' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8316124304242898}]: dispatch 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[213841]: pgmap v642: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[213841]: failed to encode map e243 with expected crc 2023-10-09T04:13:44.881 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[213841]: failed to encode map e243 with expected crc 2023-10-09T04:13:44.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[213841]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:13:44.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2560055188' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8316124304242898}]: dispatch 2023-10-09T04:13:44.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:44 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:44.884 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:44 smithi181 ceph-mon[196979]: pgmap v642: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:44 smithi181 ceph-mon[196979]: failed to encode map e243 with expected crc 2023-10-09T04:13:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:44 smithi181 ceph-mon[196979]: failed to encode map e243 with expected crc 2023-10-09T04:13:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:44 smithi181 ceph-mon[196979]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:13:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2560055188' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8316124304242898}]: dispatch 2023-10-09T04:13:45.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:44 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:45.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:45.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:45.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:45.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:45.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:45.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:45.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:45.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:45.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:13:45.385 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:45.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:45.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:45.535 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:45.563 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:45.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:45.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:45.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:45.810 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:45.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[221067]: failed to encode map e244 with expected crc 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[221067]: failed to encode map e244 with expected crc 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2560055188' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8316124304242898}]': finished 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[221067]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[221067]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[213841]: failed to encode map e244 with expected crc 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[213841]: failed to encode map e244 with expected crc 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2560055188' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8316124304242898}]': finished 2023-10-09T04:13:45.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[213841]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:13:45.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:45 smithi163 ceph-mon[213841]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:13:45.907 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:45.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:45.988 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:13:45.989 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:13:45.989 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:13:45.989 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:13:46.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:45 smithi181 ceph-mon[196979]: failed to encode map e244 with expected crc 2023-10-09T04:13:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:45 smithi181 ceph-mon[196979]: failed to encode map e244 with expected crc 2023-10-09T04:13:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:45 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2560055188' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8316124304242898}]': finished 2023-10-09T04:13:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:45 smithi181 ceph-mon[196979]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:13:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:45 smithi181 ceph-mon[196979]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:13:46.042 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:46.647 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:46.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:46.751 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:46.777 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:46.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:13:46.989 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:46.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[221067]: pgmap v645: 89 pgs: 2 peering, 5 unknown, 82 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:46.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[221067]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[221067]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:13:46.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[221067]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[221067]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[213841]: pgmap v645: 89 pgs: 2 peering, 5 unknown, 82 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[213841]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[213841]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[213841]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:46 smithi163 ceph-mon[213841]: failed to encode map e245 with expected crc 2023-10-09T04:13:46.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:46.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:47.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:46 smithi181 ceph-mon[196979]: pgmap v645: 89 pgs: 2 peering, 5 unknown, 82 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:46 smithi181 ceph-mon[196979]: failed to encode map e245 with expected crc 2023-10-09T04:13:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:46 smithi181 ceph-mon[196979]: failed to encode map e245 with expected crc 2023-10-09T04:13:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:46 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:13:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:46 smithi181 ceph-mon[196979]: failed to encode map e245 with expected crc 2023-10-09T04:13:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:46 smithi181 ceph-mon[196979]: failed to encode map e245 with expected crc 2023-10-09T04:13:47.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:13:47.095 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:47.127 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:47.143 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:47.175 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:47.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:47.202 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:47.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:47.310 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:47.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:47.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:47.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:47.489 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:47.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:47.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:47.747 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:47.784 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:47.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:47.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:47.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:47.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:47.972 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:48.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:48.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:48.127 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:13:48.155 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:48.351 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:13:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:48.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:13:48.356 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T04:13:48.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:48.488 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:48.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:48.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:48.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:48 smithi163 ceph-mon[213841]: pgmap v647: 89 pgs: 1 activating, 2 peering, 86 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4040055333' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:48 smithi163 ceph-mon[221067]: pgmap v647: 89 pgs: 1 activating, 2 peering, 86 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4040055333' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:48.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:48.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:48.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:48 smithi181 ceph-mon[196979]: pgmap v647: 89 pgs: 1 activating, 2 peering, 86 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:48.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4040055333' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:48.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:48.673 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:48.717 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:48.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:48.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:48.849 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:48.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:48.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:48.999 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:49.027 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:49.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:49.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:49.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:49.206 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:49.250 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:49.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:49.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:49.355 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:49.384 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:49.467 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: failed to encode map e246 with expected crc 2023-10-09T04:13:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: failed to encode map e246 with expected crc 2023-10-09T04:13:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:13:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:13:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:49.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: failed to encode map e246 with expected crc 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: failed to encode map e246 with expected crc 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:49.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:49.604 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:13:49.605 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:13:49.620 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:13:49.638 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:13:49.657 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:13:49.676 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:13:49.695 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:13:49.716 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:13:49.718 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.718 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.718 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.718 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.718 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.718 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.719 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:13:49.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: failed to encode map e246 with expected crc 2023-10-09T04:13:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: failed to encode map e246 with expected crc 2023-10-09T04:13:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:13:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:13:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:49.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:49.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:49.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:49.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:49.907 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:50.184 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T04:13:50.186 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:13:50.186 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:13:50.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:50.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:50.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T04:13:50.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:13:50.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_404719 2023-10-09T04:13:50.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:13:50.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:13:50.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:13:50.359 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:50.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:13:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:50.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:13:50.459 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:50 smithi163 ceph-mon[213841]: pgmap v649: 89 pgs: 1 activating, 2 peering, 86 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:50.460 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/316206602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:50.460 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:50 smithi163 ceph-mon[221067]: pgmap v649: 89 pgs: 1 activating, 2 peering, 86 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:50.460 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/316206602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:50.540 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:50.596 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:50.710 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:13:50.710 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:13:50.710 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T04:13:50.710 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:13:50.710 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:13:50.710 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T04:13:50.731 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:13:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:13:50] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-10-09T04:13:50.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:50.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:50.759 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:50 smithi181 ceph-mon[196979]: pgmap v649: 89 pgs: 1 activating, 2 peering, 86 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:50.759 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/316206602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:13:50.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:50.902 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:51.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:51.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:51.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:51.166 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:51.287 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 3323 3307 206.675 206.688 0.00397554 0.00482398 2023-10-09T04:13:51.298 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:51.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:51.402 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:51.429 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:51.487 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:51.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:51.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:51.632 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:51.681 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:51.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:51.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:51.844 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:51.967 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:51.971 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:51 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2023-10-09T04:13:51.971 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/398563919' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:51.971 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:51.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:52.073 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:52.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:51 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2023-10-09T04:13:52.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/398563919' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:52.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:51 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2023-10-09T04:13:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/398563919' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:13:52.137 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:52.265 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:52.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:52.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 5541 5525 172.641 138.625 0.0031825 0.0057637 2023-10-09T04:13:52.354 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:52.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:52.422 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:52.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:13:52.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:52.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:52.609 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:13:52.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:52.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: pgmap v650: 89 pgs: 89 active+clean; 25 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 57 op/s 2023-10-09T04:13:52.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:52.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:52.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:52.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: pgmap v650: 89 pgs: 89 active+clean; 25 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 57 op/s 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:13:52.726 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:13:52.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:52.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:52 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:52.733 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:52.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:52.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:53.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: pgmap v650: 89 pgs: 89 active+clean; 25 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 57 op/s 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:52 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:53.055 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:53.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 8268 8252 171.901 170.438 0.00553742 0.00581167 2023-10-09T04:13:53.767 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:13:53.767 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:13:54.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:53 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:53 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:53 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:53 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:53 smithi181 ceph-mon[196979]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[221067]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:53 smithi163 ceph-mon[213841]: failed to encode map e247 with expected crc 2023-10-09T04:13:54.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 11356 11340 177.172 193 0.00556806 0.0056364 2023-10-09T04:13:54.881 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:54.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:54.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:55.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:54 smithi181 ceph-mon[196979]: pgmap v652: 89 pgs: 89 active+clean; 119 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 239 op/s 2023-10-09T04:13:55.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:54 smithi163 ceph-mon[213841]: pgmap v652: 89 pgs: 89 active+clean; 119 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 239 op/s 2023-10-09T04:13:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:54 smithi163 ceph-mon[221067]: pgmap v652: 89 pgs: 89 active+clean; 119 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 239 op/s 2023-10-09T04:13:55.152 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:55.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 13553 13537 169.198 137.312 0.00370588 0.00590445 2023-10-09T04:13:55.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:55.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:55.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:13:55.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:55.752 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:55.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:55.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:56.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:13:56.062 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:56.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:13:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:56.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:13:56.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:13:56.286 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 16124 16108 167.775 160.688 0.0082393 0.00595306 2023-10-09T04:13:56.326 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:57.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:56 smithi181 ceph-mon[196979]: pgmap v653: 89 pgs: 89 active+clean; 483 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-10-09T04:13:57.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:13:57.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:56 smithi163 ceph-mon[213841]: pgmap v653: 89 pgs: 89 active+clean; 483 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-10-09T04:13:57.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:56 smithi163 ceph-mon[221067]: pgmap v653: 89 pgs: 89 active+clean; 483 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-10-09T04:13:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:57.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:13:57.160 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:57.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:13:57.242 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:57.287 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 15 17472 17457 155.851 84.3125 0.00671247 0.00640765 2023-10-09T04:13:57.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:13:57.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:13:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:57.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:13:57.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:13:57.554 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:57.622 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:13:57.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:57.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:13:57.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:13:57.773 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:57.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:13:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:57.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:13:57.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:57.964 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:13:57.964 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:13:57.965 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:13:58.162 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:58.294 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19554 19538 152.496 130.062 0.00319185 0.00654905 2023-10-09T04:13:58.708 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:13:58 smithi181 ceph-mon[196979]: pgmap v654: 89 pgs: 89 active+clean; 741 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:13:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:13:58 smithi163 ceph-mon[213841]: pgmap v654: 89 pgs: 89 active+clean; 741 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:13:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:13:58 smithi163 ceph-mon[221067]: pgmap v654: 89 pgs: 89 active+clean; 741 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:13:58.856 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:13:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:13:58.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:13:58.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:13:59.034 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:13:59.294 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 21740 21724 150.733 136.625 0.0040301 0.00662859 2023-10-09T04:14:00.294 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 23650 23634 147.598 119.375 0.00565789 0.00677048 2023-10-09T04:14:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:00 smithi163 ceph-mon[221067]: pgmap v655: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T04:14:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:00.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:14:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:14:00] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-10-09T04:14:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:00 smithi163 ceph-mon[213841]: pgmap v655: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T04:14:00.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:00 smithi181 ceph-mon[196979]: pgmap v655: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T04:14:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:01.294 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 25956 25940 147.281 144.125 0.00919345 0.00678458 2023-10-09T04:14:01.670 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:01.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:01.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:01.838 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:01.905 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:01.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:02.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:02.020 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:02.051 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:02.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:02.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:02.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:02.265 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:02.294 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 27912 27896 145.196 122.25 0.00535142 0.00688001 2023-10-09T04:14:02.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:02.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:02.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:02.447 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:14:02.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:02.489 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:14:02.493 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:02.524 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:14:02.559 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:02.562 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:14:02.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:02.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:02.601 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:14:02.640 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:14:02.666 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:14:02.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:02.693 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:14:02.699 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:14:02.709 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:02 smithi181 ceph-mon[196979]: pgmap v656: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T04:14:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/736418944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:02.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:02.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:02.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:02.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:02 smithi163 ceph-mon[221067]: pgmap v656: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T04:14:02.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/736418944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:02.813 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:02 smithi163 ceph-mon[213841]: pgmap v656: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2023-10-09T04:14:02.813 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/736418944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:02.885 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:02.904 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:02.904 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:14:02.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:14:02.904 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118445 6.10 6:0cc09dea:::benchmark_data_smithi163_287034_object10182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.52527208499998, 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1585794680000001, 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.905 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118445 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:14:02.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375074+0000", 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375077+0000", 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.993e-06 2023-10-09T04:14:02.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375083+0000", 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.409e-06 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375088+0000", 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6820000000000002e-06 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375142+0000", 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3881000000000001e-05 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.375152+0000", 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0315e-05 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328144+0000", 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95299157199999995 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328159+0000", 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5143999999999999e-05 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.470403+0000", 2023-10-09T04:14:02.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1422438539999999 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.471213+0000", 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00081007099999999999 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.524358+0000", 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.053145539999999998 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.524485+0000", 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012661700000000001 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.524664+0000", 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017896200000000001 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528975+0000", 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043109400000000001 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.530404+0000", 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014287880000000001 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531875+0000", 2023-10-09T04:14:02.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014716200000000001 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531932+0000", 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7028999999999999e-05 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.533654+0000", 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017214610000000001 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.914 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118476 6.10 6:09db8b21:::benchmark_data_smithi163_287034_object10213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.49721100199997, 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1736697060000001, 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.915 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118476 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403135+0000", 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403138+0000", 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0920000000000002e-06 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403145+0000", 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0759999999999997e-06 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.403150+0000", 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3800000000000002e-06 2023-10-09T04:14:02.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.404136+0000", 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00098518299999999994 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.404146+0000", 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0443e-05 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328185+0000", 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.92403893599999998 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328196+0000", 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0682999999999999e-05 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.526057+0000", 2023-10-09T04:14:02.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.197861617 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.526773+0000", 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00071598699999999996 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.538943+0000", 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012170006000000001 2023-10-09T04:14:02.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.539024+0000", 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1154000000000003e-05 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.539117+0000", 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2702999999999997e-05 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.544254+0000", 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051368059999999998 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.575431+0000", 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031177375 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.575473+0000", 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1443000000000003e-05 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.575512+0000", 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9471000000000002e-05 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576805+0000", 2023-10-09T04:14:02.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00129284 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118507 6.10 6:0ec16c38:::benchmark_data_smithi163_287034_object10244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.433804+0000", 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.466542758, 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1442500999999998, 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.925 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118507 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433804+0000", 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433804+0000", 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433803+0000", 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433806+0000", 2023-10-09T04:14:02.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6350000000000002e-06 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433812+0000", 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9490000000000001e-06 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.433817+0000", 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4620000000000004e-06 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.435949+0000", 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002131853 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.435958+0000", 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8629999999999997e-06 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328221+0000", 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.89226339099999996 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328233+0000", 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1858000000000001e-05 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.526823+0000", 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.198589455 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.527446+0000", 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062304899999999998 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.542998+0000", 2023-10-09T04:14:02.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015552237 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.543076+0000", 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8610999999999997e-05 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.543171+0000", 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4475999999999995e-05 2023-10-09T04:14:02.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552461+0000", 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0092900210000000007 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576876+0000", 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024414754 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576913+0000", 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6848999999999999e-05 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576947+0000", 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4910000000000003e-05 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578054+0000", 2023-10-09T04:14:02.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001106201 2023-10-09T04:14:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118571 6.10 6:0880eece:::benchmark_data_smithi163_287034_object10308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.494993+0000", 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.405353553, 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0842064360000001, 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118571 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.938 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494993+0000", 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494993+0000", 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494991+0000", 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.494998+0000", 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7440000000000001e-06 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495013+0000", 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4965e-05 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495021+0000", 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8159999999999994e-06 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495069+0000", 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8427999999999998e-05 2023-10-09T04:14:02.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.495077+0000", 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8250000000000005e-06 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328259+0000", 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.83318210100000001 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328270+0000", 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0679000000000001e-05 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.527503+0000", 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992328809999999 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528252+0000", 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00074927999999999995 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.547348+0000", 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019096418 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.547427+0000", 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8205000000000001e-05 2023-10-09T04:14:02.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.547521+0000", 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4179999999999996e-05 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.556222+0000", 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087014689999999999 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578120+0000", 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021897371999999998 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578158+0000", 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8798999999999999e-05 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578193+0000", 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4233999999999998e-05 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579199+0000", 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010065149999999999 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.947 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118621 6.10 6:0b7ca3a2:::benchmark_data_smithi163_287034_object10358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.684903+0000", 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.21544347100001, 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.895705559, 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118621 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684903+0000", 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684903+0000", 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684901+0000", 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684909+0000", 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5310000000000004e-06 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684921+0000", 2023-10-09T04:14:02.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2441e-05 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684929+0000", 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1200000000000002e-06 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684983+0000", 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4024999999999999e-05 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.684994+0000", 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1004e-05 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328294+0000", 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.64329977900000002 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328305+0000", 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0750999999999999e-05 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528308+0000", 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2000031630000001 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.528949+0000", 2023-10-09T04:14:02.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00064121999999999996 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552514+0000", 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023565136 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552642+0000", 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012748800000000001 2023-10-09T04:14:02.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.552829+0000", 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000187472 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.559228+0000", 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0063989119999999997 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579251+0000", 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020022604999999999 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579295+0000", 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4267999999999997e-05 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.579360+0000", 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4722999999999997e-05 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580608+0000", 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001248664 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118638 6.10 6:0fa2b777:::benchmark_data_smithi163_287034_object10375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.701481+0000", 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.19886496599997, 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8804589630000001, 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118638 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.958 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701481+0000", 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701481+0000", 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701480+0000", 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701487+0000", 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0879999999999997e-06 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701503+0000", 2023-10-09T04:14:02.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6024000000000001e-05 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701511+0000", 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.106e-06 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701592+0000", 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.072e-05 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.701602+0000", 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0186999999999999e-05 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328348+0000", 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.626745777 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328356+0000", 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1200000000000002e-06 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.529034+0000", 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.200677835 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.529558+0000", 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00052381699999999999 2023-10-09T04:14:02.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.558015+0000", 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028457805999999999 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.558099+0000", 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3707000000000003e-05 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.558198+0000", 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9327999999999995e-05 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.563233+0000", 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050347070000000002 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580680+0000", 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017446990999999999 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580738+0000", 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8322000000000002e-05 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.580798+0000", 2023-10-09T04:14:02.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9917000000000002e-05 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.581940+0000", 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011419970000000001 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118657 6.10 6:0f9ab685:::benchmark_data_smithi163_287034_object10394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.967 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.18086595699998, 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8638215659999999, 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118657 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719480+0000", 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719482+0000", 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4830000000000002e-06 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719488+0000", 2023-10-09T04:14:02.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.066e-06 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.719493+0000", 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7650000000000001e-06 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.721632+0000", 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021384300000000002 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.721641+0000", 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6739999999999992e-06 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328375+0000", 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.60673376999999995 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328382+0000", 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.153e-06 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.529613+0000", 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2012305969999999 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.530319+0000", 2023-10-09T04:14:02.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070645599999999995 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.562059+0000", 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031739648000000002 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.562132+0000", 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2917000000000007e-05 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.562242+0000", 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011035300000000001 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.567349+0000", 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005106836 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.582003+0000", 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014653875 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.582060+0000", 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6601999999999998e-05 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.582113+0000", 2023-10-09T04:14:02.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3189999999999997e-05 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583302+0000", 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011892000000000001 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118725 6.10 6:09136b91:::benchmark_data_smithi163_287034_object10462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.789819+0000", 2023-10-09T04:14:02.977 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.11052739399997, 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.794865508, 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118725 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789819+0000", 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789819+0000", 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789817+0000", 2023-10-09T04:14:02.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789825+0000", 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1850000000000004e-06 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789837+0000", 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1620999999999999e-05 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789852+0000", 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5384999999999999e-05 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789921+0000", 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8412999999999997e-05 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.789930+0000", 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1169999999999993e-06 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328400+0000", 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.53847029599999996 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328408+0000", 2023-10-09T04:14:02.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.525e-06 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.530450+0000", 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2020422900000001 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531212+0000", 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00076219199999999997 2023-10-09T04:14:02.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.566185+0000", 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034972681999999998 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.566265+0000", 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0339000000000004e-05 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.566377+0000", 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111296 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.571433+0000", 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050563200000000004 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583380+0000", 2023-10-09T04:14:02.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011947157999999999 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583434+0000", 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3634000000000002e-05 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.583464+0000", 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0400999999999999e-05 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584684+0000", 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001220308 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118727 6.10 6:0e812dc0:::benchmark_data_smithi163_287034_object10464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.795748+0000", 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.104598506, 2023-10-09T04:14:02.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7904910080000001, 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118727 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.988 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:02.989 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:02.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795748+0000", 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795748+0000", 2023-10-09T04:14:02.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795746+0000", 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795756+0000", 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9990000000000003e-06 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795768+0000", 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2173e-05 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795776+0000", 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2729999999999997e-06 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795834+0000", 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8752999999999998e-05 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.795847+0000", 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2395000000000001e-05 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328426+0000", 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.53257936699999997 2023-10-09T04:14:02.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328433+0000", 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8689999999999998e-06 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531246+0000", 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.202813353 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.531805+0000", 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055812899999999998 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.570272+0000", 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.038467012000000002 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.570363+0000", 2023-10-09T04:14:02.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0966999999999998e-05 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.570457+0000", 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4271999999999995e-05 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.576817+0000", 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0063599110000000002 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584737+0000", 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079197989999999999 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584794+0000", 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7494999999999998e-05 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.584847+0000", 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2562000000000003e-05 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586239+0000", 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013921890000000001 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:118738 6.10 6:0d6f2f89:::benchmark_data_smithi163_287034_object10475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:42.803706+0000", 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 440.09664052099998, 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7839791599999999, 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 118738 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:02.999 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803706+0000", 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803706+0000", 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.000 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803705+0000", 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803711+0000", 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7169999999999996e-06 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803722+0000", 2023-10-09T04:14:03.001 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1712000000000001e-05 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803730+0000", 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7000000000000008e-06 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803790+0000", 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9763999999999999e-05 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.002 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:42.803800+0000", 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9850000000000001e-06 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328453+0000", 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.52465375999999997 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:03.003 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:43.328460+0000", 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9609999999999999e-06 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.533713+0000", 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2052529590000001 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.534961+0000", 2023-10-09T04:14:03.004 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012478809999999999 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.574203+0000", 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039241693000000001 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.574281+0000", 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7752999999999999e-05 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.005 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.574387+0000", 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106183 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.578065+0000", 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036780369999999999 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.006 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586286+0000", 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082206889999999998 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586342+0000", 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6122999999999998e-05 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.007 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.586390+0000", 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8096999999999997e-05 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:44.587685+0000", 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012951200000000001 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.008 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80549 7.12 7:4a26220b:::benchmark_data_smithi163_346082_object80548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 204.42993099, 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5311827440000001, 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80549 2023-10-09T04:14:03.009 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.010 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470446+0000", 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0809000000000003e-05 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.011 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470453+0000", 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6939999999999995e-06 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470459+0000", 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8830000000000003e-06 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.012 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470525+0000", 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5624999999999996e-05 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470537+0000", 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1918e-05 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.968283+0000", 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4977461619999999 2023-10-09T04:14:03.013 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.972146+0000", 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038629369999999999 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.978159+0000", 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060134689999999996 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.014 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.994408+0000", 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016248729999999999 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.995092+0000", 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068414799999999998 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.015 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.995313+0000", 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000220638 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.996444+0000", 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011307999999999999 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.001598+0000", 2023-10-09T04:14:03.016 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051543429999999996 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80579 7.12 7:4e07a344:::benchmark_data_smithi163_346082_object80578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 204.380101731, 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4902949560000001, 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.017 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80579 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.018 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520243+0000", 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.019 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520271+0000", 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8158999999999999e-05 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520288+0000", 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7629999999999999e-05 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523866+0000", 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035770730000000001 2023-10-09T04:14:03.020 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524028+0000", 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016236000000000001 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524050+0000", 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1882000000000002e-05 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.021 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.982571+0000", 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.458521492 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.985387+0000", 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028160400000000001 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.022 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.989941+0000", 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045534929999999996 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.001841+0000", 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011900571 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.023 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.002750+0000", 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090909300000000003 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.003526+0000", 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077512299999999996 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.024 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.004995+0000", 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001469143 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.010540+0000", 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055447580000000003 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.025 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80582 7.12 7:4a340bf1:::benchmark_data_smithi163_346082_object80581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 204.37642822199999, 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5356164219999999, 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80582 2023-10-09T04:14:03.026 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.027 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523917+0000", 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523945+0000", 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7841999999999999e-05 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.028 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523957+0000", 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2449000000000001e-05 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523974+0000", 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7101000000000001e-05 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.029 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524082+0000", 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107395 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524090+0000", 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8189999999999998e-06 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.010809+0000", 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4867180810000002 2023-10-09T04:14:03.030 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.014231+0000", 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003422043 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.019765+0000", 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055343019999999996 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.031 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.037504+0000", 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017739296000000002 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.038374+0000", 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086940900000000002 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.032 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.052524+0000", 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014150563999999999 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.053699+0000", 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011743890000000001 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.059534+0000", 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058359709999999997 2023-10-09T04:14:03.033 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80583 7.12 7:4f777d92:::benchmark_data_smithi163_346082_object80582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 204.37603769500001, 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5426957880000001, 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.034 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80583 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.035 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524308+0000", 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524363+0000", 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5349999999999997e-05 2023-10-09T04:14:03.036 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524377+0000", 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4075e-05 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524384+0000", 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1230000000000002e-06 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.037 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524440+0000", 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5284999999999997e-05 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524453+0000", 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3614999999999999e-05 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.038 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.024073+0000", 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.499619434 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.027624+0000", 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003551102 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.032819+0000", 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051955509999999996 2023-10-09T04:14:03.039 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.039280+0000", 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064610379999999997 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.052274+0000", 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012993754 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.040 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.059828+0000", 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075536780000000003 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.061217+0000", 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001389033 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.041 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.067004+0000", 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005787665 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80604 7.12 7:4ae32c24:::benchmark_data_smithi163_346082_object80603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 203.47068514899999, 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.646247169, 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.042 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80604 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.043 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429660+0000", 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.044 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429684+0000", 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4735e-05 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429700+0000", 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6118999999999999e-05 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429710+0000", 2023-10-09T04:14:03.045 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4110000000000002e-06 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.039654+0000", 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6099443359999999 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.042687+0000", 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030327050000000001 2023-10-09T04:14:03.046 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.048100+0000", 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054131359999999998 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.067276+0000", 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019175580000000001 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.047 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.068101+0000", 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082490899999999997 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.068965+0000", 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086460499999999995 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.070415+0000", 2023-10-09T04:14:03.048 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001449547 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.075908+0000", 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054935929999999997 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.049 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80621 7.12 7:4a7d6f10:::benchmark_data_smithi163_346082_object80620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 203.44779306199999, 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6714043699999999, 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80621 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.050 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.051 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452552+0000", 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452577+0000", 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5196000000000001e-05 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452588+0000", 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1482e-05 2023-10-09T04:14:03.052 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452595+0000", 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8430000000000004e-06 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.076258+0000", 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6236622060000001 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.053 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.079917+0000", 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003659425 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.085706+0000", 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057886170000000002 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.054 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.115020+0000", 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.029313949999999998 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.115996+0000", 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097686999999999991 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.116902+0000", 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090546300000000001 2023-10-09T04:14:03.055 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.118150+0000", 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001248377 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.123958+0000", 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058073669999999999 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.056 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80659 7.12 7:4da2b418:::benchmark_data_smithi163_346082_object80658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 203.40489537900001, 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.638010832, 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.057 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80659 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.058 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495449+0000", 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495472+0000", 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2648999999999999e-05 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.059 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495487+0000", 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4714e-05 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495494+0000", 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6179999999999997e-06 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.060 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.089728+0000", 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5942332260000001 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.092886+0000", 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031585609999999998 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.098139+0000", 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052528790000000002 2023-10-09T04:14:03.061 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.124279+0000", 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.026139367 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.125465+0000", 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011861739999999999 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.062 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.126393+0000", 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092876999999999999 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.127602+0000", 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012081450000000001 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.063 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.133462+0000", 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058601410000000001 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80699 7.12 7:49a7773c:::benchmark_data_smithi163_346082_object80698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 203.35828446900001, 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.601076436, 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.064 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80699 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.065 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542061+0000", 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.066 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542082+0000", 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0676999999999999e-05 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542089+0000", 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1969999999999999e-06 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.067 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542095+0000", 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8889999999999999e-06 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.101752+0000", 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.559657235 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.105743+0000", 2023-10-09T04:14:03.068 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039902699999999998 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.111032+0000", 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052895010000000003 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.124581+0000", 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013548701999999999 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.069 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.134215+0000", 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096345360000000008 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.135040+0000", 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082435799999999997 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.070 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.136244+0000", 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00120403 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.143138+0000", 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006894608 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.071 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80726 7.12 7:4d882541:::benchmark_data_smithi163_346082_object80725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 203.32152127000001, 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6025676880000002, 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80726 2023-10-09T04:14:03.072 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.073 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578824+0000", 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578846+0000", 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2328e-05 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.074 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578858+0000", 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1749999999999999e-05 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578866+0000", 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.542e-06 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.075 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.143524+0000", 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.564657902 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.146910+0000", 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033861049999999999 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.152321+0000", 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054116600000000004 2023-10-09T04:14:03.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.172638+0000", 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020316186999999999 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.172924+0000", 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028628699999999998 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.173994+0000", 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010702839999999999 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.175304+0000", 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013095439999999999 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.181393+0000", 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060889680000000002 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80738 7.12 7:4f007b37:::benchmark_data_smithi163_346082_object80737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 203.29883938200001, 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5902670159999999, 2023-10-09T04:14:03.079 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80738 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601505+0000", 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601561+0000", 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6643e-05 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601585+0000", 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3541e-05 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601597+0000", 2023-10-09T04:14:03.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2396e-05 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.156776+0000", 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5551789820000002 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.160320+0000", 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035439960000000002 2023-10-09T04:14:03.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.165689+0000", 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053691839999999999 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.173228+0000", 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075388089999999996 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.182616+0000", 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093880849999999991 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.183537+0000", 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092063400000000004 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.185023+0000", 2023-10-09T04:14:03.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014860819999999999 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.191774+0000", 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067509279999999998 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:03.086 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:03.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:03.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:03.188 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:03.250 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:03.295 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 29654 29638 142.394 108.875 0.00854691 0.00701703 2023-10-09T04:14:04.295 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 31406 31390 140.043 109.5 0.00561424 0.0071351 2023-10-09T04:14:04.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:04 smithi163 ceph-mon[213841]: pgmap v657: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-10-09T04:14:04.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:04.836 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:04.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:04 smithi163 ceph-mon[221067]: pgmap v657: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-10-09T04:14:04.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:04.836 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:04.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:04.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:04 smithi181 ceph-mon[196979]: pgmap v657: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-10-09T04:14:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:05.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:05.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:05.194 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:05.295 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 32827 32811 136.628 88.8125 0.0072959 0.00726422 2023-10-09T04:14:05.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:14:05.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:05.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:05.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:05.965 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:06.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:06.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:06.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:06.186 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:06.252 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:06.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:06.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:06.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 34474 34458 134.523 102.938 0.012955 0.00742867 2023-10-09T04:14:06.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:06.389 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:06.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:06.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:06.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:06.590 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:06.654 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:06.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:06.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:06.666 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:06.688 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:14:06.688 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:14:06.688 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:14:06.688 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T04:14:06.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:06 smithi163 ceph-mon[213841]: pgmap v658: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T04:14:06.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:06.754 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:06.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:06 smithi163 ceph-mon[221067]: pgmap v658: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T04:14:06.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:06.755 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:06.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:06.804 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:06.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:06.874 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:06 smithi181 ceph-mon[196979]: pgmap v658: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-09T04:14:06.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:06.875 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:06.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:07.013 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:07.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:07.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:07.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:07.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:07.235 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:07.295 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 35797 35781 131.474 82.6875 0.00510184 0.00759933 2023-10-09T04:14:07.300 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:07.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:07.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:07.476 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:07.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:07.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:07.618 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:14:07.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:07.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2223637595' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:07.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:07.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:07.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:07.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:07.716 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2223637595' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:07.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:07.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:07.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:07.717 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:07.718 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:07.727 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:14:07.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:07.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:07.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:07.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2223637595' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:07.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:07.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:07.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:07.977 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:07.981 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:08.050 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:08.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:08.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:08.233 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:08.298 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 37367 37351 129.622 98.125 0.00834228 0.00770797 2023-10-09T04:14:08.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:08.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[213841]: pgmap v659: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T04:14:08.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:08.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[213841]: failed to encode map e248 with expected crc 2023-10-09T04:14:08.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:14:08.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[213841]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:14:08.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[213841]: failed to encode map e248 with expected crc 2023-10-09T04:14:08.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[221067]: pgmap v659: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T04:14:08.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:08.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[221067]: failed to encode map e248 with expected crc 2023-10-09T04:14:08.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:14:08.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[221067]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:14:08.886 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:08 smithi163 ceph-mon[221067]: failed to encode map e248 with expected crc 2023-10-09T04:14:08.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:08.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:08.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:09.017 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:08 smithi181 ceph-mon[196979]: pgmap v659: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T04:14:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:08 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:08 smithi181 ceph-mon[196979]: failed to encode map e248 with expected crc 2023-10-09T04:14:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:14:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:08 smithi181 ceph-mon[196979]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:14:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:08 smithi181 ceph-mon[196979]: failed to encode map e248 with expected crc 2023-10-09T04:14:09.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 38804 38788 127.527 89.8125 0.00496851 0.00783605 2023-10-09T04:14:09.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:09.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:09.528 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:09.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:09.606 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:09.665 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:09.775 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:09.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:09.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:09.949 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:10.199 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:10.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:10.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:14:10.296548+0000 min lat: 0.0010792 max lat: 0.284494 avg lat: 0.00794625 2023-10-09T04:14:10.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:14:10.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 40279 40263 125.76 92.1875 0.00763565 0.00794625 2023-10-09T04:14:10.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:10.318 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:10.318 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:14:10.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.318 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:40301 8.1 8:8107596a:::benchmark_data_smithi163_404719_object40300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:10.315496+0000", 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0018036969999999999, 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001854503, 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40301 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:10.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315496+0000", 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315496+0000", 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315495+0000", 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315520+0000", 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4649000000000001e-05 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315527+0000", 2023-10-09T04:14:10.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5440000000000001e-06 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315534+0000", 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0110000000000003e-06 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315909+0000", 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00037440100000000001 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316083+0000", 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000174437 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316229+0000", 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014615 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:40303 8.7 8.af966637 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:10.315543+0000", 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001757275, 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018904340000000001, 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40303 2023-10-09T04:14:10.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315543+0000", 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315543+0000", 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315542+0000", 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315564+0000", 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1367000000000001e-05 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315570+0000", 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7970000000000003e-06 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315576+0000", 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6160000000000001e-06 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.327 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:40294 8.f e248/241 8:f93cec54:::benchmark_data_smithi163_404719_object40293:head v 248'2597, mlcod=248'2595)", 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:10.315541+0000", 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001759193, 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001952968, 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315541+0000", 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315541+0000", 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315540+0000", 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:10.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315564+0000", 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4017999999999999e-05 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315572+0000", 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4359999999999996e-06 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315577+0000", 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.317293+0000", 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001716064 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.317319+0000", 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5806e-05 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:40300 8.f e248/241)", 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:10.315621+0000", 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0016791340000000001, 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001926749, 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315621+0000", 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315621+0000", 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315620+0000", 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315642+0000", 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1310000000000001e-05 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315649+0000", 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1099999999999997e-06 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:10.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.315654+0000", 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0889999999999999e-06 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:40305 8.1 8.8929c451 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:10.316095+0000", 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0012052790000000001, 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001500782, 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:10.335 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40305 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316095+0000", 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316095+0000", 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316094+0000", 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316119+0000", 2023-10-09T04:14:10.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5162000000000001e-05 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316130+0000", 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0896999999999999e-05 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316138+0000", 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7209999999999994e-06 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:10.338 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:40311 8.3 8:c8c583ec:::benchmark_data_smithi163_404719_object40310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:10.316244+0000", 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001056232, 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001409965, 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:14:10.339 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 40311 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316244+0000", 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316244+0000", 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:10.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316243+0000", 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316267+0000", 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3998999999999998e-05 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:10.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316276+0000", 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4059999999999994e-06 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316283+0000", 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1809999999999998e-06 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316384+0000", 2023-10-09T04:14:10.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101331 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316523+0000", 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013910499999999999 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:10.316652+0000", 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000128656 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:10.343 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.344 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:10.344 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:14:10.344 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 6 2023-10-09T04:14:10.344 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:10.354 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:10.430 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:10.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:10.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:10.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:10.587 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:10.611 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:10 smithi181 ceph-mon[196979]: pgmap v661: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:14:10.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:10.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:10.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:10.811 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:10.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:10.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:10.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:10.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:10 smithi163 ceph-mon[213841]: pgmap v661: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:14:10.967 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:14:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:14:10] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:14:10.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:10 smithi163 ceph-mon[221067]: pgmap v661: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-10-09T04:14:10.994 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:11.058 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:11.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 41875 41859 124.522 99.75 0.00232202 0.00800244 2023-10-09T04:14:11.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:14:11.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:11.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:11.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:11.677 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:11.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:11.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:11.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:11.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:11.858 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:11.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:11.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:11.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:12.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:12.049 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:12.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:12.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:12.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:12.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 43063 43047 122.237 74.25 0.00912841 0.00817571 2023-10-09T04:14:12.313 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:12.411 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:14:12.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:12.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:12.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:12.636 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:12.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:12.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:12 smithi163 ceph-mon[221067]: pgmap v662: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:14:12.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:12 smithi163 ceph-mon[213841]: pgmap v662: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:14:12.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:12.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:12 smithi181 ceph-mon[196979]: pgmap v662: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-09T04:14:13.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:13.130 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:13.261 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:13.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:13.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 44223 44207 120.075 72.5 0.0164313 0.00832181 2023-10-09T04:14:13.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:13.450 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:13.490 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:13.496 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:13.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:13.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:13.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:13.751 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:14.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 45310 45294 117.903 67.9375 0.00938028 0.00847677 2023-10-09T04:14:14.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:14:14.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:14:14.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:14:14.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:14:14.313 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:14:14.314 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:14:14.330 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:14:14.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:13 smithi163 conmon[213817]: 2023-10-09T04:14:13.939+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e248 definitely_dead 0 2023-10-09T04:14:14.372 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:14:14.379 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.380 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:14:14.652 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.4. 2023-10-09T04:14:14.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[213841]: pgmap v663: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T04:14:14.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1643825054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:14.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2949070498' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:14:14.665 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:14:14.666 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[213841]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:14:14.666 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[221067]: pgmap v663: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T04:14:14.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1643825054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:14.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2949070498' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:14:14.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:14:14.667 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:14 smithi163 ceph-mon[221067]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:14:15.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:14 smithi181 ceph-mon[196979]: pgmap v663: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T04:14:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1643825054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2949070498' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:14:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:14:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:14 smithi181 ceph-mon[196979]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:14:15.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 45852 45836 114.543 33.875 0.00213815 0.00850448 2023-10-09T04:14:15.323 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:15.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:15.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:15.426 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:15.459 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:15.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.960 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[221067]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.961 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: failed to encode map e249 with expected crc 2023-10-09T04:14:15.962 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:15 smithi163 ceph-mon[213841]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T04:14:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:15.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: failed to encode map e249 with expected crc 2023-10-09T04:14:16.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:15 smithi181 ceph-mon[196979]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T04:14:16.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:16.091 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:16.132 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:16.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:16.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:16.237 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:16.262 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:16.296 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 45852 45836 110.139 0 - 0.00850448 2023-10-09T04:14:16.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:16.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:16.409 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:16.435 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:16.476 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:16.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:16.583 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:16.612 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: pgmap v665: 89 pgs: 3 peering, 86 active+clean; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:14:16.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:16 smithi181 ceph-mon[196979]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T04:14:16.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:16.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:16.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:16.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:16.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: pgmap v665: 89 pgs: 3 peering, 86 active+clean; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:14:16.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.929 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.930 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:14:16.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[221067]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T04:14:16.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: pgmap v665: 89 pgs: 3 peering, 86 active+clean; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:14:16.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: failed to encode map e250 with expected crc 2023-10-09T04:14:16.932 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:14:16.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:16 smithi163 ceph-mon[213841]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T04:14:16.951 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:17.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:17.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:17.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:17.168 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:17.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:17.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:17.250 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:14:17.250 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:14:17.297 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 46071 46055 106.567 6.84375 0.00596108 0.00900602 2023-10-09T04:14:17.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:17.364 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:17.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:17.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:17.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:17.598 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:17.652 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:17.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:17.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:17.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:17.783 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:17.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:17.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:17.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:17.956 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:17 smithi181 ceph-mon[196979]: failed to encode map e251 with expected crc 2023-10-09T04:14:17.956 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:17 smithi181 ceph-mon[196979]: failed to encode map e251 with expected crc 2023-10-09T04:14:17.956 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:17 smithi181 ceph-mon[196979]: failed to encode map e251 with expected crc 2023-10-09T04:14:17.959 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:18.003 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:18.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:18.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:14:18.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:17 smithi163 ceph-mon[221067]: failed to encode map e251 with expected crc 2023-10-09T04:14:18.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:17 smithi163 ceph-mon[221067]: failed to encode map e251 with expected crc 2023-10-09T04:14:18.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:17 smithi163 ceph-mon[221067]: failed to encode map e251 with expected crc 2023-10-09T04:14:18.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:17 smithi163 ceph-mon[213841]: failed to encode map e251 with expected crc 2023-10-09T04:14:18.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:17 smithi163 ceph-mon[213841]: failed to encode map e251 with expected crc 2023-10-09T04:14:18.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:17 smithi163 ceph-mon[213841]: failed to encode map e251 with expected crc 2023-10-09T04:14:18.112 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:18.138 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:18.183 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:18.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:18.287 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:18.291 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:14:18.297 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 46071 46055 102.763 0 - 0.00900602 2023-10-09T04:14:18.313 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:18.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:18.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:18.467 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:18.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:18.494 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:18.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:18.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:18.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:18.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:18.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:18 smithi181 ceph-mon[196979]: pgmap v668: 89 pgs: 1 remapped+peering, 1 activating, 10 peering, 2 active+recovery_wait+degraded, 75 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 873 op/s; 2/133791 objects degraded (0.001%) 2023-10-09T04:14:18.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/942217442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:18.660 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:18 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 2/133791 objects degraded (0.001%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:18.672 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:18.680 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:14:18.931 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:18.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:18 smithi163 ceph-mon[221067]: pgmap v668: 89 pgs: 1 remapped+peering, 1 activating, 10 peering, 2 active+recovery_wait+degraded, 75 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 873 op/s; 2/133791 objects degraded (0.001%) 2023-10-09T04:14:18.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/942217442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:18.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:18 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 2/133791 objects degraded (0.001%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:18.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:18.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:18 smithi163 ceph-mon[213841]: pgmap v668: 89 pgs: 1 remapped+peering, 1 activating, 10 peering, 2 active+recovery_wait+degraded, 75 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 873 op/s; 2/133791 objects degraded (0.001%) 2023-10-09T04:14:18.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/942217442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:18.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:18 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 2/133791 objects degraded (0.001%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:19.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:19.062 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:19.297 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 46071 46055 99.22 0 - 0.00900602 2023-10-09T04:14:19.894 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:19.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:19.930 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:14:19.930 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[213841]: failed to encode map e252 with expected crc 2023-10-09T04:14:19.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/942217442' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:14:19.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[213841]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T04:14:19.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:19.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:19.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:19.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:14:19.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[221067]: failed to encode map e252 with expected crc 2023-10-09T04:14:19.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/942217442' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:14:19.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[221067]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T04:14:19.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:19.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:20.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:19 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:14:20.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:19 smithi181 ceph-mon[196979]: failed to encode map e252 with expected crc 2023-10-09T04:14:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/942217442' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:14:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:19 smithi181 ceph-mon[196979]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T04:14:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:20.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:20.059 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:20.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:20.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:20.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:20.243 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:20.297 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 46071 46055 95.9136 0 - 0.00900602 2023-10-09T04:14:20.396 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:20.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:20.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:20.530 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:20.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:20.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:20.644 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:14:20.666 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:14:20.686 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:14:20.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:14:20.735 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:14:20.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:20.767 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:20.777 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:14:20.802 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:14:20.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[221067]: failed to encode map e252 with expected crc 2023-10-09T04:14:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[221067]: pgmap v670: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 10 peering, 2 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 2889/135807 objects degraded (2.127%) 2023-10-09T04:14:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[221067]: failed to encode map e252 with expected crc 2023-10-09T04:14:20.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3175867302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:20.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:20.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:20.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[213841]: failed to encode map e252 with expected crc 2023-10-09T04:14:20.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[213841]: pgmap v670: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 10 peering, 2 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 2889/135807 objects degraded (2.127%) 2023-10-09T04:14:20.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[213841]: failed to encode map e252 with expected crc 2023-10-09T04:14:20.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3175867302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:20.836 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:14:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:14:20] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T04:14:20.845 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:14:20.849 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.849 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.849 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.850 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.850 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.850 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.850 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.850 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:14:20.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:21.016 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:21.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:20 smithi181 ceph-mon[196979]: failed to encode map e252 with expected crc 2023-10-09T04:14:21.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:20 smithi181 ceph-mon[196979]: pgmap v670: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 10 peering, 2 active+recovery_wait+degraded, 74 active+clean; 2.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 2889/135807 objects degraded (2.127%) 2023-10-09T04:14:21.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:20 smithi181 ceph-mon[196979]: failed to encode map e252 with expected crc 2023-10-09T04:14:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3175867302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:21.050 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:14:21.050 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:14:21.050 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T04:14:21.050 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:14:21.050 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:14:21.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T04:14:21.198 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:21.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:21.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:21.297 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 46630 46614 93.947 8.73438 0.00325246 0.010632 2023-10-09T04:14:21.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:21.358 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:21.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:21.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:21.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:21.871 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:14:21.893 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:21 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:14:21.893 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:21 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:14:21.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:21.964 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:22.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:22.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:21 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:14:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:22.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:22.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:22.217 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:22.297 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 47407 47391 92.529 48.5625 0.0322154 0.0108003 2023-10-09T04:14:22.301 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:22.363 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:22.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119029 6.11 6:89ce47bf:::benchmark_data_smithi163_287034_object10766:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.470 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 457.12918370800003, 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.554635572, 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119029 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335400+0000", 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335398+0000", 2023-10-09T04:14:22.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.335406+0000", 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7929999999999998e-06 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.363872+0000", 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028466115 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.474 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.363891+0000", 2023-10-09T04:14:22.474 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8961e-05 2023-10-09T04:14:22.474 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.474 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364339+0000", 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000447935 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364350+0000", 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0750999999999999e-05 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.475 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.862665+0000", 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.498315641 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.863639+0000", 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097311200000000004 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877198+0000", 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013559584 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877283+0000", 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5289000000000003e-05 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.877425+0000", 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000141619 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.881660+0000", 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042347840000000001 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882652+0000", 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0009924109999999999 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882695+0000", 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2449e-05 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.887923+0000", 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052283570000000003 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.887979+0000", 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5566000000000003e-05 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.890035+0000", 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002056745 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119035 6.11 6:88144e9f:::benchmark_data_smithi163_287034_object10772:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 457.10043739000002, 2023-10-09T04:14:22.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.535802179, 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119035 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:14:22.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364146+0000", 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364145+0000", 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364152+0000", 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9800000000000001e-06 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364162+0000", 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5899999999999997e-06 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364169+0000", 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3880000000000002e-06 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364483+0000", 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031385899999999999 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.484 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.364491+0000", 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.867872+0000", 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.503381112 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.871582+0000", 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037102459999999999 2023-10-09T04:14:22.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.885871+0000", 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014288876000000001 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.886009+0000", 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013767400000000001 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.886197+0000", 2023-10-09T04:14:22.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188591 2023-10-09T04:14:22.487 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.890058+0000", 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038603420000000001 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897207+0000", 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071496529999999997 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897263+0000", 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5933999999999997e-05 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897307+0000", 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3574e-05 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.897386+0000", 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.928e-05 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.899948+0000", 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002562174 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.490 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119089 6.11 6:8c811572:::benchmark_data_smithi163_287034_object10826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.93480861199998, 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.375641546, 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.491 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119089 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.492 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529775+0000", 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529777+0000", 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.531e-06 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529784+0000", 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1779999999999997e-06 2023-10-09T04:14:22.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529789+0000", 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0509999999999996e-06 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529844+0000", 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4928e-05 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529854+0000", 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9369999999999999e-06 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.881692+0000", 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3518380210000001 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.882588+0000", 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000895884 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895389+0000", 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012800420999999999 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.497 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895468+0000", 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9363000000000006e-05 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.895563+0000", 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5142000000000005e-05 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903262+0000", 2023-10-09T04:14:22.498 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076987920000000003 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903275+0000", 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2758999999999999e-05 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903318+0000", 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3312000000000001e-05 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903378+0000", 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0386000000000001e-05 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903429+0000", 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0822000000000002e-05 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.500 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.905416+0000", 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019873859999999998 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119092 6.11 6:883a032b:::benchmark_data_smithi163_287034_object10829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.93476241100001, 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4310658030000001, 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.501 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119092 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529821+0000", 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529820+0000", 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529825+0000", 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7060000000000003e-06 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529837+0000", 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1731e-05 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529844+0000", 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6240000000000002e-06 2023-10-09T04:14:22.504 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529878+0000", 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.392e-05 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.529887+0000", 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7059999999999993e-06 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.900013+0000", 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.370126255 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.902419+0000", 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024062950000000001 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.927730+0000", 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.025310295999999999 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.927860+0000", 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012995199999999999 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.928026+0000", 2023-10-09T04:14:22.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016576500000000001 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.945127+0000", 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017101775 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958766+0000", 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013638477 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958829+0000", 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3446000000000002e-05 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959082+0000", 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00025267399999999999 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959123+0000", 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0979999999999997e-05 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960887+0000", 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00176406 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.510 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119103 6.11 6:8d75c45e:::benchmark_data_smithi163_287034_object10840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.92281295700002, 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4205783109999999, 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119103 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541771+0000", 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541769+0000", 2023-10-09T04:14:22.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541776+0000", 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1169999999999998e-06 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541795+0000", 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9114999999999999e-05 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541807+0000", 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1369000000000001e-05 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541880+0000", 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3739000000000007e-05 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.541891+0000", 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1107000000000001e-05 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.902472+0000", 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.36058053 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.903197+0000", 2023-10-09T04:14:22.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072483200000000002 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933631+0000", 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.030434745999999999 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933720+0000", 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8212000000000002e-05 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.933838+0000", 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011806 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.945137+0000", 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011299208 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958878+0000", 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013741547 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958932+0000", 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3193999999999999e-05 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960951+0000", 2023-10-09T04:14:22.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020194599999999998 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.960992+0000", 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1319000000000002e-05 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962349+0000", 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013564200000000001 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.519 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119123 6.11 6:892f37da:::benchmark_data_smithi163_287034_object10860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.90476279299997, 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.403942767, 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.520 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119123 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559821+0000", 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559819+0000", 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559825+0000", 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2280000000000003e-06 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559842+0000", 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6141999999999999e-05 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559850+0000", 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9079999999999995e-06 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.523 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559902+0000", 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2420999999999998e-05 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.559917+0000", 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5003e-05 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.905461+0000", 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.345543943 2023-10-09T04:14:22.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.906296+0000", 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083475499999999996 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.938986+0000", 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032690364 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.939133+0000", 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014743800000000001 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.939247+0000", 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113915 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958704+0000", 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019456505999999998 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958968+0000", 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026427900000000002 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959024+0000", 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5674000000000003e-05 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962395+0000", 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003370736 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.962431+0000", 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6906000000000002e-05 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963764+0000", 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00133204 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119141 6.11 6:8c64562a:::benchmark_data_smithi163_287034_object10878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.88948417400002, 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.396054788, 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.529 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119141 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.530 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575099+0000", 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575102+0000", 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9490000000000001e-06 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575108+0000", 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1140000000000002e-06 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575113+0000", 2023-10-09T04:14:22.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2360000000000004e-06 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575184+0000", 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0494000000000006e-05 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.575195+0000", 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1283e-05 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.908246+0000", 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.333050761 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.909908+0000", 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001662455 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949276+0000", 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039368001 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949574+0000", 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029827500000000003 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.949695+0000", 2023-10-09T04:14:22.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120931 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.958714+0000", 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0090190619999999996 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963822+0000", 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051071249999999997 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963923+0000", 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101548 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.969034+0000", 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051108899999999999 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.969082+0000", 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8161999999999997e-05 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971154+0000", 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020719359999999999 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.538 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119147 6.11 6:8f3284c4:::benchmark_data_smithi163_287034_object10884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.87924263899998, 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3874358629999999, 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119147 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.539 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585341+0000", 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.540 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585339+0000", 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585345+0000", 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.02e-06 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585357+0000", 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1371e-05 2023-10-09T04:14:22.541 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585362+0000", 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1259999999999997e-06 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585402+0000", 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0420999999999998e-05 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.542 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.585413+0000", 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0242e-05 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.911820+0000", 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.326407552 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.543 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.912798+0000", 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097778699999999993 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.954865+0000", 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.042066631 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.954986+0000", 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121303 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.544 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.955142+0000", 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015607500000000001 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.959037+0000", 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003894837 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.545 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963876+0000", 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048392319999999997 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.963970+0000", 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4040000000000001e-05 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.546 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971217+0000", 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072466520000000001 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.971271+0000", 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4704999999999999e-05 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.972777+0000", 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001505337 2023-10-09T04:14:22.547 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119163 6.11 6:8b083f99:::benchmark_data_smithi163_287034_object10900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.86307329700003, 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3956605739999999, 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.548 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119163 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601510+0000", 2023-10-09T04:14:22.549 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601509+0000", 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601516+0000", 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3549999999999999e-06 2023-10-09T04:14:22.550 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601528+0000", 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1885000000000001e-05 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601537+0000", 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9060000000000008e-06 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.551 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601581+0000", 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4168000000000001e-05 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.601589+0000", 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3240000000000003e-06 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.552 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.964003+0000", 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.362413941 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.965225+0000", 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012218419999999999 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.553 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977192+0000", 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011966976000000001 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977278+0000", 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5663999999999998e-05 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.554 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.977394+0000", 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011626 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.982063+0000", 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046684430000000004 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.555 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995092+0000", 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013029099000000001 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995125+0000", 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3803999999999997e-05 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.556 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995155+0000", 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9683999999999999e-05 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.995201+0000", 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6093000000000001e-05 2023-10-09T04:14:22.557 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997171+0000", 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019696169999999999 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.558 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119188 6.11 6:8a56da9b:::benchmark_data_smithi163_287034_object10925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.84070409700001, 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3781419879999999, 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.559 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119188 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623879+0000", 2023-10-09T04:14:22.560 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623878+0000", 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623884+0000", 2023-10-09T04:14:22.561 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623900+0000", 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6373000000000001e-05 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.562 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623908+0000", 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1419999999999993e-06 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623965+0000", 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6709000000000002e-05 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.623977+0000", 2023-10-09T04:14:22.563 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2267e-05 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.965277+0000", 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3412998730000001 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968073+0000", 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027958570000000001 2023-10-09T04:14:22.564 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986363+0000", 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018290041 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986467+0000", 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010372199999999999 2023-10-09T04:14:22.565 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.986609+0000", 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014165400000000001 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.566 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997219+0000", 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010610519000000001 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997253+0000", 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3803999999999997e-05 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.567 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997288+0000", 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5064999999999997e-05 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000032+0000", 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027444190000000001 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.568 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000092+0000", 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9963000000000002e-05 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002021+0000", 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001929107 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.569 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119196 6.11 6:8fdab617:::benchmark_data_smithi163_287034_object10933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.83320033000001, 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3723983319999999, 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.570 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119196 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.571 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631383+0000", 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631382+0000", 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.572 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631387+0000", 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5260000000000004e-06 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631396+0000", 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0110000000000006e-06 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.573 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631402+0000", 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631439+0000", 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6520999999999998e-05 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.574 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.631446+0000", 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4059999999999999e-06 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968116+0000", 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.33666961 2023-10-09T04:14:22.575 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.968985+0000", 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086886199999999998 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993266+0000", 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024281193999999999 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.576 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993347+0000", 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1148000000000003e-05 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.993458+0000", 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111158 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.577 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.000020+0000", 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065613119999999997 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002036+0000", 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020161950000000001 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.578 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002091+0000", 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5470999999999998e-05 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002144+0000", 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2332e-05 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.579 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.002203+0000", 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9389999999999999e-05 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.003782+0000", 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015784429999999999 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.580 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119206 6.11 6:8f80b5b4:::benchmark_data_smithi163_287034_object10943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.82405366699999, 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.397280665, 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.581 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119206 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.582 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640530+0000", 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640528+0000", 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.583 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640535+0000", 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0119999999999999e-06 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640550+0000", 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4673e-05 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640559+0000", 2023-10-09T04:14:22.584 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.935e-06 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640615+0000", 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6740999999999998e-05 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.640627+0000", 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.189e-05 2023-10-09T04:14:22.585 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.997311+0000", 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3566840010000001 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.998135+0000", 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082370499999999999 2023-10-09T04:14:22.586 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012025+0000", 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013889755 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012139+0000", 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011416099999999999 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.587 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.012292+0000", 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015364299999999999 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026026+0000", 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013733176999999999 2023-10-09T04:14:22.588 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026062+0000", 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6235999999999997e-05 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032222+0000", 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0061599669999999997 2023-10-09T04:14:22.589 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.036596+0000", 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043742829999999996 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.036661+0000", 2023-10-09T04:14:22.590 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5092000000000006e-05 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037811+0000", 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001149279 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.591 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119231 6.11 6:8f1bfbf5:::benchmark_data_smithi163_287034_object10968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.79350250300001, 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3681440409999999, 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119231 2023-10-09T04:14:22.592 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671081+0000", 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.593 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671079+0000", 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671087+0000", 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5410000000000003e-06 2023-10-09T04:14:22.594 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671103+0000", 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6531000000000001e-05 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671113+0000", 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7170000000000003e-06 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.595 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671174+0000", 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1267999999999997e-05 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.671184+0000", 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9890000000000004e-06 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.596 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.998170+0000", 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.326985353 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999028+0000", 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000858163 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.597 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018202+0000", 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019174388000000001 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018330+0000", 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127269 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.598 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.018487+0000", 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015788 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032196+0000", 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013708549 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.599 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032241+0000", 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5118999999999999e-05 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037827+0000", 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055862890000000004 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.600 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037934+0000", 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106397 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037980+0000", 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6433000000000001e-05 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.039225+0000", 2023-10-09T04:14:22.601 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001244818 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119251 6.11 6:8bf159d8:::benchmark_data_smithi163_287034_object10988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.766522893, 2023-10-09T04:14:22.602 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3497973839999999, 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119251 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.603 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698061+0000", 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.604 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698059+0000", 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698065+0000", 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.908e-06 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.605 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698078+0000", 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.296e-05 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698086+0000", 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7570000000000004e-06 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.606 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698139+0000", 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3701e-05 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.698150+0000", 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0548e-05 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.607 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999079+0000", 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.30092889 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:46.999966+0000", 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088695600000000005 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.608 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024113+0000", 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.02414738 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024224+0000", 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011063799999999999 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.609 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.024372+0000", 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000148123 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.032262+0000", 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078901960000000004 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.610 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.037837+0000", 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055749579999999996 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.039272+0000", 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001435012 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.611 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.046029+0000", 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067568610000000003 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.046059+0000", 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9821000000000001e-05 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.612 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.047858+0000", 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00179923 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119258 6.11 6:89e8583e:::benchmark_data_smithi163_287034_object10995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.756591861, 2023-10-09T04:14:22.613 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3578773550000001, 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119258 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.614 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707992+0000", 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.615 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707990+0000", 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.707997+0000", 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8560000000000001e-06 2023-10-09T04:14:22.616 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708014+0000", 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6804999999999999e-05 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708021+0000", 2023-10-09T04:14:22.617 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0180000000000004e-06 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708074+0000", 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2914999999999997e-05 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.618 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.708088+0000", 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3818999999999999e-05 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026085+0000", 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3179974219999999 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026910+0000", 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082452299999999997 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050702+0000", 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023792316000000001 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050818+0000", 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116173 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.050970+0000", 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015151300000000001 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060167+0000", 2023-10-09T04:14:22.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091971970000000007 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060243+0000", 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6215000000000001e-05 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060374+0000", 2023-10-09T04:14:22.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000130894 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.063837+0000", 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003463224 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.063891+0000", 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4011999999999997e-05 2023-10-09T04:14:22.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.065869+0000", 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001977768 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.624 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:119295 6.11 6:8feb1e91:::benchmark_data_smithi163_287034_object11032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 456.638890301, 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2504799099999999, 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 119295 2023-10-09T04:14:22.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825693+0000", 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825692+0000", 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825700+0000", 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1559999999999995e-06 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825719+0000", 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9177999999999998e-05 2023-10-09T04:14:22.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825727+0000", 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4630000000000001e-06 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825788+0000", 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0730999999999998e-05 2023-10-09T04:14:22.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:45.825803+0000", 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4347e-05 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.026947+0000", 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2011442640000001 2023-10-09T04:14:22.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.027704+0000", 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00075679299999999996 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.056255+0000", 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028550981 2023-10-09T04:14:22.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.057403+0000", 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011480500000000001 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,5", 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.057552+0000", 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014943500000000001 2023-10-09T04:14:22.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.060380+0000", 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002827867 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.068125+0000", 2023-10-09T04:14:22.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0077448899999999999 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.069348+0000", 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001223285 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.074303+0000", 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049545429999999996 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.074377+0000", 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4524999999999995e-05 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:47.076173+0000", 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017959460000000001 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80474 7.2 7:413fef99:::benchmark_data_smithi163_346082_object80473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.449723+0000", 2023-10-09T04:14:22.636 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 225.014860642, 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97738634599999996, 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80474 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.637 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449723+0000", 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449723+0000", 2023-10-09T04:14:22.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449721+0000", 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449750+0000", 2023-10-09T04:14:22.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8515999999999999e-05 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449768+0000", 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7963000000000001e-05 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449773+0000", 2023-10-09T04:14:22.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4280000000000004e-06 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.449833+0000", 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9419999999999997e-05 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450001+0000", 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016866599999999999 2023-10-09T04:14:22.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450183+0000", 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018147700000000001 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.460339+0000", 2023-10-09T04:14:22.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010156505 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.460355+0000", 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5838e-05 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.410485+0000", 2023-10-09T04:14:22.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.950129377 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.411250+0000", 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000765616 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.427003+0000", 2023-10-09T04:14:22.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01575269 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.427109+0000", 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010626800000000001 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.645 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80490 7.2 7:445fadfb:::benchmark_data_smithi163_346082_object80489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.450738+0000", 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 225.01384552600001, 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97800665600000003, 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:22.646 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80490 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450738+0000", 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450738+0000", 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450737+0000", 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450794+0000", 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7027000000000001e-05 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450802+0000", 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0530000000000001e-06 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.450807+0000", 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9710000000000001e-06 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.451719+0000", 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00091187600000000005 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.451838+0000", 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000118956 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.451989+0000", 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015018099999999999 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472667+0000", 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020678228 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472677+0000", 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0037999999999999e-05 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.412196+0000", 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.93951964399999999 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.412822+0000", 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062551600000000001 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428634+0000", 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01581221 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428745+0000", 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110512 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.654 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80507 7.2 7:421785fa:::benchmark_data_smithi163_346082_object80506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.466106+0000", 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 224.998477286, 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96350955699999996, 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.655 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80507 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466106+0000", 2023-10-09T04:14:22.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466106+0000", 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466105+0000", 2023-10-09T04:14:22.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466135+0000", 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0056000000000002e-05 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466144+0000", 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9320000000000002e-06 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.466150+0000", 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2009999999999996e-06 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472699+0000", 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065491109999999998 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.472704+0000", 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5940000000000004e-06 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.412881+0000", 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94017724199999997 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.413037+0000", 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000156253 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.413175+0000", 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013784299999999999 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.417403+0000", 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042277529999999999 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.420553+0000", 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031505410000000002 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428790+0000", 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082364510000000005 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.428843+0000", 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2750999999999997e-05 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.429616+0000", 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077321300000000003 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.664 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80519 7.2 7:42c84ec1:::benchmark_data_smithi163_346082_object80518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:37.483278+0000", 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 224.981305988, 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94722626799999998, 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:22.665 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80519 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483278+0000", 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483278+0000", 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483276+0000", 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:22.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483297+0000", 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0234e-05 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483308+0000", 2023-10-09T04:14:22.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.165e-05 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483314+0000", 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0290000000000002e-06 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483363+0000", 2023-10-09T04:14:22.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9240999999999998e-05 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:37.483374+0000", 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0234000000000001e-05 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:22.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.414927+0000", 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.93155327300000002 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.415090+0000", 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016282499999999999 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:14:22.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.415295+0000", 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020525900000000001 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.418308+0000", 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003013253 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.420597+0000", 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022882129999999999 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.429667+0000", 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.009070013 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:22.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.429718+0000", 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1289000000000003e-05 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.430504+0000", 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078599099999999999 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.674 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:22.675 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:22.675 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:22.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:22.708 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:22.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:22.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:22 smithi163 ceph-mon[221067]: pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 32 MiB/s wr, 514 op/s; 11572/138243 objects degraded (8.371%); 2341/138243 objects misplaced (1.693%); 5.5 MiB/s, 1 keys/s, 90 objects/s recovering 2023-10-09T04:14:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:22 smithi163 ceph-mon[213841]: pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 32 MiB/s wr, 514 op/s; 11572/138243 objects degraded (8.371%); 2341/138243 objects misplaced (1.693%); 5.5 MiB/s, 1 keys/s, 90 objects/s recovering 2023-10-09T04:14:22.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:22.916 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:22.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:22.993 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:22 smithi181 ceph-mon[196979]: pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 32 MiB/s wr, 514 op/s; 11572/138243 objects degraded (8.371%); 2341/138243 objects misplaced (1.693%); 5.5 MiB/s, 1 keys/s, 90 objects/s recovering 2023-10-09T04:14:22.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:22.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:23.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:23.174 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:23.299 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 48383 48367 91.5693 61 0.0120471 0.0109137 2023-10-09T04:14:23.916 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:14:24.006 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:24.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:24.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:24.156 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:24.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:24.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:24.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:24.299 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 49198 49182 90.3744 50.9375 0.0134113 0.0110476 2023-10-09T04:14:24.388 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:24.414 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:24.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:24.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:24.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:24.619 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:24.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:24.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:24.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:14:24.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:24.831 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:24.866 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:24.889 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:24.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:24.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:25.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:24 smithi181 ceph-mon[196979]: pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 24 MiB/s wr, 390 op/s; 11572/138243 objects degraded (8.371%); 2341/138243 objects misplaced (1.693%); 4.2 MiB/s, 1 keys/s, 68 objects/s recovering 2023-10-09T04:14:25.029 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:25.081 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:24 smithi163 ceph-mon[221067]: pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 24 MiB/s wr, 390 op/s; 11572/138243 objects degraded (8.371%); 2341/138243 objects misplaced (1.693%); 4.2 MiB/s, 1 keys/s, 68 objects/s recovering 2023-10-09T04:14:25.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:24 smithi163 ceph-mon[213841]: pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 24 MiB/s wr, 390 op/s; 11572/138243 objects degraded (8.371%); 2341/138243 objects misplaced (1.693%); 4.2 MiB/s, 1 keys/s, 68 objects/s recovering 2023-10-09T04:14:25.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:25.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:25.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:25.227 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:25.292 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:25.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:25.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 50055 50039 89.3157 53.5625 0.0109311 0.0111884 2023-10-09T04:14:25.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:25.480 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:25.711 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:25.800 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:25.800 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:14:25.800 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:25.812 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:25.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:25.853 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:25.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:26.001 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:26.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:25 smithi163 ceph-mon[213841]: pgmap v673: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 20 MiB/s wr, 312 op/s; 11572/142146 objects degraded (8.141%); 1950/142146 objects misplaced (1.372%); 6.4 MiB/s, 1 keys/s, 104 objects/s recovering 2023-10-09T04:14:26.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:25 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 11572/142146 objects degraded (8.141%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:26.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:25 smithi163 ceph-mon[221067]: pgmap v673: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 20 MiB/s wr, 312 op/s; 11572/142146 objects degraded (8.141%); 1950/142146 objects misplaced (1.372%); 6.4 MiB/s, 1 keys/s, 104 objects/s recovering 2023-10-09T04:14:26.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:25 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 11572/142146 objects degraded (8.141%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:25 smithi181 ceph-mon[196979]: pgmap v673: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 20 MiB/s wr, 312 op/s; 11572/142146 objects degraded (8.141%); 1950/142146 objects misplaced (1.372%); 6.4 MiB/s, 1 keys/s, 104 objects/s recovering 2023-10-09T04:14:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:25 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 11572/142146 objects degraded (8.141%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:26.301 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 50962 50946 88.4095 56.6875 0.033776 0.0113032 2023-10-09T04:14:26.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:26.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:26.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:27.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:27.063 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:27.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:27.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:27.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:27.235 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:27.261 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:27.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 51683 51667 87.2383 45.0625 0.00394683 0.0114369 2023-10-09T04:14:27.309 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:27.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:27.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:27.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:27.456 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:27.525 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:27.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:27.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:27.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:27.721 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:28.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:28.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:28.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:28.211 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:28.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:28.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:28.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:28.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 52588 52572 86.4311 56.5625 0.0121508 0.011564 2023-10-09T04:14:28.391 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:28.428 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:28.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:28.483 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:28 smithi181 ceph-mon[196979]: pgmap v674: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s; 8665/149412 objects degraded (5.799%); 4659/149412 objects misplaced (3.118%); 7.0 MiB/s, 0 keys/s, 112 objects/s recovering 2023-10-09T04:14:28.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:28.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:28.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:28.624 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:28.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:28 smithi163 ceph-mon[213841]: pgmap v674: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s; 8665/149412 objects degraded (5.799%); 4659/149412 objects misplaced (3.118%); 7.0 MiB/s, 0 keys/s, 112 objects/s recovering 2023-10-09T04:14:28.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:28 smithi163 ceph-mon[221067]: pgmap v674: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 33 MiB/s wr, 520 op/s; 8665/149412 objects degraded (5.799%); 4659/149412 objects misplaced (3.118%); 7.0 MiB/s, 0 keys/s, 112 objects/s recovering 2023-10-09T04:14:28.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:28.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:28.789 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:28.870 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:29.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 53401 53385 85.5179 50.8125 0.0112402 0.0116886 2023-10-09T04:14:29.914 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:29.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:30.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:30.121 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:30.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:30.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:30.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:30.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:14:30.303044+0000 min lat: 0.0010792 max lat: 6.1654 avg lat: 0.0118171 2023-10-09T04:14:30.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:14:30.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 54165 54149 84.5734 47.75 0.016494 0.0118171 2023-10-09T04:14:30.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:30.402 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:30.468 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:30.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:30.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:30.694 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:30.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:30 smithi181 ceph-mon[196979]: pgmap v675: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 8665/153261 objects degraded (5.654%); 4659/153261 objects misplaced (3.040%); 6.4 MiB/s, 0 keys/s, 103 objects/s recovering 2023-10-09T04:14:30.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:30 smithi163 ceph-mon[221067]: pgmap v675: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 8665/153261 objects degraded (5.654%); 4659/153261 objects misplaced (3.040%); 6.4 MiB/s, 0 keys/s, 103 objects/s recovering 2023-10-09T04:14:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:30.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:30 smithi163 ceph-mon[213841]: pgmap v675: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 8665/153261 objects degraded (5.654%); 4659/153261 objects misplaced (3.040%); 6.4 MiB/s, 0 keys/s, 103 objects/s recovering 2023-10-09T04:14:30.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:30.835 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:14:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:14:30] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T04:14:30.884 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:30.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:30.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:30.920 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:14:30.966 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:14:30.975 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:14:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.975 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:54699 8.1 e252/241 8:842610a2:::benchmark_data_smithi163_404719_object54698:head v 252'3491, mlcod=252'3489)", 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:30.963623+0000", 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.010493771000000001, 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010524286000000001, 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.963623+0000", 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.963623+0000", 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.963622+0000", 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.963644+0000", 2023-10-09T04:14:30.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.266e-05 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.963660+0000", 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5503999999999999e-05 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.963667+0000", 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0450000000000003e-06 2023-10-09T04:14:30.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.964994+0000", 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001327429 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965013+0000", 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9113000000000001e-05 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:30.979 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:54701 8.f 8:f4850390:::benchmark_data_smithi163_404719_object54700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:30.965611+0000", 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0085051069999999996, 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086335119999999994, 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:14:30.980 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 54701 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965611+0000", 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965611+0000", 2023-10-09T04:14:30.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965609+0000", 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965637+0000", 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7291e-05 2023-10-09T04:14:30.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965655+0000", 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8168999999999998e-05 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965663+0000", 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1289999999999996e-06 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965781+0000", 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011762 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.965950+0000", 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000169295 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966112+0000", 2023-10-09T04:14:30.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016212900000000001 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.968649+0000", 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025371389999999999 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974119+0000", 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054698949999999998 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.985 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:54700 8.b e252/250 8:d731ea05:::benchmark_data_smithi163_404719_object54699:head v 252'3444, mlcod=0'0)", 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:30.966318+0000", 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0077980590000000004, 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080377299999999999, 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:30.986 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966318+0000", 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966318+0000", 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:30.987 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966317+0000", 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966354+0000", 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7004e-05 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:30.988 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966366+0000", 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1268e-05 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966373+0000", 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4159999999999998e-06 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966452+0000", 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8650000000000001e-05 2023-10-09T04:14:30.989 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.966479+0000", 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7169e-05 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:54705 8.1 e252/241 8:886d2175:::benchmark_data_smithi163_404719_object54704:head v 252'3492, mlcod=252'3489)", 2023-10-09T04:14:30.990 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:30.970081+0000", 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.004035467, 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043184599999999997, 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970081+0000", 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.991 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970081+0000", 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970080+0000", 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.992 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970105+0000", 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4978e-05 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970114+0000", 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0319999999999992e-06 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970120+0000", 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6740000000000002e-06 2023-10-09T04:14:30.993 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970157+0000", 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7323999999999997e-05 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.970177+0000", 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9423999999999999e-05 2023-10-09T04:14:30.994 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:54711 8.f 8.b05b02df (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e252)", 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:30.974321+0000", 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 4294967295.9997954, 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013991700000000001, 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:30.995 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 54711 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974321+0000", 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974321+0000", 2023-10-09T04:14:30.996 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974320+0000", 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974370+0000", 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0819000000000003e-05 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.997 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974383+0000", 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.292e-05 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974391+0000", 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0579999999999998e-06 2023-10-09T04:14:30.998 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:30.974443+0000", 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1471999999999998e-05 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:14:30.999 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 5 2023-10-09T04:14:31.000 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:31.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:31.015 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:14:31.059 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:14:31.082 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:31.098 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:14:31.146 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:14:31.184 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:14:31.244 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:14:31.276 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:14:31.302 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 54994 54978 83.7744 51.8125 0.0221574 0.0119309 2023-10-09T04:14:31.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:31.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:31.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:31.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:31.503 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:31.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4170914715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:14:31.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3526665175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:31.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4170914715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:14:31.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3526665175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:31.621 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:14:32.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4170914715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:14:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3526665175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:32.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 55769 55753 82.9332 48.4375 0.012307 0.0120525 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[221067]: pgmap v676: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 8665/161973 objects degraded (5.350%); 4132/161973 objects misplaced (2.551%); 8.5 MiB/s, 0 keys/s, 137 objects/s recovering 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 8665/161973 objects degraded (5.350%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4170914715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[221067]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[221067]: failed to encode map e253 with expected crc 2023-10-09T04:14:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[213841]: pgmap v676: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 8665/161973 objects degraded (5.350%); 4132/161973 objects misplaced (2.551%); 8.5 MiB/s, 0 keys/s, 137 objects/s recovering 2023-10-09T04:14:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 8665/161973 objects degraded (5.350%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4170914715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:14:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[213841]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T04:14:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:32 smithi163 ceph-mon[213841]: failed to encode map e253 with expected crc 2023-10-09T04:14:33.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:32 smithi181 ceph-mon[196979]: pgmap v676: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 8665/161973 objects degraded (5.350%); 4132/161973 objects misplaced (2.551%); 8.5 MiB/s, 0 keys/s, 137 objects/s recovering 2023-10-09T04:14:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 8665/161973 objects degraded (5.350%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:32 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:32 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4170914715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:14:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:32 smithi181 ceph-mon[196979]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T04:14:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:32 smithi181 ceph-mon[196979]: failed to encode map e253 with expected crc 2023-10-09T04:14:33.060 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:14:33.060 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:14:33.060 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T04:14:33.060 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:14:33.060 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:14:33.060 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T04:14:33.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 56663 56647 82.304 55.875 0.0127332 0.0121435 2023-10-09T04:14:33.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:33.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:33.685 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:33.711 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:33.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:33.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:33.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:33.893 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:33.940 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:33.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:33.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:34.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:34.145 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:34.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:14:34.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:34.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:34.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:34.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 57413 57397 81.499 46.875 0.0195185 0.0122605 2023-10-09T04:14:34.397 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:34.433 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:34.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:34.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:34.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:34.503 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:34.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:34.710 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:34.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:34 smithi181 ceph-mon[196979]: pgmap v678: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 8665/161973 objects degraded (5.350%); 4132/161973 objects misplaced (2.551%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:14:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:34.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:34.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:34 smithi163 ceph-mon[221067]: pgmap v678: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 8665/161973 objects degraded (5.350%); 4132/161973 objects misplaced (2.551%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:14:34.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:34.872 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:34.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:34 smithi163 ceph-mon[213841]: pgmap v678: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 8665/161973 objects degraded (5.350%); 4132/161973 objects misplaced (2.551%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:14:34.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:14:34.874 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:34.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:34.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:35.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:35.081 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:35.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:35.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:35.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 58313 58297 80.9379 56.25 0.0258982 0.0123493 2023-10-09T04:14:35.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:35.434 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:35.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:35.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:35.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:35.655 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:35.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:35.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:35.834 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:35.871 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:35.932 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:35.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:35.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:36.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:36.069 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:36.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:36.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:36.163 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:36.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:14:36.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:36.277 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:36.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:14:36.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 59310 59294 80.533 62.3125 0.026977 0.0124102 2023-10-09T04:14:36.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:36.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:36.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:36.470 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:36.526 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:36.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:36.632 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:36.748 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:36 smithi163 ceph-mon[221067]: pgmap v679: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 8665/167271 objects degraded (5.180%); 3768/167271 objects misplaced (2.253%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:14:36.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:36 smithi163 ceph-mon[213841]: pgmap v679: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 8665/167271 objects degraded (5.180%); 3768/167271 objects misplaced (2.253%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:14:36.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:36.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:37.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:36 smithi181 ceph-mon[196979]: pgmap v679: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 8665/167271 objects degraded (5.180%); 3768/167271 objects misplaced (2.253%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:14:37.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:37.045 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:37.045 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T04:14:37.045 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:37.123 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:37.277 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:37.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:37.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 60061 60045 79.8183 46.9375 0.0101576 0.0125206 2023-10-09T04:14:37.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:37.465 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:37.709 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:37.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:37.719 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 8665/167271 objects degraded (5.180%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:37.720 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 8665/167271 objects degraded (5.180%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:37.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:37.853 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:37.917 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:37.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:37.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 8665/167271 objects degraded (5.180%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:38.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:38.093 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:38.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 60966 60950 79.3338 56.5625 0.00608301 0.0125986 2023-10-09T04:14:38.868 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:38.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:38.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:38.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:38 smithi163 ceph-mon[221067]: pgmap v680: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 835 op/s; 8665/174471 objects degraded (4.966%); 3547/174471 objects misplaced (2.033%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T04:14:38.906 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:38 smithi163 ceph-mon[213841]: pgmap v680: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 835 op/s; 8665/174471 objects degraded (4.966%); 3547/174471 objects misplaced (2.033%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T04:14:38.963 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:39.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:39.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:38 smithi181 ceph-mon[196979]: pgmap v680: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 52 MiB/s wr, 835 op/s; 8665/174471 objects degraded (4.966%); 3547/174471 objects misplaced (2.033%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T04:14:39.036 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:39.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:39.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:39.201 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:39.243 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:39.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 61885 61869 78.8869 57.4375 0.0205463 0.0126713 2023-10-09T04:14:39.308 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:39.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:39.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:39.449 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:39.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:39.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:39.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:39.652 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:39.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:39.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:39.732 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:14:39.733 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:14:39.827 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:39.920 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:40.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 62787 62771 78.4367 56.375 0.0180121 0.012744 2023-10-09T04:14:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:40 smithi181 ceph-mon[196979]: pgmap v681: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s; 8665/178545 objects degraded (4.853%); 3547/178545 objects misplaced (1.987%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T04:14:41.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:40 smithi163 ceph-mon[213841]: pgmap v681: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s; 8665/178545 objects degraded (4.853%); 3547/178545 objects misplaced (1.987%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T04:14:41.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:40 smithi163 ceph-mon[221067]: pgmap v681: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s; 8665/178545 objects degraded (4.853%); 3547/178545 objects misplaced (1.987%); 6.9 MiB/s, 111 objects/s recovering 2023-10-09T04:14:41.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:14:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:14:40] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:14:41.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 63592 63576 77.8853 50.3125 0.0156319 0.0128332 2023-10-09T04:14:42.303 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 64402 64386 77.3611 50.625 0.0126633 0.0129212 2023-10-09T04:14:42.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:42.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:42.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:42.492 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:14:42.571 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:42.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:42.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:42 smithi163 ceph-mon[221067]: pgmap v682: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 849 op/s; 8665/187467 objects degraded (4.622%); 2940/187467 objects misplaced (1.568%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T04:14:42.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 8665/187467 objects degraded (4.622%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:42.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:42.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:42 smithi163 ceph-mon[213841]: pgmap v682: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 849 op/s; 8665/187467 objects degraded (4.622%); 2940/187467 objects misplaced (1.568%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T04:14:42.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 8665/187467 objects degraded (4.622%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:42.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:42.987 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:43.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:42 smithi181 ceph-mon[196979]: pgmap v682: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 53 MiB/s wr, 849 op/s; 8665/187467 objects degraded (4.622%); 2940/187467 objects misplaced (1.568%); 7.4 MiB/s, 119 objects/s recovering 2023-10-09T04:14:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 8665/187467 objects degraded (4.622%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:43.100 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:43.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:43.204 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:43.235 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:43.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:43.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:43.309 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 65177 65161 76.8066 48.4375 0.0188035 0.0130148 2023-10-09T04:14:43.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:43.406 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:14:43.421 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:43.441 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:43.441 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:14:43.470 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:43.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:43.480 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:14:43.524 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:14:43.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:43.577 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:14:43.609 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:43.613 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:14:43.653 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:14:43.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:43.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:43.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:43.689 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:14:43.701 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:14:43.715 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.716 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:14:43.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:43.785 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:14:43.785 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:14:43.785 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:14:43.785 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:14:43.785 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:14:43.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:14:43.858 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:43.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/216931066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:43.905 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/50540231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:43.906 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:43 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:43.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/216931066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:43.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/50540231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:43.906 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:43 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/216931066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/50540231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:44.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:43 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:14:44.032 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:44.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:44.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:44.230 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:44.310 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 65975 65959 76.308 49.875 0.0285641 0.0130984 2023-10-09T04:14:44.323 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:14:44.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:44.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:44.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:44.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:44.520 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:44.589 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:44.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:44.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:44.698 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:44.729 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:44.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: pgmap v683: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 8665/187467 objects degraded (4.622%); 2940/187467 objects misplaced (1.568%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:14:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: failed to encode map e254 with expected crc 2023-10-09T04:14:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:14:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T04:14:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: failed to encode map e254 with expected crc 2023-10-09T04:14:44.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:44 smithi181 ceph-mon[196979]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: pgmap v683: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 8665/187467 objects degraded (4.622%); 2940/187467 objects misplaced (1.568%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:14:45.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:45.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[213841]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: pgmap v683: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 8665/187467 objects degraded (4.622%); 2940/187467 objects misplaced (1.568%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T04:14:45.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:44 smithi163 ceph-mon[221067]: failed to encode map e254 with expected crc 2023-10-09T04:14:45.312 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 66623 66607 75.6538 40.5 0.0148528 0.0132124 2023-10-09T04:14:45.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:45.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:45.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:45.765 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:46.293 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:46.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:46.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:46.312 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 67498 67482 75.2794 54.6875 0.0241224 0.013278 2023-10-09T04:14:46.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:46.501 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:46.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:46.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:46.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:46.805 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:47.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:47.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:46 smithi181 ceph-mon[196979]: pgmap v685: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 8665/192873 objects degraded (4.493%); 2535/192873 objects misplaced (1.314%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:47.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:46 smithi163 ceph-mon[213841]: pgmap v685: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 8665/192873 objects degraded (4.493%); 2535/192873 objects misplaced (1.314%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:47.028 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:46 smithi163 ceph-mon[221067]: pgmap v685: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 8665/192873 objects degraded (4.493%); 2535/192873 objects misplaced (1.314%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:47.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181790 6.12 6:483ee06b:::benchmark_data_smithi163_287034_object73527:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:14:47.094 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 397.877362844, 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2147281759999999, 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181790 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206412+0000", 2023-10-09T04:14:47.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206414+0000", 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.855e-06 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206419+0000", 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7840000000000003e-06 2023-10-09T04:14:47.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206423+0000", 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5449999999999997e-06 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207972+0000", 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001549031 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.098 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.207978+0000", 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9889999999999997e-06 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388116+0000", 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.180137601 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for rw locks", 2023-10-09T04:14:47.099 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.388262+0000", 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145642 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408226+0000", 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019964064 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408317+0000", 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1717000000000003e-05 2023-10-09T04:14:47.100 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.408461+0000", 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014404700000000001 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418786+0000", 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010325051 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.101 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420556+0000", 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017692529999999999 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420629+0000", 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2847000000000002e-05 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.102 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.420693+0000", 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4324000000000003e-05 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.421141+0000", 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044771999999999998 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.103 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181893 6.12 6:48514e0f:::benchmark_data_smithi163_287034_object73630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 397.87727523199999, 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2117616529999999, 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181893 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.104 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.105 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206500+0000", 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206502+0000", 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0099999999999998e-06 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206513+0000", 2023-10-09T04:14:47.106 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206517+0000", 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5499999999999996e-06 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208079+0000", 2023-10-09T04:14:47.107 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015617529999999999 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208087+0000", 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5059999999999998e-06 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395090+0000", 2023-10-09T04:14:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1870032690000001 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395194+0000", 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010448 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.395313+0000", 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011805 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400612+0000", 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052990609999999999 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407098+0000", 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064860760000000003 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417540+0000", 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010442168999999999 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.417604+0000", 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4577000000000002e-05 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418262+0000", 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00065729299999999998 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181924 6.12 6:4956f571:::benchmark_data_smithi163_287034_object73661:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 397.87723409400002, 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.212897227, 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181924 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206541+0000", 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206543+0000", 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0640000000000001e-06 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206549+0000", 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6949999999999997e-06 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206554+0000", 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208118+0000", 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015642570000000001 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208123+0000", 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6370000000000004e-06 2023-10-09T04:14:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397380+0000", 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1892568610000001 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397556+0000", 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017565199999999999 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.397712+0000", 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015616 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400663+0000", 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002950664 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407160+0000", 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064973219999999998 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418821+0000", 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011661227 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.418878+0000", 2023-10-09T04:14:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6666999999999998e-05 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419438+0000", 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056037000000000003 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181928 6.12 6:4a3131f6:::benchmark_data_smithi163_287034_object73665:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 397.87720336400002, 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2134125929999999, 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181928 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206572+0000", 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206574+0000", 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0389999999999999e-06 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206579+0000", 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.206585+0000", 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6330000000000001e-06 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208144+0000", 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015595439999999999 2023-10-09T04:14:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:09.208151+0000", 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400687+0000", 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.192535828 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.400844+0000", 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015750000000000001 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.401023+0000", 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017885000000000001 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407172+0000", 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006149374 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.407223+0000", 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0392999999999997e-05 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419479+0000", 2023-10-09T04:14:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01225652 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419533+0000", 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3735e-05 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:10.419985+0000", 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045140000000000002 2023-10-09T04:14:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181971 6.13 6:ce89c71c:::benchmark_data_smithi163_287034_object73708:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86749658100001, 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.344004741, 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181971 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216279+0000", 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216277+0000", 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216286+0000", 2023-10-09T04:14:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7050000000000005e-06 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216564+0000", 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027831900000000002 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216577+0000", 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.29e-05 2023-10-09T04:14:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217701+0000", 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001124002 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217710+0000", 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6999999999999997e-06 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.538610+0000", 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3209002599999999 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.538722+0000", 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112537 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.539153+0000", 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000430362 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542629+0000", 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034758459999999999 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.545992+0000", 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033635409999999998 2023-10-09T04:14:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552225+0000", 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062325189999999997 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.559513+0000", 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072881919999999998 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.559580+0000", 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7296000000000001e-05 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560283+0000", 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070328800000000005 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181975 6.13 6:caf04be6:::benchmark_data_smithi163_287034_object73712:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86685020200002, 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3577086709999999, 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181975 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216925+0000", 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216924+0000", 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216930+0000", 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8719999999999999e-06 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216938+0000", 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8380000000000008e-06 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216944+0000", 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4310000000000002e-06 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217731+0000", 2023-10-09T04:14:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078680899999999997 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217736+0000", 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7459999999999997e-06 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542662+0000", 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3249255600000001 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.542848+0000", 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000186174 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.543402+0000", 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000553489 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552236+0000", 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0088346299999999996 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552280+0000", 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3537e-05 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560349+0000", 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080691140000000005 2023-10-09T04:14:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.573767+0000", 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013417861 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.573826+0000", 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9379999999999997e-05 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574634+0000", 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080766300000000002 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:181980 6.13 6:cdaa0fdf:::benchmark_data_smithi163_287034_object73717:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86682645399998, 2023-10-09T04:14:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3655362260000001, 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181980 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:14:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216949+0000", 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216948+0000", 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216951+0000", 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6579999999999998e-06 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216958+0000", 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5350000000000001e-06 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216963+0000", 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3650000000000003e-06 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217756+0000", 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00079339899999999997 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217762+0000", 2023-10-09T04:14:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6180000000000003e-06 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546040+0000", 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.328277887 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546236+0000", 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019608900000000001 2023-10-09T04:14:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.546794+0000", 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055758299999999995 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552243+0000", 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054494230000000001 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552318+0000", 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5481000000000005e-05 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566793+0000", 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014474614 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581590+0000", 2023-10-09T04:14:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014796659 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581645+0000", 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5427999999999999e-05 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582485+0000", 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083988900000000002 2023-10-09T04:14:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182015 6.13 6:cae1eea5:::benchmark_data_smithi163_287034_object73752:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86680793300002, 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3664138910000001, 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182015 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216967+0000", 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216970+0000", 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5299999999999999e-06 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216975+0000", 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3689999999999998e-06 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216979+0000", 2023-10-09T04:14:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5589999999999999e-06 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217782+0000", 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080258200000000003 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217788+0000", 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.502e-06 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.548975+0000", 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3311876169999999 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.549135+0000", 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015948399999999999 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.549566+0000", 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043101400000000002 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552286+0000", 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027206219999999998 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552369+0000", 2023-10-09T04:14:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3139000000000005e-05 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566902+0000", 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014533001 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582622+0000", 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015720073000000001 2023-10-09T04:14:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582685+0000", 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2966999999999995e-05 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583381+0000", 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000695814 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182032 6.13 6:cff9d57a:::benchmark_data_smithi163_287034_object73769:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86679211400002, 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3672618480000001, 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182032 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216983+0000", 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216985+0000", 2023-10-09T04:14:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1789999999999998e-06 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216990+0000", 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3109999999999996e-06 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216995+0000", 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9280000000000001e-06 2023-10-09T04:14:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217808+0000", 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00081240700000000002 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217813+0000", 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4009999999999996e-06 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552391+0000", 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3345780999999999 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.552538+0000", 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014700800000000001 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.553135+0000", 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059664400000000002 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563747+0000", 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010612430000000001 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566840+0000", 2023-10-09T04:14:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030923819999999999 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574748+0000", 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079079549999999995 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583498+0000", 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087501799999999998 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583544+0000", 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6165999999999998e-05 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584245+0000", 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00070112899999999999 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182044 6.13 6:c8672a95:::benchmark_data_smithi163_287034_object73781:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86677627199998, 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.367953709, 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182044 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:14:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.216999+0000", 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217001+0000", 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9290000000000001e-06 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217007+0000", 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1700000000000002e-06 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217011+0000", 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3850000000000004e-06 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217834+0000", 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082231700000000003 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217839+0000", 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556028+0000", 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.338188543 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556217+0000", 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189452 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.556804+0000", 2023-10-09T04:14:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058719899999999995 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563759+0000", 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069549829999999997 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566968+0000", 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003208598 2023-10-09T04:14:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582584+0000", 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015615655000000001 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584293+0000", 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017097309999999999 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584348+0000", 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4750999999999998e-05 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584953+0000", 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060460600000000005 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182130 6.13 6:ce90ea3b:::benchmark_data_smithi163_287034_object73867:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86676026399999, 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.376683477, 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182130 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:14:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217015+0000", 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217017+0000", 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.464e-06 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217021+0000", 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6940000000000001e-06 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217024+0000", 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2760000000000001e-06 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217860+0000", 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083570200000000002 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217865+0000", 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5400000000000003e-06 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560446+0000", 2023-10-09T04:14:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.342580544 2023-10-09T04:14:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.560636+0000", 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019010500000000001 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.561233+0000", 2023-10-09T04:14:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059681600000000001 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.566854+0000", 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056207490000000004 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567010+0000", 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015621499999999999 2023-10-09T04:14:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583462+0000", 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016452205000000001 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.592834+0000", 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093721150000000003 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.592892+0000", 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7810999999999999e-05 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593699+0000", 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080657999999999999 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182187 6.13 6:c86ec581:::benchmark_data_smithi163_287034_object73924:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86674886700001, 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3896110020000001, 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182187 2023-10-09T04:14:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217026+0000", 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217028+0000", 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.906e-06 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217031+0000", 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.095e-06 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217034+0000", 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6529999999999998e-06 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217885+0000", 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00085116200000000003 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217891+0000", 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4920000000000001e-06 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563798+0000", 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3459071169999999 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.563972+0000", 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017474299999999999 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.564543+0000", 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00057071800000000001 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567022+0000", 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024785779999999999 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567079+0000", 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7258e-05 2023-10-09T04:14:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597561+0000", 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.030482183999999999 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.605598+0000", 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080370560000000008 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.605665+0000", 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7268999999999996e-05 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.606637+0000", 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097197400000000002 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182188 6.13 6:cef9c490:::benchmark_data_smithi163_287034_object73925:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86673826800001, 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4008471199999999, 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182188 2023-10-09T04:14:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217037+0000", 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217039+0000", 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8160000000000001e-06 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217044+0000", 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.169e-06 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217048+0000", 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1980000000000003e-06 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217910+0000", 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086211000000000002 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217915+0000", 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2850000000000002e-06 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567119+0000", 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3492036760000001 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567314+0000", 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000195353 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.567922+0000", 2023-10-09T04:14:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060742399999999996 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574701+0000", 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067789490000000003 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.577940+0000", 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032392279999999998 2023-10-09T04:14:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601664+0000", 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023723782999999998 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.616941+0000", 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015276853 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617000+0000", 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9185999999999999e-05 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617884+0000", 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088436199999999997 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182235 6.13 6:cbfff063:::benchmark_data_smithi163_287034_object73972:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86672410800003, 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4014693499999999, 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182235 2023-10-09T04:14:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217051+0000", 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217053+0000", 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3099999999999999e-06 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217059+0000", 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3759999999999999e-06 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217063+0000", 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.318e-06 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217936+0000", 2023-10-09T04:14:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00087284899999999998 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217941+0000", 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.643e-06 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570245+0000", 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3523035450000001 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570415+0000", 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016968699999999999 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.570978+0000", 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056366499999999998 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.577951+0000", 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069725350000000002 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.581550+0000", 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035993399999999999 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601696+0000", 2023-10-09T04:14:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020146045000000001 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617929+0000", 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016232449999999999 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.617984+0000", 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5387999999999999e-05 2023-10-09T04:14:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618521+0000", 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053653999999999996 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182259 6.13 6:ca05850d:::benchmark_data_smithi163_287034_object73996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:14:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86670851899999, 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4019898450000001, 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182259 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217067+0000", 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217066+0000", 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217069+0000", 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.227e-06 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217074+0000", 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6829999999999996e-06 2023-10-09T04:14:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217079+0000", 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2880000000000003e-06 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217961+0000", 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088278599999999999 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217967+0000", 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574785+0000", 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3568176510000001 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.574950+0000", 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000164869 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.575563+0000", 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061337800000000004 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.582540+0000", 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069775280000000002 2023-10-09T04:14:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593712+0000", 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011171096 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.606689+0000", 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012977087 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618561+0000", 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011872297 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.618604+0000", 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3266999999999998e-05 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619057+0000", 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00045241999999999998 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.228 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182286 6.13 6:c82cebba:::benchmark_data_smithi163_287034_object74023:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.866692941, 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4024858179999999, 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182286 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:14:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217082+0000", 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217089+0000", 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1029999999999996e-06 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217096+0000", 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4010000000000002e-06 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217101+0000", 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9690000000000002e-06 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217987+0000", 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088635799999999996 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217993+0000", 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8619999999999999e-06 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.578004+0000", 2023-10-09T04:14:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.360011112 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.578616+0000", 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061207500000000001 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.579171+0000", 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00055508900000000004 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.583427+0000", 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.004255549 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593719+0000", 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010292066000000001 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.610148+0000", 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016429684 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619093+0000", 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0089444580000000006 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619134+0000", 2023-10-09T04:14:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0794000000000003e-05 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619568+0000", 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043460500000000002 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182384 6.13 6:cc63ec28:::benchmark_data_smithi163_287034_object74121:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86667122699998, 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4029806759999999, 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182384 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217104+0000", 2023-10-09T04:14:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217106+0000", 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1689999999999999e-06 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217113+0000", 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9739999999999996e-06 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217117+0000", 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4499999999999997e-06 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218013+0000", 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00089540900000000001 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218019+0000", 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.584986+0000", 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3669674590000001 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.585163+0000", 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000176995 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.585767+0000", 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00060421699999999999 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593726+0000", 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079584719999999994 2023-10-09T04:14:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597514+0000", 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037886840000000001 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.614382+0000", 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016867727999999999 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619605+0000", 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005223073 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.619647+0000", 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2032e-05 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.620085+0000", 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00043757399999999998 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182402 6.13 6:cc13d46a:::benchmark_data_smithi163_287034_object74139:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86665440500002, 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.407380751, 2023-10-09T04:14:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182402 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217121+0000", 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217123+0000", 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0609999999999998e-06 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217128+0000", 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5969999999999999e-06 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217133+0000", 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.34e-06 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218039+0000", 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090607999999999997 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218044+0000", 2023-10-09T04:14:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.589640+0000", 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3715955420000001 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.589766+0000", 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126234 2023-10-09T04:14:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.590263+0000", 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000496529 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597468+0000", 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0072057340000000001 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601593+0000", 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041247039999999999 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.620123+0000", 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018529468 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.623617+0000", 2023-10-09T04:14:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034940879999999998 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.623677+0000", 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0062000000000002e-05 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624502+0000", 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000824952 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182403 6.13 6:ccb27798:::benchmark_data_smithi163_287034_object74140:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.866637739, 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.407780327, 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182403 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217138+0000", 2023-10-09T04:14:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217137+0000", 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217139+0000", 2023-10-09T04:14:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2689999999999998e-06 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217145+0000", 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7980000000000002e-06 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217150+0000", 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4229999999999998e-06 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218064+0000", 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00091458200000000005 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218070+0000", 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4979999999999997e-06 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593753+0000", 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.375683465 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.593914+0000", 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016051499999999999 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.594483+0000", 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00056889400000000002 2023-10-09T04:14:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597523+0000", 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030404329999999999 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601631+0000", 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041084379999999998 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624529+0000", 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.022897438999999999 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624548+0000", 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8601999999999999e-05 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624576+0000", 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8637999999999999e-05 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624918+0000", 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00034171100000000002 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:182407 6.13 6:cd9f07b1:::benchmark_data_smithi163_287034_object74144:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 395.86662182600003, 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4088424019999999, 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 182407 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217153+0000", 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217155+0000", 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2699999999999999e-06 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217161+0000", 2023-10-09T04:14:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6840000000000001e-06 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.217166+0000", 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6439999999999999e-06 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218093+0000", 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092731699999999999 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:11.218100+0000", 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7399999999999998e-06 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.597985+0000", 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.37988534 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.598147+0000", 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000162189 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.598729+0000", 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00058185599999999995 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.601789+0000", 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003059712 2023-10-09T04:14:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.603630+0000", 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001841051 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.624943+0000", 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021312827999999999 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625489+0000", 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00054580099999999995 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625535+0000", 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6610000000000003e-05 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:14:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:08:12.625996+0000", 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000460715 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:47.273 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:47.279 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:47.309 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:14:47.310 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:14:47.310 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:14:47.312 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 68371 68355 74.916 54.5625 0.0152771 0.0133424 2023-10-09T04:14:47.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:47.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:47.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:47.527 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:47.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:47.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:47.659 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 8665/192873 objects degraded (4.493%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:47.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:47.788 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:47.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:47.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:47.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:47.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 8665/192873 objects degraded (4.493%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:47.964 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:47.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 8665/192873 objects degraded (4.493%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:48.059 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:48.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:48.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:48.238 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:48.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:14:48.312 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 69144 69128 74.4574 48.3125 0.0113692 0.0134238 2023-10-09T04:14:48.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:14:48.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:48.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:14:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:48.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:48.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:48.679 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:48.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:48.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:48.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:48 smithi163 ceph-mon[213841]: pgmap v686: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 8665/199470 objects degraded (4.344%); 2309/199470 objects misplaced (1.158%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:48.858 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:48 smithi163 ceph-mon[221067]: pgmap v686: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 8665/199470 objects degraded (4.344%); 2309/199470 objects misplaced (1.158%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:48.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:48.987 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:48 smithi181 ceph-mon[196979]: pgmap v686: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 8665/199470 objects degraded (4.344%); 2309/199470 objects misplaced (1.158%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:48.990 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:49.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:49.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:49.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:49.189 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:49.211 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:49.212 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:49.247 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:49.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:49.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 69843 69827 73.936 43.6875 0.024919 0.0135197 2023-10-09T04:14:49.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:49.401 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:49.475 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:49.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:49.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:49.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:49.737 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:49.737 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:49.756 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:49.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:49.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:49.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:50.005 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:50.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:50.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:50.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:50.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:50.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:50.282 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:50.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:14:50.313431+0000 min lat: 0.0010792 max lat: 6.1654 avg lat: 0.0135962 2023-10-09T04:14:50.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:14:50.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 70631 70615 73.5246 49.25 0.0303288 0.0135962 2023-10-09T04:14:50.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:50.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:50.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:50.505 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:50.568 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:50.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:50.640 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:14:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:14:50] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:14:50.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:50.712 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:50.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:50.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:50.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:50.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:14:50.933 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:50 smithi163 ceph-mon[213841]: pgmap v687: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 8665/203139 objects degraded (4.266%); 2309/203139 objects misplaced (1.137%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:50.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:50 smithi163 ceph-mon[221067]: pgmap v687: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 8665/203139 objects degraded (4.266%); 2309/203139 objects misplaced (1.137%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:50.944 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:51.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:50 smithi181 ceph-mon[196979]: pgmap v687: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 8665/203139 objects degraded (4.266%); 2309/203139 objects misplaced (1.137%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:14:51.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 71240 71224 72.9434 38.0625 0.00299304 0.0136059 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[213841]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[213841]: failed to encode map e255 with expected crc 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[213841]: failed to encode map e255 with expected crc 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[213841]: failed to encode map e255 with expected crc 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[221067]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[221067]: failed to encode map e255 with expected crc 2023-10-09T04:14:51.981 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[221067]: failed to encode map e255 with expected crc 2023-10-09T04:14:51.982 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:51 smithi163 ceph-mon[221067]: failed to encode map e255 with expected crc 2023-10-09T04:14:52.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:51 smithi181 ceph-mon[196979]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T04:14:52.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:51 smithi181 ceph-mon[196979]: failed to encode map e255 with expected crc 2023-10-09T04:14:52.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:51 smithi181 ceph-mon[196979]: failed to encode map e255 with expected crc 2023-10-09T04:14:52.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:51 smithi181 ceph-mon[196979]: failed to encode map e255 with expected crc 2023-10-09T04:14:52.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 71586 71570 72.1159 21.625 0.0225776 0.0138608 2023-10-09T04:14:52.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:14:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:52.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:14:52.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:52.647 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:52.711 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:52.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[213841]: pgmap v689: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 997 op/s; 5779/211425 objects degraded (2.733%); 4503/211425 objects misplaced (2.130%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[213841]: failed to encode map e256 with expected crc 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[213841]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[213841]: failed to encode map e256 with expected crc 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/211425 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[221067]: pgmap v689: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 997 op/s; 5779/211425 objects degraded (2.733%); 4503/211425 objects misplaced (2.130%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:14:52.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[221067]: failed to encode map e256 with expected crc 2023-10-09T04:14:52.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[221067]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T04:14:52.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[221067]: failed to encode map e256 with expected crc 2023-10-09T04:14:52.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/211425 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:52.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:52.860 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:52.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:52.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:52 smithi181 ceph-mon[196979]: pgmap v689: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 997 op/s; 5779/211425 objects degraded (2.733%); 4503/211425 objects misplaced (2.130%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:14:52.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:52 smithi181 ceph-mon[196979]: failed to encode map e256 with expected crc 2023-10-09T04:14:52.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:52 smithi181 ceph-mon[196979]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T04:14:52.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:52 smithi181 ceph-mon[196979]: failed to encode map e256 with expected crc 2023-10-09T04:14:52.905 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/211425 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:52.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:53.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:53.033 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:53.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:53.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:53.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:53.285 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:14:53.286 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:14:53.286 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:14:53.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:14:53.287 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:53.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 72291 72275 71.6707 44.0625 0.0257238 0.0139473 2023-10-09T04:14:53.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:53.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:53.857 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:53.891 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:53.942 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:53.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:54.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:53 smithi181 ceph-mon[196979]: failed to encode map e256 with expected crc 2023-10-09T04:14:54.051 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:14:54.078 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:54.088 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:53 smithi163 ceph-mon[221067]: failed to encode map e256 with expected crc 2023-10-09T04:14:54.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:53 smithi163 ceph-mon[213841]: failed to encode map e256 with expected crc 2023-10-09T04:14:54.137 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:14:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:54.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:14:54.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:54.275 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:54.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 73000 72984 71.2433 44.3125 0.0196892 0.0140313 2023-10-09T04:14:54.350 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:54.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:54.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:54.529 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:54.565 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:73218 8.c e256/241 8:3bf26793:::benchmark_data_smithi163_404719_object73217:head v 256'4529, mlcod=256'4526)", 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:54.670457+0000", 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0055704680000000003, 2023-10-09T04:14:54.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056077890000000002, 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670457+0000", 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670457+0000", 2023-10-09T04:14:54.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670457+0000", 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670476+0000", 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8610999999999999e-05 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670481+0000", 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.604e-06 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670486+0000", 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6199999999999998e-06 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:54.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.673461+0000", 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029746899999999999 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.673487+0000", 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6891999999999999e-05 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.694 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:73207 8.2 e256/250 8:4f768744:::benchmark_data_smithi163_404719_object73206:head v 256'4667, mlcod=247'1611)", 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:54.657309+0000", 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.018718764999999998, 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018827232999999999, 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.657309+0000", 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.657309+0000", 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.657309+0000", 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:54.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.657342+0000", 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3772999999999997e-05 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.657350+0000", 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3270000000000003e-06 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.657356+0000", 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.003e-06 2023-10-09T04:14:54.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.673757+0000", 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016401763999999999 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.673783+0000", 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5527000000000001e-05 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:54.698 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:73209 8.c e256/241 8:3b0ff620:::benchmark_data_smithi163_404719_object73208:head v 256'4528, mlcod=256'4526)", 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:54.670348+0000", 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0056792099999999996, 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058510699999999999, 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:54.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670348+0000", 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670348+0000", 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670348+0000", 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:54.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670380+0000", 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2394999999999997e-05 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670398+0000", 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8017e-05 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670406+0000", 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2330000000000006e-06 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670646+0000", 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023988500000000001 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670671+0000", 2023-10-09T04:14:54.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4803e-05 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:73208 8.2 e256/250)", 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:14:54.670419+0000", 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0056090569999999998, 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058373239999999996, 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:14:54.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670419+0000", 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670419+0000", 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670418+0000", 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670442+0000", 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3875000000000001e-05 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670448+0000", 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1500000000000004e-06 2023-10-09T04:14:54.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:14:54.670453+0000", 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5040000000000004e-06 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 4 2023-10-09T04:14:54.706 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:14:54.707 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:54.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:54.810 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:14:54.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:54.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:54 smithi163 ceph-mon[221067]: pgmap v691: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 5779/211425 objects degraded (2.733%); 4503/211425 objects misplaced (2.130%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:14:54.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:54 smithi163 ceph-mon[213841]: pgmap v691: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 5779/211425 objects degraded (2.733%); 4503/211425 objects misplaced (2.130%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:14:54.844 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:54.907 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:14:54.907 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:14:54.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:54.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:54.951 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:14:54.981 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:14:55.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:55.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:54 smithi181 ceph-mon[196979]: pgmap v691: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 5779/211425 objects degraded (2.733%); 4503/211425 objects misplaced (2.130%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:14:55.063 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:14:55.097 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:14:55.141 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:55.161 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:14:55.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:14:55.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 73670 73654 70.7916 41.875 0.0126894 0.0141214 2023-10-09T04:14:55.850 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:14:56.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2521808574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:56.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2520305647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:56.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:55 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:56.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:55 smithi181 ceph-mon[196979]: pgmap v692: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 5779/215430 objects degraded (2.683%); 4485/215430 objects misplaced (2.082%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:14:56.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2521808574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:56.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2520305647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:56.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:56.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[213841]: pgmap v692: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 5779/215430 objects degraded (2.683%); 4485/215430 objects misplaced (2.082%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:14:56.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2521808574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:14:56.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2520305647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:56.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:14:56.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:55 smithi163 ceph-mon[221067]: pgmap v692: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 5779/215430 objects degraded (2.683%); 4485/215430 objects misplaced (2.082%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:14:56.316 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 74418 74402 70.4274 46.75 0.0143131 0.0141924 2023-10-09T04:14:56.769 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:56.769 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[213841]: failed to encode map e257 with expected crc 2023-10-09T04:14:56.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:14:56.770 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[213841]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T04:14:56.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:56.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:56.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:57.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:14:57.023 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:56 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:57.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:56 smithi181 ceph-mon[196979]: failed to encode map e257 with expected crc 2023-10-09T04:14:57.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:56 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:14:57.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:56 smithi181 ceph-mon[196979]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T04:14:57.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:56 smithi181 ceph-mon[196979]: failed to encode map e257 with expected crc 2023-10-09T04:14:57.026 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:57.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:14:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:57.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:14:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:14:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[221067]: failed to encode map e257 with expected crc 2023-10-09T04:14:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:14:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[221067]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T04:14:57.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[221067]: failed to encode map e257 with expected crc 2023-10-09T04:14:57.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:56 smithi163 ceph-mon[213841]: failed to encode map e257 with expected crc 2023-10-09T04:14:57.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:57.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:57.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:57.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:57.313 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 75121 75105 70.032 43.9375 0.00313982 0.0142698 2023-10-09T04:14:57.372 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:57.400 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:57.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:57.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:57.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:14:58.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:14:58.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:57 smithi163 ceph-mon[213841]: failed to encode map e257 with expected crc 2023-10-09T04:14:58.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/215430 objects degraded (2.683%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:58.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:14:57 smithi163 ceph-mon[213841]: pgmap v694: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 591 op/s; 5779/220635 objects degraded (2.619%); 3833/220635 objects misplaced (1.737%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T04:14:58.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:57 smithi163 ceph-mon[221067]: failed to encode map e257 with expected crc 2023-10-09T04:14:58.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/215430 objects degraded (2.683%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:58.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:14:57 smithi163 ceph-mon[221067]: pgmap v694: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 591 op/s; 5779/220635 objects degraded (2.619%); 3833/220635 objects misplaced (1.737%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T04:14:58.088 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:57 smithi181 ceph-mon[196979]: failed to encode map e257 with expected crc 2023-10-09T04:14:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/215430 objects degraded (2.683%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:14:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:14:57 smithi181 ceph-mon[196979]: pgmap v694: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 591 op/s; 5779/220635 objects degraded (2.619%); 3833/220635 objects misplaced (1.737%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T04:14:58.314 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 75849 75833 69.6705 45.5 0.0224705 0.0143486 2023-10-09T04:14:58.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:14:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:58.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:14:58.803 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:14:58.833 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:58.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:14:58.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:58.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:14:58.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:14:59.026 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:59.092 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:59.275 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:14:59.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:59.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:14:59.314 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 76635 76619 69.3728 49.125 0.0131998 0.0144098 2023-10-09T04:14:59.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:14:59.425 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:59.477 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:14:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:59.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:14:59.582 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:14:59.735 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:14:59.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:14:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:14:59.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:14:59.973 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:14:59.978 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:15:00.002 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:00.039 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:00.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:00.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:00.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:00.263 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:00.316 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 77416 77400 69.078 48.8125 0.0260132 0.0144706 2023-10-09T04:15:00.320 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:00.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:00.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:00.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:00.505 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:00.622 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:00 smithi181 ceph-mon[196979]: pgmap v695: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 523 op/s; 5779/223992 objects degraded (2.580%); 3833/223992 objects misplaced (1.711%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:15:00.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:15:00.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:15:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:00 smithi163 ceph-mon[213841]: pgmap v695: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 523 op/s; 5779/223992 objects degraded (2.580%); 3833/223992 objects misplaced (1.711%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:15:00.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:15:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:15:00] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:15:00.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:00 smithi163 ceph-mon[221067]: pgmap v695: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 523 op/s; 5779/223992 objects degraded (2.580%); 3833/223992 objects misplaced (1.711%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:15:01.316 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 78192 78176 68.7882 48.5 0.0161385 0.0145313 2023-10-09T04:15:01.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:01.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:01.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:01.537 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:01.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:01 smithi163 ceph-mon[213841]: from='client.35541 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:15:01.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:01 smithi163 ceph-mon[221067]: from='client.35541 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:15:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:01 smithi181 ceph-mon[196979]: from='client.35541 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:15:02.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:02.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:02.285 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":695,"stamp":"2023-10-09T04:14:59.558126+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4892202912,"num_objects":74664,"num_object_clones":0,"num_object_copies":223992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5779,"num_objects_misplaced":3833,"num_objects_unfound":0,"num_objects_dirty":74664,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":74940,"num_write_kb":4779405,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4851,"num_bytes_recovered":318119364,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74856,"ondisk_log_size":74856,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16107996,"kb_used_data":13577956,"kb_used_omap":126,"kb_used_meta":2529857,"kb_avail":640108068,"statfs":{"total":671965249536,"available":655470661632,"internally_reserved":0,"allocated":13903826944,"data_stored":13897558799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":2590574203},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":274530304,"num_objects":4189,"num_object_clones":0,"num_object_copies":12567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-670,"num_objects_unfound":0,"num_objects_dirty":4189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4189,"num_write_kb":268096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":670,"num_bytes_recovered":43909120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005166"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":19,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745690+0000","last_change":"2023-10-09T04:14:15.636875+0000","last_active":"2023-10-09T04:14:51.745690+0000","last_peered":"2023-10-09T04:14:51.745690+0000","last_clean":"2023-10-09T04:14:51.745690+0000","last_became_active":"2023-10-09T04:14:15.635748+0000","last_became_peered":"2023-10-09T04:14:15.635748+0000","last_unstale":"2023-10-09T04:14:51.745690+0000","last_undegraded":"2023-10-09T04:14:51.745690+0000","last_fullsized":"2023-10-09T04:14:51.745690+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:50.038839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":203,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745297+0000","last_change":"2023-10-09T04:14:15.639488+0000","last_active":"2023-10-09T04:14:51.745297+0000","last_peered":"2023-10-09T04:14:51.745297+0000","last_clean":"2023-10-09T04:14:51.745297+0000","last_became_active":"2023-10-09T04:14:15.639163+0000","last_became_peered":"2023-10-09T04:14:15.639163+0000","last_unstale":"2023-10-09T04:14:51.745297+0000","last_undegraded":"2023-10-09T04:14:51.745297+0000","last_fullsized":"2023-10-09T04:14:51.745297+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:05.253619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":218,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.874619+0000","last_change":"2023-10-09T04:14:15.642122+0000","last_active":"2023-10-09T04:14:51.874619+0000","last_peered":"2023-10-09T04:14:51.874619+0000","last_clean":"2023-10-09T04:14:51.874619+0000","last_became_active":"2023-10-09T04:14:15.641842+0000","last_became_peered":"2023-10-09T04:14:15.641842+0000","last_unstale":"2023-10-09T04:14:51.874619+0000","last_undegraded":"2023-10-09T04:14:51.874619+0000","last_fullsized":"2023-10-09T04:14:51.874619+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:39.620348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":213,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.707705+0000","last_change":"2023-10-09T04:03:37.223407+0000","last_active":"2023-10-09T04:14:50.707705+0000","last_peered":"2023-10-09T04:14:50.707705+0000","last_clean":"2023-10-09T04:14:50.707705+0000","last_became_active":"2023-10-09T04:03:37.223062+0000","last_became_peered":"2023-10-09T04:03:37.223062+0000","last_unstale":"2023-10-09T04:14:50.707705+0000","last_undegraded":"2023-10-09T04:14:50.707705+0000","last_fullsized":"2023-10-09T04:14:50.707705+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:13.125408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":211,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701116+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:14:50.701116+0000","last_peered":"2023-10-09T04:14:50.701116+0000","last_clean":"2023-10-09T04:14:50.701116+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:14:50.701116+0000","last_undegraded":"2023-10-09T04:14:50.701116+0000","last_fullsized":"2023-10-09T04:14:50.701116+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":189,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701337+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:14:50.701337+0000","last_peered":"2023-10-09T04:14:50.701337+0000","last_clean":"2023-10-09T04:14:50.701337+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:14:50.701337+0000","last_undegraded":"2023-10-09T04:14:50.701337+0000","last_fullsized":"2023-10-09T04:14:50.701337+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":212,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704798+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:14:50.704798+0000","last_peered":"2023-10-09T04:14:50.704798+0000","last_clean":"2023-10-09T04:14:50.704798+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:14:50.704798+0000","last_undegraded":"2023-10-09T04:14:50.704798+0000","last_fullsized":"2023-10-09T04:14:50.704798+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":236,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.706126+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:14:51.706126+0000","last_peered":"2023-10-09T04:14:51.706126+0000","last_clean":"2023-10-09T04:14:51.706126+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:14:51.706126+0000","last_undegraded":"2023-10-09T04:14:51.706126+0000","last_fullsized":"2023-10-09T04:14:51.706126+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":215,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701240+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:14:50.701240+0000","last_peered":"2023-10-09T04:14:50.701240+0000","last_clean":"2023-10-09T04:14:50.701240+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:14:50.701240+0000","last_undegraded":"2023-10-09T04:14:50.701240+0000","last_fullsized":"2023-10-09T04:14:50.701240+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":74,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.705556+0000","last_change":"2023-10-09T04:13:46.101209+0000","last_active":"2023-10-09T04:14:51.705556+0000","last_peered":"2023-10-09T04:14:51.705556+0000","last_clean":"2023-10-09T04:14:51.705556+0000","last_became_active":"2023-10-09T04:13:46.100828+0000","last_became_peered":"2023-10-09T04:13:46.100828+0000","last_unstale":"2023-10-09T04:14:51.705556+0000","last_undegraded":"2023-10-09T04:14:51.705556+0000","last_fullsized":"2023-10-09T04:14:51.705556+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:59.155026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":234,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703678+0000","last_change":"2023-10-09T04:14:15.683188+0000","last_active":"2023-10-09T04:14:50.703678+0000","last_peered":"2023-10-09T04:14:50.703678+0000","last_clean":"2023-10-09T04:14:50.703678+0000","last_became_active":"2023-10-09T04:14:15.650576+0000","last_became_peered":"2023-10-09T04:14:15.650576+0000","last_unstale":"2023-10-09T04:14:50.703678+0000","last_undegraded":"2023-10-09T04:14:50.703678+0000","last_fullsized":"2023-10-09T04:14:50.703678+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:53.591553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":210,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745823+0000","last_change":"2023-10-09T04:03:06.341096+0000","last_active":"2023-10-09T04:14:51.745823+0000","last_peered":"2023-10-09T04:14:51.745823+0000","last_clean":"2023-10-09T04:14:51.745823+0000","last_became_active":"2023-10-09T04:03:06.340800+0000","last_became_peered":"2023-10-09T04:03:06.340800+0000","last_unstale":"2023-10-09T04:14:51.745823+0000","last_undegraded":"2023-10-09T04:14:51.745823+0000","last_fullsized":"2023-10-09T04:14:51.745823+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:42.079100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":72,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703252+0000","last_change":"2023-10-09T04:14:15.679609+0000","last_active":"2023-10-09T04:14:50.703252+0000","last_peered":"2023-10-09T04:14:50.703252+0000","last_clean":"2023-10-09T04:14:50.703252+0000","last_became_active":"2023-10-09T04:14:15.647071+0000","last_became_peered":"2023-10-09T04:14:15.647071+0000","last_unstale":"2023-10-09T04:14:50.703252+0000","last_undegraded":"2023-10-09T04:14:50.703252+0000","last_fullsized":"2023-10-09T04:14:50.703252+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:20.539552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.868014+0000","last_change":"2023-10-09T04:14:15.642958+0000","last_active":"2023-10-09T04:14:51.868014+0000","last_peered":"2023-10-09T04:14:51.868014+0000","last_clean":"2023-10-09T04:14:51.868014+0000","last_became_active":"2023-10-09T04:14:15.639394+0000","last_became_peered":"2023-10-09T04:14:15.639394+0000","last_unstale":"2023-10-09T04:14:51.868014+0000","last_undegraded":"2023-10-09T04:14:51.868014+0000","last_fullsized":"2023-10-09T04:14:51.868014+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:47.459899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":233,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703141+0000","last_change":"2023-10-09T04:14:15.672582+0000","last_active":"2023-10-09T04:14:50.703141+0000","last_peered":"2023-10-09T04:14:50.703141+0000","last_clean":"2023-10-09T04:14:50.703141+0000","last_became_active":"2023-10-09T04:14:15.644962+0000","last_became_peered":"2023-10-09T04:14:15.644962+0000","last_unstale":"2023-10-09T04:14:50.703141+0000","last_undegraded":"2023-10-09T04:14:50.703141+0000","last_fullsized":"2023-10-09T04:14:50.703141+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:34.505000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745738+0000","last_change":"2023-10-09T04:06:45.469791+0000","last_active":"2023-10-09T04:14:51.745738+0000","last_peered":"2023-10-09T04:14:51.745738+0000","last_clean":"2023-10-09T04:14:51.745738+0000","last_became_active":"2023-10-09T04:06:45.469453+0000","last_became_peered":"2023-10-09T04:06:45.469453+0000","last_unstale":"2023-10-09T04:14:51.745738+0000","last_undegraded":"2023-10-09T04:14:51.745738+0000","last_fullsized":"2023-10-09T04:14:51.745738+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:02.162913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.706283+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:14:50.706283+0000","last_peered":"2023-10-09T04:14:50.706283+0000","last_clean":"2023-10-09T04:14:50.706283+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:14:50.706283+0000","last_undegraded":"2023-10-09T04:14:50.706283+0000","last_fullsized":"2023-10-09T04:14:50.706283+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":214,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701157+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:14:50.701157+0000","last_peered":"2023-10-09T04:14:50.701157+0000","last_clean":"2023-10-09T04:14:50.701157+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:14:50.701157+0000","last_undegraded":"2023-10-09T04:14:50.701157+0000","last_fullsized":"2023-10-09T04:14:50.701157+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.706276+0000","last_change":"2023-10-09T04:14:15.641267+0000","last_active":"2023-10-09T04:14:50.706276+0000","last_peered":"2023-10-09T04:14:50.706276+0000","last_clean":"2023-10-09T04:14:50.706276+0000","last_became_active":"2023-10-09T04:14:15.640941+0000","last_became_peered":"2023-10-09T04:14:15.640941+0000","last_unstale":"2023-10-09T04:14:50.706276+0000","last_undegraded":"2023-10-09T04:14:50.706276+0000","last_fullsized":"2023-10-09T04:14:50.706276+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:03.007095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":212,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.707769+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:14:50.707769+0000","last_peered":"2023-10-09T04:14:50.707769+0000","last_clean":"2023-10-09T04:14:50.707769+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:14:50.707769+0000","last_undegraded":"2023-10-09T04:14:50.707769+0000","last_fullsized":"2023-10-09T04:14:50.707769+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":148,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701007+0000","last_change":"2023-10-09T04:06:57.519209+0000","last_active":"2023-10-09T04:14:50.701007+0000","last_peered":"2023-10-09T04:14:50.701007+0000","last_clean":"2023-10-09T04:14:50.701007+0000","last_became_active":"2023-10-09T04:06:57.518824+0000","last_became_peered":"2023-10-09T04:06:57.518824+0000","last_unstale":"2023-10-09T04:14:50.701007+0000","last_undegraded":"2023-10-09T04:14:50.701007+0000","last_fullsized":"2023-10-09T04:14:50.701007+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:25.736268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.707809+0000","last_change":"2023-10-09T04:03:37.217150+0000","last_active":"2023-10-09T04:14:51.707809+0000","last_peered":"2023-10-09T04:14:51.707809+0000","last_clean":"2023-10-09T04:14:51.707809+0000","last_became_active":"2023-10-09T04:03:37.216877+0000","last_became_peered":"2023-10-09T04:03:37.216877+0000","last_unstale":"2023-10-09T04:14:51.707809+0000","last_undegraded":"2023-10-09T04:14:51.707809+0000","last_fullsized":"2023-10-09T04:14:51.707809+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:46.752974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"256'4640","reported_seq":4677,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:14:55.617450+0000","last_change":"2023-10-09T04:14:16.770519+0000","last_active":"2023-10-09T04:14:55.617450+0000","last_peered":"2023-10-09T04:14:55.617450+0000","last_clean":"2023-10-09T04:14:14.612673+0000","last_became_active":"2023-10-09T04:14:16.764516+0000","last_became_peered":"2023-10-09T04:14:16.764516+0000","last_unstale":"2023-10-09T04:14:55.617450+0000","last_undegraded":"2023-10-09T04:14:16.724188+0000","last_fullsized":"2023-10-09T04:14:16.635177+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4640,"log_dups_size":0,"ondisk_log_size":4640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:24:43.257977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304087040,"num_objects":4640,"num_object_clones":0,"num_object_copies":13920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2892,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4640,"num_write_kb":296960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":2892},{"shards":"2,5","objects":1748}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":230,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703672+0000","last_change":"2023-10-09T04:14:15.666853+0000","last_active":"2023-10-09T04:14:50.703672+0000","last_peered":"2023-10-09T04:14:50.703672+0000","last_clean":"2023-10-09T04:14:50.703672+0000","last_became_active":"2023-10-09T04:14:15.644577+0000","last_became_peered":"2023-10-09T04:14:15.644577+0000","last_unstale":"2023-10-09T04:14:50.703672+0000","last_undegraded":"2023-10-09T04:14:50.703672+0000","last_fullsized":"2023-10-09T04:14:50.703672+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:24.186724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":197,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.696716+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:14:50.696716+0000","last_peered":"2023-10-09T04:14:50.696716+0000","last_clean":"2023-10-09T04:14:50.696716+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:14:50.696716+0000","last_undegraded":"2023-10-09T04:14:50.696716+0000","last_fullsized":"2023-10-09T04:14:50.696716+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"256'4623","reported_seq":10402,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.320929+0000","last_change":"2023-10-09T04:14:51.740175+0000","last_active":"2023-10-09T04:14:55.320929+0000","last_peered":"2023-10-09T04:14:55.320929+0000","last_clean":"2023-10-09T04:14:55.320929+0000","last_became_active":"2023-10-09T04:14:51.711647+0000","last_became_peered":"2023-10-09T04:14:51.711647+0000","last_unstale":"2023-10-09T04:14:55.320929+0000","last_undegraded":"2023-10-09T04:14:55.320929+0000","last_fullsized":"2023-10-09T04:14:55.320929+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4623,"log_dups_size":0,"ondisk_log_size":4623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:02.019730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302972928,"num_objects":4623,"num_object_clones":0,"num_object_copies":13869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4623,"num_write_kb":295872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2867,"num_bytes_recovered":187891712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":231,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703947+0000","last_change":"2023-10-09T04:14:15.685058+0000","last_active":"2023-10-09T04:14:50.703947+0000","last_peered":"2023-10-09T04:14:50.703947+0000","last_clean":"2023-10-09T04:14:50.703947+0000","last_became_active":"2023-10-09T04:14:15.650391+0000","last_became_peered":"2023-10-09T04:14:15.650391+0000","last_unstale":"2023-10-09T04:14:50.703947+0000","last_undegraded":"2023-10-09T04:14:50.703947+0000","last_fullsized":"2023-10-09T04:14:50.703947+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:18.666467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.874564+0000","last_change":"2023-10-09T04:14:15.643389+0000","last_active":"2023-10-09T04:14:51.874564+0000","last_peered":"2023-10-09T04:14:51.874564+0000","last_clean":"2023-10-09T04:14:51.874564+0000","last_became_active":"2023-10-09T04:14:15.643000+0000","last_became_peered":"2023-10-09T04:14:15.643000+0000","last_unstale":"2023-10-09T04:14:51.874564+0000","last_undegraded":"2023-10-09T04:14:51.874564+0000","last_fullsized":"2023-10-09T04:14:51.874564+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:16.257255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"256'4634","reported_seq":4657,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.743916+0000","last_change":"2023-10-09T04:13:42.646173+0000","last_active":"2023-10-09T04:14:55.743916+0000","last_peered":"2023-10-09T04:14:55.743916+0000","last_clean":"2023-10-09T04:14:55.743916+0000","last_became_active":"2023-10-09T04:13:42.645934+0000","last_became_peered":"2023-10-09T04:13:42.645934+0000","last_unstale":"2023-10-09T04:14:55.743916+0000","last_undegraded":"2023-10-09T04:14:55.743916+0000","last_fullsized":"2023-10-09T04:14:55.743916+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4634,"log_dups_size":0,"ondisk_log_size":4634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.323104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303693824,"num_objects":4634,"num_object_clones":0,"num_object_copies":13902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4634,"num_write_kb":296576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":199,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703450+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:14:50.703450+0000","last_peered":"2023-10-09T04:14:50.703450+0000","last_clean":"2023-10-09T04:14:50.703450+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:14:50.703450+0000","last_undegraded":"2023-10-09T04:14:50.703450+0000","last_fullsized":"2023-10-09T04:14:50.703450+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":177,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745544+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:14:51.745544+0000","last_peered":"2023-10-09T04:14:51.745544+0000","last_clean":"2023-10-09T04:14:51.745544+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:14:51.745544+0000","last_undegraded":"2023-10-09T04:14:51.745544+0000","last_fullsized":"2023-10-09T04:14:51.745544+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"257'4813","reported_seq":4838,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-09T04:14:58.745907+0000","last_change":"2023-10-09T04:13:42.847572+0000","last_active":"2023-10-09T04:14:58.745907+0000","last_peered":"2023-10-09T04:14:58.745907+0000","last_clean":"2023-10-09T04:14:58.745907+0000","last_became_active":"2023-10-09T04:13:42.847257+0000","last_became_peered":"2023-10-09T04:13:42.847257+0000","last_unstale":"2023-10-09T04:14:58.745907+0000","last_undegraded":"2023-10-09T04:14:58.745907+0000","last_fullsized":"2023-10-09T04:14:58.745907+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4813,"log_dups_size":0,"ondisk_log_size":4813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:19.215045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4813,"num_write_kb":308032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":215,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.697133+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:14:50.697133+0000","last_peered":"2023-10-09T04:14:50.697133+0000","last_clean":"2023-10-09T04:14:50.697133+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:14:50.697133+0000","last_undegraded":"2023-10-09T04:14:50.697133+0000","last_fullsized":"2023-10-09T04:14:50.697133+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":221,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704509+0000","last_change":"2023-10-09T04:14:15.646072+0000","last_active":"2023-10-09T04:14:50.704509+0000","last_peered":"2023-10-09T04:14:50.704509+0000","last_clean":"2023-10-09T04:14:50.704509+0000","last_became_active":"2023-10-09T04:14:15.645316+0000","last_became_peered":"2023-10-09T04:14:15.645316+0000","last_unstale":"2023-10-09T04:14:50.704509+0000","last_undegraded":"2023-10-09T04:14:50.704509+0000","last_fullsized":"2023-10-09T04:14:50.704509+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:46.990033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"256'4584","reported_seq":4607,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.639225+0000","last_change":"2023-10-09T04:13:42.847429+0000","last_active":"2023-10-09T04:14:55.639225+0000","last_peered":"2023-10-09T04:14:55.639225+0000","last_clean":"2023-10-09T04:14:55.639225+0000","last_became_active":"2023-10-09T04:13:42.847130+0000","last_became_peered":"2023-10-09T04:13:42.847130+0000","last_unstale":"2023-10-09T04:14:55.639225+0000","last_undegraded":"2023-10-09T04:14:55.639225+0000","last_fullsized":"2023-10-09T04:14:55.639225+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4584,"log_dups_size":0,"ondisk_log_size":4584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.587257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300417024,"num_objects":4584,"num_object_clones":0,"num_object_copies":13752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4584,"num_write_kb":293376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":174,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704433+0000","last_change":"2023-10-09T04:14:15.670698+0000","last_active":"2023-10-09T04:14:50.704433+0000","last_peered":"2023-10-09T04:14:50.704433+0000","last_clean":"2023-10-09T04:14:50.704433+0000","last_became_active":"2023-10-09T04:14:15.644875+0000","last_became_peered":"2023-10-09T04:14:15.644875+0000","last_unstale":"2023-10-09T04:14:50.704433+0000","last_undegraded":"2023-10-09T04:14:50.704433+0000","last_fullsized":"2023-10-09T04:14:50.704433+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:32.256058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":18,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745671+0000","last_change":"2023-10-09T04:14:15.649112+0000","last_active":"2023-10-09T04:14:51.745671+0000","last_peered":"2023-10-09T04:14:51.745671+0000","last_clean":"2023-10-09T04:14:51.745671+0000","last_became_active":"2023-10-09T04:14:15.648799+0000","last_became_peered":"2023-10-09T04:14:15.648799+0000","last_unstale":"2023-10-09T04:14:51.745671+0000","last_undegraded":"2023-10-09T04:14:51.745671+0000","last_fullsized":"2023-10-09T04:14:51.745671+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:56.095569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"257'4827","reported_seq":4852,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-09T04:14:58.737375+0000","last_change":"2023-10-09T04:13:42.633796+0000","last_active":"2023-10-09T04:14:58.737375+0000","last_peered":"2023-10-09T04:14:58.737375+0000","last_clean":"2023-10-09T04:14:58.737375+0000","last_became_active":"2023-10-09T04:13:42.633490+0000","last_became_peered":"2023-10-09T04:13:42.633490+0000","last_unstale":"2023-10-09T04:14:58.737375+0000","last_undegraded":"2023-10-09T04:14:58.737375+0000","last_fullsized":"2023-10-09T04:14:58.737375+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4827,"log_dups_size":0,"ondisk_log_size":4827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:46.137500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4827,"num_write_kb":308928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":63,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745614+0000","last_change":"2023-10-09T04:14:15.641213+0000","last_active":"2023-10-09T04:14:51.745614+0000","last_peered":"2023-10-09T04:14:51.745614+0000","last_clean":"2023-10-09T04:14:51.745614+0000","last_became_active":"2023-10-09T04:14:15.640957+0000","last_became_peered":"2023-10-09T04:14:15.640957+0000","last_unstale":"2023-10-09T04:14:51.745614+0000","last_undegraded":"2023-10-09T04:14:51.745614+0000","last_fullsized":"2023-10-09T04:14:51.745614+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:59.293162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.715285+0000","last_change":"2023-10-09T04:14:15.633658+0000","last_active":"2023-10-09T04:14:50.715285+0000","last_peered":"2023-10-09T04:14:50.715285+0000","last_clean":"2023-10-09T04:14:50.715285+0000","last_became_active":"2023-10-09T04:14:15.633317+0000","last_became_peered":"2023-10-09T04:14:15.633317+0000","last_unstale":"2023-10-09T04:14:50.715285+0000","last_undegraded":"2023-10-09T04:14:50.715285+0000","last_fullsized":"2023-10-09T04:14:50.715285+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:04.006584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"256'4718","reported_seq":8090,"reported_epoch":256,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:14:55.748072+0000","last_change":"2023-10-09T04:14:20.797226+0000","last_active":"2023-10-09T04:14:55.748072+0000","last_peered":"2023-10-09T04:14:55.748072+0000","last_clean":"2023-10-09T04:14:14.619766+0000","last_became_active":"2023-10-09T04:14:20.638588+0000","last_became_peered":"2023-10-09T04:14:20.638588+0000","last_unstale":"2023-10-09T04:14:55.748072+0000","last_undegraded":"2023-10-09T04:14:55.748072+0000","last_fullsized":"2023-10-09T04:14:20.591070+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4718,"log_dups_size":0,"ondisk_log_size":4718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:24.430787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309198848,"num_objects":4718,"num_object_clones":0,"num_object_copies":14154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1236,"num_objects_unfound":0,"num_objects_dirty":4718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4718,"num_write_kb":301952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1669,"num_bytes_recovered":109379584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":1236},{"shards":"0,5","objects":3482}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":63,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.710301+0000","last_change":"2023-10-09T04:14:15.637545+0000","last_active":"2023-10-09T04:14:50.710301+0000","last_peered":"2023-10-09T04:14:50.710301+0000","last_clean":"2023-10-09T04:14:50.710301+0000","last_became_active":"2023-10-09T04:14:15.637207+0000","last_became_peered":"2023-10-09T04:14:15.637207+0000","last_unstale":"2023-10-09T04:14:50.710301+0000","last_undegraded":"2023-10-09T04:14:50.710301+0000","last_fullsized":"2023-10-09T04:14:50.710301+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:00.721792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701168+0000","last_change":"2023-10-09T04:14:15.650761+0000","last_active":"2023-10-09T04:14:50.701168+0000","last_peered":"2023-10-09T04:14:50.701168+0000","last_clean":"2023-10-09T04:14:50.701168+0000","last_became_active":"2023-10-09T04:14:15.650370+0000","last_became_peered":"2023-10-09T04:14:15.650370+0000","last_unstale":"2023-10-09T04:14:50.701168+0000","last_undegraded":"2023-10-09T04:14:50.701168+0000","last_fullsized":"2023-10-09T04:14:50.701168+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:32.735819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.3","version":"257'4691","reported_seq":4716,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-09T04:14:58.747465+0000","last_change":"2023-10-09T04:13:42.636533+0000","last_active":"2023-10-09T04:14:58.747465+0000","last_peered":"2023-10-09T04:14:58.747465+0000","last_clean":"2023-10-09T04:14:58.747465+0000","last_became_active":"2023-10-09T04:13:42.636242+0000","last_became_peered":"2023-10-09T04:13:42.636242+0000","last_unstale":"2023-10-09T04:14:58.747465+0000","last_undegraded":"2023-10-09T04:14:58.747465+0000","last_fullsized":"2023-10-09T04:14:58.747465+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4691,"log_dups_size":0,"ondisk_log_size":4691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:13.621286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307429376,"num_objects":4691,"num_object_clones":0,"num_object_copies":14073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4691,"num_write_kb":300224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":224,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.705506+0000","last_change":"2023-10-09T04:03:37.221254+0000","last_active":"2023-10-09T04:14:51.705506+0000","last_peered":"2023-10-09T04:14:51.705506+0000","last_clean":"2023-10-09T04:14:51.705506+0000","last_became_active":"2023-10-09T04:03:37.220387+0000","last_became_peered":"2023-10-09T04:03:37.220387+0000","last_unstale":"2023-10-09T04:14:51.705506+0000","last_undegraded":"2023-10-09T04:14:51.705506+0000","last_fullsized":"2023-10-09T04:14:51.705506+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:52.648118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":219,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.865234+0000","last_change":"2023-10-09T04:14:15.640479+0000","last_active":"2023-10-09T04:14:51.865234+0000","last_peered":"2023-10-09T04:14:51.865234+0000","last_clean":"2023-10-09T04:14:51.865234+0000","last_became_active":"2023-10-09T04:14:15.640075+0000","last_became_peered":"2023-10-09T04:14:15.640075+0000","last_unstale":"2023-10-09T04:14:51.865234+0000","last_undegraded":"2023-10-09T04:14:51.865234+0000","last_fullsized":"2023-10-09T04:14:51.865234+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:52.022199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"256'4669","reported_seq":5280,"reported_epoch":256,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:14:55.854097+0000","last_change":"2023-10-09T04:14:50.526000+0000","last_active":"2023-10-09T04:14:55.854097+0000","last_peered":"2023-10-09T04:14:55.854097+0000","last_clean":"2023-10-09T04:14:14.637748+0000","last_became_active":"2023-10-09T04:14:16.736171+0000","last_became_peered":"2023-10-09T04:14:16.736171+0000","last_unstale":"2023-10-09T04:14:55.854097+0000","last_undegraded":"2023-10-09T04:14:55.854097+0000","last_fullsized":"2023-10-09T04:14:16.634796+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4669,"log_dups_size":0,"ondisk_log_size":4669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:16.840067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305987584,"num_objects":4669,"num_object_clones":0,"num_object_copies":14007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2597,"num_objects_unfound":0,"num_objects_dirty":4669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4669,"num_write_kb":298816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2","objects":2072},{"shards":"0,2,4","objects":2597}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":233,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745872+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:14:51.745872+0000","last_peered":"2023-10-09T04:14:51.745872+0000","last_clean":"2023-10-09T04:14:51.745872+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:14:51.745872+0000","last_undegraded":"2023-10-09T04:14:51.745872+0000","last_fullsized":"2023-10-09T04:14:51.745872+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":232,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701240+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:14:50.701240+0000","last_peered":"2023-10-09T04:14:50.701240+0000","last_clean":"2023-10-09T04:14:50.701240+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:14:50.701240+0000","last_undegraded":"2023-10-09T04:14:50.701240+0000","last_fullsized":"2023-10-09T04:14:50.701240+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":204,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745756+0000","last_change":"2023-10-09T04:14:15.636591+0000","last_active":"2023-10-09T04:14:51.745756+0000","last_peered":"2023-10-09T04:14:51.745756+0000","last_clean":"2023-10-09T04:14:51.745756+0000","last_became_active":"2023-10-09T04:14:15.636051+0000","last_became_peered":"2023-10-09T04:14:15.636051+0000","last_unstale":"2023-10-09T04:14:51.745756+0000","last_undegraded":"2023-10-09T04:14:51.745756+0000","last_fullsized":"2023-10-09T04:14:51.745756+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:19:44.727156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"256'4613","reported_seq":4636,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.657226+0000","last_change":"2023-10-09T04:13:42.847477+0000","last_active":"2023-10-09T04:14:55.657226+0000","last_peered":"2023-10-09T04:14:55.657226+0000","last_clean":"2023-10-09T04:14:55.657226+0000","last_became_active":"2023-10-09T04:13:42.847180+0000","last_became_peered":"2023-10-09T04:13:42.847180+0000","last_unstale":"2023-10-09T04:14:55.657226+0000","last_undegraded":"2023-10-09T04:14:55.657226+0000","last_fullsized":"2023-10-09T04:14:55.657226+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4613,"log_dups_size":0,"ondisk_log_size":4613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:23.775147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302317568,"num_objects":4613,"num_object_clones":0,"num_object_copies":13839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4613,"num_write_kb":295232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":219,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.696968+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:14:50.696968+0000","last_peered":"2023-10-09T04:14:50.696968+0000","last_clean":"2023-10-09T04:14:50.696968+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:14:50.696968+0000","last_undegraded":"2023-10-09T04:14:50.696968+0000","last_fullsized":"2023-10-09T04:14:50.696968+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704261+0000","last_change":"2023-10-09T04:14:15.645858+0000","last_active":"2023-10-09T04:14:50.704261+0000","last_peered":"2023-10-09T04:14:50.704261+0000","last_clean":"2023-10-09T04:14:50.704261+0000","last_became_active":"2023-10-09T04:14:15.644387+0000","last_became_peered":"2023-10-09T04:14:15.644387+0000","last_unstale":"2023-10-09T04:14:50.704261+0000","last_undegraded":"2023-10-09T04:14:50.704261+0000","last_fullsized":"2023-10-09T04:14:50.704261+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:25.666964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745807+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:14:51.745807+0000","last_peered":"2023-10-09T04:14:51.745807+0000","last_clean":"2023-10-09T04:14:51.745807+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:14:51.745807+0000","last_undegraded":"2023-10-09T04:14:51.745807+0000","last_fullsized":"2023-10-09T04:14:51.745807+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"257'4742","reported_seq":4773,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:14:58.734039+0000","last_change":"2023-10-09T04:14:16.750688+0000","last_active":"2023-10-09T04:14:58.734039+0000","last_peered":"2023-10-09T04:14:58.734039+0000","last_clean":"2023-10-09T04:14:14.593157+0000","last_became_active":"2023-10-09T04:14:16.744338+0000","last_became_peered":"2023-10-09T04:14:16.744338+0000","last_unstale":"2023-10-09T04:14:58.734039+0000","last_undegraded":"2023-10-09T04:14:16.651180+0000","last_fullsized":"2023-10-09T04:14:16.635133+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4742,"log_dups_size":0,"ondisk_log_size":4742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:46.659139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310771712,"num_objects":4742,"num_object_clones":0,"num_object_copies":14226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2887,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4742,"num_write_kb":303488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3","objects":1855},{"shards":"1,3,4","objects":2887}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":71,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.710152+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:14:50.710152+0000","last_peered":"2023-10-09T04:14:50.710152+0000","last_clean":"2023-10-09T04:14:50.710152+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:14:50.710152+0000","last_undegraded":"2023-10-09T04:14:50.710152+0000","last_fullsized":"2023-10-09T04:14:50.710152+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":228,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.744993+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:14:51.744993+0000","last_peered":"2023-10-09T04:14:51.744993+0000","last_clean":"2023-10-09T04:14:51.744993+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:14:51.744993+0000","last_undegraded":"2023-10-09T04:14:51.744993+0000","last_fullsized":"2023-10-09T04:14:51.744993+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":228,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.865311+0000","last_change":"2023-10-09T04:02:35.088051+0000","last_active":"2023-10-09T04:14:51.865311+0000","last_peered":"2023-10-09T04:14:51.865311+0000","last_clean":"2023-10-09T04:14:51.865311+0000","last_became_active":"2023-10-09T04:02:35.087741+0000","last_became_peered":"2023-10-09T04:02:35.087741+0000","last_unstale":"2023-10-09T04:14:51.865311+0000","last_undegraded":"2023-10-09T04:14:51.865311+0000","last_fullsized":"2023-10-09T04:14:51.865311+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:23:24.750765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"256'4592","reported_seq":4615,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.609620+0000","last_change":"2023-10-09T04:13:42.638373+0000","last_active":"2023-10-09T04:14:55.609620+0000","last_peered":"2023-10-09T04:14:55.609620+0000","last_clean":"2023-10-09T04:14:55.609620+0000","last_became_active":"2023-10-09T04:13:42.637990+0000","last_became_peered":"2023-10-09T04:13:42.637990+0000","last_unstale":"2023-10-09T04:14:55.609620+0000","last_undegraded":"2023-10-09T04:14:55.609620+0000","last_fullsized":"2023-10-09T04:14:55.609620+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4592,"log_dups_size":0,"ondisk_log_size":4592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:37.637304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300941312,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4592,"num_write_kb":293888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":239,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.696889+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:14:50.696889+0000","last_peered":"2023-10-09T04:14:50.696889+0000","last_clean":"2023-10-09T04:14:50.696889+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:14:50.696889+0000","last_undegraded":"2023-10-09T04:14:50.696889+0000","last_fullsized":"2023-10-09T04:14:50.696889+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":596,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.702380+0000","last_change":"2023-10-09T04:02:03.586300+0000","last_active":"2023-10-09T04:14:50.702380+0000","last_peered":"2023-10-09T04:14:50.702380+0000","last_clean":"2023-10-09T04:14:50.702380+0000","last_became_active":"2023-10-09T04:02:03.585818+0000","last_became_peered":"2023-10-09T04:02:03.585818+0000","last_unstale":"2023-10-09T04:14:50.702380+0000","last_undegraded":"2023-10-09T04:14:50.702380+0000","last_fullsized":"2023-10-09T04:14:50.702380+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:02.630936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":152,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.696878+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:14:50.696878+0000","last_peered":"2023-10-09T04:14:50.696878+0000","last_clean":"2023-10-09T04:14:50.696878+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:14:50.696878+0000","last_undegraded":"2023-10-09T04:14:50.696878+0000","last_fullsized":"2023-10-09T04:14:50.696878+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":23,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703749+0000","last_change":"2023-10-09T04:14:15.650958+0000","last_active":"2023-10-09T04:14:50.703749+0000","last_peered":"2023-10-09T04:14:50.703749+0000","last_clean":"2023-10-09T04:14:50.703749+0000","last_became_active":"2023-10-09T04:14:15.649812+0000","last_became_peered":"2023-10-09T04:14:15.649812+0000","last_unstale":"2023-10-09T04:14:50.703749+0000","last_undegraded":"2023-10-09T04:14:50.703749+0000","last_fullsized":"2023-10-09T04:14:50.703749+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:01.038037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"257'4595","reported_seq":4620,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-09T04:14:58.731898+0000","last_change":"2023-10-09T04:13:42.635883+0000","last_active":"2023-10-09T04:14:58.731898+0000","last_peered":"2023-10-09T04:14:58.731898+0000","last_clean":"2023-10-09T04:14:58.731898+0000","last_became_active":"2023-10-09T04:13:42.635571+0000","last_became_peered":"2023-10-09T04:13:42.635571+0000","last_unstale":"2023-10-09T04:14:58.731898+0000","last_undegraded":"2023-10-09T04:14:58.731898+0000","last_fullsized":"2023-10-09T04:14:58.731898+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4595,"log_dups_size":0,"ondisk_log_size":4595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:34.753582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301137920,"num_objects":4595,"num_object_clones":0,"num_object_copies":13785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4595,"num_write_kb":294080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":236,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.702308+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:14:50.702308+0000","last_peered":"2023-10-09T04:14:50.702308+0000","last_clean":"2023-10-09T04:14:50.702308+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:14:50.702308+0000","last_undegraded":"2023-10-09T04:14:50.702308+0000","last_fullsized":"2023-10-09T04:14:50.702308+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":222,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.706052+0000","last_change":"2023-10-09T04:14:15.639626+0000","last_active":"2023-10-09T04:14:50.706052+0000","last_peered":"2023-10-09T04:14:50.706052+0000","last_clean":"2023-10-09T04:14:50.706052+0000","last_became_active":"2023-10-09T04:14:15.638140+0000","last_became_peered":"2023-10-09T04:14:15.638140+0000","last_unstale":"2023-10-09T04:14:50.706052+0000","last_undegraded":"2023-10-09T04:14:50.706052+0000","last_fullsized":"2023-10-09T04:14:50.706052+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:10.472687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":245,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.707562+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:14:50.707562+0000","last_peered":"2023-10-09T04:14:50.707562+0000","last_clean":"2023-10-09T04:14:50.707562+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:14:50.707562+0000","last_undegraded":"2023-10-09T04:14:50.707562+0000","last_fullsized":"2023-10-09T04:14:50.707562+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"256'4771","reported_seq":4795,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.166109+0000","last_change":"2023-10-09T04:13:42.847637+0000","last_active":"2023-10-09T04:14:55.166109+0000","last_peered":"2023-10-09T04:14:55.166109+0000","last_clean":"2023-10-09T04:14:55.166109+0000","last_became_active":"2023-10-09T04:13:42.847359+0000","last_became_peered":"2023-10-09T04:13:42.847359+0000","last_unstale":"2023-10-09T04:14:55.166109+0000","last_undegraded":"2023-10-09T04:14:55.166109+0000","last_fullsized":"2023-10-09T04:14:55.166109+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4771,"log_dups_size":0,"ondisk_log_size":4771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:39.956849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312672256,"num_objects":4771,"num_object_clones":0,"num_object_copies":14313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4771,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":229,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703535+0000","last_change":"2023-10-09T04:14:15.652476+0000","last_active":"2023-10-09T04:14:50.703535+0000","last_peered":"2023-10-09T04:14:50.703535+0000","last_clean":"2023-10-09T04:14:50.703535+0000","last_became_active":"2023-10-09T04:14:15.652202+0000","last_became_peered":"2023-10-09T04:14:15.652202+0000","last_unstale":"2023-10-09T04:14:50.703535+0000","last_undegraded":"2023-10-09T04:14:50.703535+0000","last_fullsized":"2023-10-09T04:14:50.703535+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:14.699527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":227,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745942+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:14:51.745942+0000","last_peered":"2023-10-09T04:14:51.745942+0000","last_clean":"2023-10-09T04:14:51.745942+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:14:51.745942+0000","last_undegraded":"2023-10-09T04:14:51.745942+0000","last_fullsized":"2023-10-09T04:14:51.745942+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":198,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704343+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:14:50.704343+0000","last_peered":"2023-10-09T04:14:50.704343+0000","last_clean":"2023-10-09T04:14:50.704343+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:14:50.704343+0000","last_undegraded":"2023-10-09T04:14:50.704343+0000","last_fullsized":"2023-10-09T04:14:50.704343+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"256'4581","reported_seq":4605,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.725768+0000","last_change":"2023-10-09T04:13:42.847596+0000","last_active":"2023-10-09T04:14:55.725768+0000","last_peered":"2023-10-09T04:14:55.725768+0000","last_clean":"2023-10-09T04:14:55.725768+0000","last_became_active":"2023-10-09T04:13:42.847298+0000","last_became_peered":"2023-10-09T04:13:42.847298+0000","last_unstale":"2023-10-09T04:14:55.725768+0000","last_undegraded":"2023-10-09T04:14:55.725768+0000","last_fullsized":"2023-10-09T04:14:55.725768+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4581,"log_dups_size":0,"ondisk_log_size":4581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:42.275480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220416,"num_objects":4581,"num_object_clones":0,"num_object_copies":13743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4581,"num_write_kb":293184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":160,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.703497+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:14:50.703497+0000","last_peered":"2023-10-09T04:14:50.703497+0000","last_clean":"2023-10-09T04:14:50.703497+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:14:50.703497+0000","last_undegraded":"2023-10-09T04:14:50.703497+0000","last_fullsized":"2023-10-09T04:14:50.703497+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":203,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745591+0000","last_change":"2023-10-09T04:14:15.638689+0000","last_active":"2023-10-09T04:14:51.745591+0000","last_peered":"2023-10-09T04:14:51.745591+0000","last_clean":"2023-10-09T04:14:51.745591+0000","last_became_active":"2023-10-09T04:14:15.638413+0000","last_became_peered":"2023-10-09T04:14:15.638413+0000","last_unstale":"2023-10-09T04:14:51.745591+0000","last_undegraded":"2023-10-09T04:14:51.745591+0000","last_fullsized":"2023-10-09T04:14:51.745591+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T03:53:53.546696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":186,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.702549+0000","last_change":"2023-10-09T04:06:45.475916+0000","last_active":"2023-10-09T04:14:50.702549+0000","last_peered":"2023-10-09T04:14:50.702549+0000","last_clean":"2023-10-09T04:14:50.702549+0000","last_became_active":"2023-10-09T04:06:45.475610+0000","last_became_peered":"2023-10-09T04:06:45.475610+0000","last_unstale":"2023-10-09T04:14:50.702549+0000","last_undegraded":"2023-10-09T04:14:50.702549+0000","last_fullsized":"2023-10-09T04:14:50.702549+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:45.192616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"256'4547","reported_seq":4570,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:55.655806+0000","last_change":"2023-10-09T04:13:42.646443+0000","last_active":"2023-10-09T04:14:55.655806+0000","last_peered":"2023-10-09T04:14:55.655806+0000","last_clean":"2023-10-09T04:14:55.655806+0000","last_became_active":"2023-10-09T04:13:42.646138+0000","last_became_peered":"2023-10-09T04:13:42.646138+0000","last_unstale":"2023-10-09T04:14:55.655806+0000","last_undegraded":"2023-10-09T04:14:55.655806+0000","last_fullsized":"2023-10-09T04:14:55.655806+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":4547,"log_dups_size":0,"ondisk_log_size":4547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:56.607273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297992192,"num_objects":4547,"num_object_clones":0,"num_object_copies":13641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4547,"num_write_kb":291008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":76,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.717363+0000","last_change":"2023-10-09T04:13:46.100979+0000","last_active":"2023-10-09T04:14:50.717363+0000","last_peered":"2023-10-09T04:14:50.717363+0000","last_clean":"2023-10-09T04:14:50.717363+0000","last_became_active":"2023-10-09T04:13:46.100727+0000","last_became_peered":"2023-10-09T04:13:46.100727+0000","last_unstale":"2023-10-09T04:14:50.717363+0000","last_undegraded":"2023-10-09T04:14:50.717363+0000","last_fullsized":"2023-10-09T04:14:50.717363+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:11.755013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":229,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.707639+0000","last_change":"2023-10-09T04:14:15.664851+0000","last_active":"2023-10-09T04:14:50.707639+0000","last_peered":"2023-10-09T04:14:50.707639+0000","last_clean":"2023-10-09T04:14:50.707639+0000","last_became_active":"2023-10-09T04:14:15.641752+0000","last_became_peered":"2023-10-09T04:14:15.641752+0000","last_unstale":"2023-10-09T04:14:50.707639+0000","last_undegraded":"2023-10-09T04:14:50.707639+0000","last_fullsized":"2023-10-09T04:14:50.707639+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:01.664828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.696875+0000","last_change":"2023-10-09T04:14:15.645994+0000","last_active":"2023-10-09T04:14:50.696875+0000","last_peered":"2023-10-09T04:14:50.696875+0000","last_clean":"2023-10-09T04:14:50.696875+0000","last_became_active":"2023-10-09T04:14:15.645531+0000","last_became_peered":"2023-10-09T04:14:15.645531+0000","last_unstale":"2023-10-09T04:14:50.696875+0000","last_undegraded":"2023-10-09T04:14:50.696875+0000","last_fullsized":"2023-10-09T04:14:50.696875+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:01.356033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":210,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704599+0000","last_change":"2023-10-09T04:14:15.646203+0000","last_active":"2023-10-09T04:14:50.704599+0000","last_peered":"2023-10-09T04:14:50.704599+0000","last_clean":"2023-10-09T04:14:50.704599+0000","last_became_active":"2023-10-09T04:14:15.645653+0000","last_became_peered":"2023-10-09T04:14:15.645653+0000","last_unstale":"2023-10-09T04:14:50.704599+0000","last_undegraded":"2023-10-09T04:14:50.704599+0000","last_fullsized":"2023-10-09T04:14:50.704599+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:29.430251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":70,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.715221+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:14:50.715221+0000","last_peered":"2023-10-09T04:14:50.715221+0000","last_clean":"2023-10-09T04:14:50.715221+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:14:50.715221+0000","last_undegraded":"2023-10-09T04:14:50.715221+0000","last_fullsized":"2023-10-09T04:14:50.715221+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":231,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.697220+0000","last_change":"2023-10-09T04:14:15.662341+0000","last_active":"2023-10-09T04:14:50.697220+0000","last_peered":"2023-10-09T04:14:50.697220+0000","last_clean":"2023-10-09T04:14:50.697220+0000","last_became_active":"2023-10-09T04:14:15.639228+0000","last_became_peered":"2023-10-09T04:14:15.639228+0000","last_unstale":"2023-10-09T04:14:50.697220+0000","last_undegraded":"2023-10-09T04:14:50.697220+0000","last_fullsized":"2023-10-09T04:14:50.697220+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:53.048941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.865380+0000","last_change":"2023-10-09T04:14:15.642021+0000","last_active":"2023-10-09T04:14:51.865380+0000","last_peered":"2023-10-09T04:14:51.865380+0000","last_clean":"2023-10-09T04:14:51.865380+0000","last_became_active":"2023-10-09T04:14:15.638972+0000","last_became_peered":"2023-10-09T04:14:15.638972+0000","last_unstale":"2023-10-09T04:14:51.865380+0000","last_undegraded":"2023-10-09T04:14:51.865380+0000","last_fullsized":"2023-10-09T04:14:51.865380+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:53.938009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":214,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.701387+0000","last_change":"2023-10-09T04:14:15.668713+0000","last_active":"2023-10-09T04:14:50.701387+0000","last_peered":"2023-10-09T04:14:50.701387+0000","last_clean":"2023-10-09T04:14:50.701387+0000","last_became_active":"2023-10-09T04:14:15.640244+0000","last_became_peered":"2023-10-09T04:14:15.640244+0000","last_unstale":"2023-10-09T04:14:50.701387+0000","last_undegraded":"2023-10-09T04:14:50.701387+0000","last_fullsized":"2023-10-09T04:14:50.701387+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:57.897594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704173+0000","last_change":"2023-10-09T04:14:15.638907+0000","last_active":"2023-10-09T04:14:50.704173+0000","last_peered":"2023-10-09T04:14:50.704173+0000","last_clean":"2023-10-09T04:14:50.704173+0000","last_became_active":"2023-10-09T04:14:15.638262+0000","last_became_peered":"2023-10-09T04:14:15.638262+0000","last_unstale":"2023-10-09T04:14:50.704173+0000","last_undegraded":"2023-10-09T04:14:50.704173+0000","last_fullsized":"2023-10-09T04:14:50.704173+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:40.485179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":71,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.715433+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:14:50.715433+0000","last_peered":"2023-10-09T04:14:50.715433+0000","last_clean":"2023-10-09T04:14:50.715433+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:14:50.715433+0000","last_undegraded":"2023-10-09T04:14:50.715433+0000","last_fullsized":"2023-10-09T04:14:50.715433+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":70,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.710229+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:14:50.710229+0000","last_peered":"2023-10-09T04:14:50.710229+0000","last_clean":"2023-10-09T04:14:50.710229+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:14:50.710229+0000","last_undegraded":"2023-10-09T04:14:50.710229+0000","last_fullsized":"2023-10-09T04:14:50.710229+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":247,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-09T04:14:50.704881+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:14:50.704881+0000","last_peered":"2023-10-09T04:14:50.704881+0000","last_clean":"2023-10-09T04:14:50.704881+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:14:50.704881+0000","last_undegraded":"2023-10-09T04:14:50.704881+0000","last_fullsized":"2023-10-09T04:14:50.704881+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-09T04:14:51.745007+0000","last_change":"2023-10-09T04:14:15.649318+0000","last_active":"2023-10-09T04:14:51.745007+0000","last_peered":"2023-10-09T04:14:51.745007+0000","last_clean":"2023-10-09T04:14:51.745007+0000","last_became_active":"2023-10-09T04:14:15.648995+0000","last_became_peered":"2023-10-09T04:14:15.648995+0000","last_unstale":"2023-10-09T04:14:51.745007+0000","last_undegraded":"2023-10-09T04:14:51.745007+0000","last_fullsized":"2023-10-09T04:14:51.745007+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:12:16.281531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5988,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4891607040,"num_objects":74640,"num_object_clones":0,"num_object_copies":223920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5779,"num_objects_misplaced":3833,"num_objects_unfound":0,"num_objects_dirty":74640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74640,"num_write_kb":4776960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4827,"num_bytes_recovered":316342272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14901903360,"data_stored":14901903360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74640,"ondisk_log_size":74640,"up":48,"acting":44,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201549,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1896712,"kb_used_data":1599424,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91848440,"statfs":{"total":95995035648,"available":94052802560,"internally_reserved":0,"allocated":1637810176,"data_stored":1637077983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:14:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":4.5970000000000004,"15min":2.2949999999999999},"min":{"1min":0.24299999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":9.2569999999999997,"5min":126.752,"15min":126.752},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":4.7590000000000003,"15min":2.3149999999999999},"min":{"1min":0.24399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.1669999999999998,"5min":128.88399999999999,"15min":128.88399999999999},"last":2.4369999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:14:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":6.4029999999999996,"15min":2.9540000000000002},"min":{"1min":0.27200000000000002,"5min":0.23899999999999999,"15min":0.21199999999999999},"max":{"1min":6.7569999999999997,"5min":245.28700000000001,"15min":245.28700000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.1830000000000001,"5min":6.1580000000000004,"15min":2.847},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":6.0510000000000002,"5min":244.626,"15min":244.626},"last":2.4710000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":2.0840000000000001,"15min":1.6859999999999999},"min":{"1min":0.29799999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":4.4809999999999999,"5min":76.774000000000001,"15min":76.774000000000001},"last":4.1420000000000003},{"interface":"front","average":{"1min":0.79400000000000004,"5min":2.2400000000000002,"15min":2.1440000000000001},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.27600000000000002},"max":{"1min":4.4950000000000001,"5min":76.715999999999994,"15min":99.372},"last":4.1920000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:14:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.952,"15min":1.9450000000000001},"min":{"1min":0.34399999999999997,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":4.46,"5min":45.475999999999999,"15min":99.134},"last":6.4269999999999996},{"interface":"front","average":{"1min":1.2949999999999999,"5min":2.0630000000000002,"15min":2.0459999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.605,"5min":45.453000000000003,"15min":99.117999999999995},"last":6.4420000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.622,"15min":0.70599999999999996},"min":{"1min":0.32000000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.8360000000000001,"5min":16.195,"15min":16.195},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66400000000000003,"15min":0.751},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.0979999999999999,"5min":4.0949999999999998,"15min":12.442},"last":0.39800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.59399999999999997,"15min":0.63700000000000001},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":1.946,"5min":4.4829999999999997,"15min":6.5439999999999996},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.67200000000000004,"15min":0.71199999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.4210000000000003,"5min":4.9359999999999999,"15min":5.6779999999999999},"last":0.45200000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.64400000000000002,"15min":0.72099999999999997},"min":{"1min":0.35999999999999999,"5min":0.34000000000000002,"15min":0.311},"max":{"1min":1.881,"5min":4.5199999999999996,"15min":8.3559999999999999},"last":0.497},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.65000000000000002,"15min":0.75600000000000001},"min":{"1min":0.34899999999999998,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":1.8919999999999999,"5min":4.9089999999999998,"15min":5.9039999999999999},"last":0.38600000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463186,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1880764,"kb_used_data":1575284,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91864388,"statfs":{"total":95995035648,"available":94069133312,"internally_reserved":0,"allocated":1613090816,"data_stored":1612370476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":3.0790000000000002,"15min":2.2669999999999999},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":6.298,"5min":77.453000000000003,"15min":78.983999999999995},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.996,"5min":3.5139999999999998,"15min":2.633},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":5.7640000000000002,"5min":77.430999999999997,"15min":78.998000000000005},"last":1.843}]},{"osd":1,"last update":"Mon Oct 9 04:13:59 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":2.25,"15min":1.9450000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.411,"5min":62.698,"15min":62.698},"last":2.141},{"interface":"front","average":{"1min":0.69699999999999995,"5min":2.4390000000000001,"15min":2.0430000000000001},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":4.633,"5min":63.530999999999999,"15min":63.530999999999999},"last":5.4420000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:14:27 2023","interfaces":[{"interface":"back","average":{"1min":8.7230000000000008,"5min":5.0309999999999997,"15min":2.6560000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.218},"max":{"1min":139.06899999999999,"5min":139.06899999999999,"15min":139.06899999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":8.7669999999999995,"5min":5.0810000000000004,"15min":2.7040000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":139.173,"5min":139.173,"15min":139.173},"last":0.68100000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:13:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.508,"15min":2.2930000000000001},"min":{"1min":0.309,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.256,"5min":53.497999999999998,"15min":57.509999999999998},"last":6.6260000000000003},{"interface":"front","average":{"1min":0.75,"5min":2.25,"15min":2.3180000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":3.6499999999999999,"5min":53.460999999999999,"15min":57.453000000000003},"last":0.48899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.497,"15min":0.497},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.282,"5min":4.282,"15min":4.282},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58599999999999997,"15min":0.64900000000000002},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.252},"max":{"1min":2.7650000000000001,"5min":3.2970000000000002,"15min":7.7000000000000002},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:14:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.83299999999999996,"15min":0.85999999999999999},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.7030000000000001,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.94699999999999995,"15min":0.97799999999999998},"min":{"1min":0.35699999999999998,"5min":0.312,"15min":0.29899999999999999},"max":{"1min":4.0759999999999996,"5min":23.006,"15min":54.853999999999999},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:14:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.66300000000000003,"15min":0.64200000000000002},"min":{"1min":0.36599999999999999,"5min":0.313,"15min":0.313},"max":{"1min":2.4649999999999999,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.749,"15min":0.73199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28699999999999998},"max":{"1min":4.1200000000000001,"5min":4.1200000000000001,"15min":4.7679999999999998},"last":0.57999999999999996}]}]},{"osd":1,"up_from":89,"seq":382252089495,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2484436,"kb_used_data":2134028,"kb_used_omap":9,"kb_used_meta":350390,"kb_avail":91260716,"statfs":{"total":95995035648,"available":93450973184,"internally_reserved":0,"allocated":2185244672,"data_stored":2184215151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":358800361},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:14:54 2023","interfaces":[{"interface":"back","average":{"1min":6.7430000000000003,"5min":5.0060000000000002,"15min":2.8919999999999999},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":58.006999999999998,"5min":68.048000000000002,"15min":85.977999999999994},"last":1.544},{"interface":"front","average":{"1min":7.0659999999999998,"5min":4.6660000000000004,"15min":3.218},"min":{"1min":0.20799999999999999,"5min":0.182,"15min":0.182},"max":{"1min":58.027000000000001,"5min":61.670000000000002,"15min":99.831999999999994},"last":2.3050000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:14:29 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":2.8450000000000002,"15min":1.984},"min":{"1min":0.27800000000000002,"5min":0.191,"15min":0.191},"max":{"1min":2.7360000000000002,"5min":77.563000000000002,"15min":121.54600000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.4359999999999999,"5min":3.2250000000000001,"15min":2.2200000000000002},"min":{"1min":0.22800000000000001,"5min":0.215,"15min":0.186},"max":{"1min":5.7409999999999997,"5min":77.504999999999995,"15min":99.899000000000001},"last":5.2759999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5190000000000001,"5min":5.1379999999999999,"15min":2.9470000000000001},"min":{"1min":0.28999999999999998,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":16.818999999999999,"5min":138.87299999999999,"15min":138.87299999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":3.605,"5min":5.7149999999999999,"15min":3.552},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":19.748000000000001,"5min":138.49799999999999,"15min":138.49799999999999},"last":5}]},{"osd":4,"last update":"Mon Oct 9 04:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":2.149,"15min":1.8180000000000001},"min":{"1min":0.32200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":11.071,"5min":15.215999999999999,"15min":99.989999999999995},"last":2.343},{"interface":"front","average":{"1min":1.827,"5min":2.0489999999999999,"15min":1.7589999999999999},"min":{"1min":0.47699999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.7609999999999992,"5min":25.957999999999998,"15min":100.249},"last":4.383}]},{"osd":5,"last update":"Mon Oct 9 04:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":2.194,"15min":1.8109999999999999},"min":{"1min":0.38600000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.1,"5min":15.266,"15min":99.933999999999997},"last":2.367},{"interface":"front","average":{"1min":1.8129999999999999,"5min":2.1429999999999998,"15min":1.8759999999999999},"min":{"1min":0.40000000000000002,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":9.7850000000000001,"5min":25.890999999999998,"15min":100.34099999999999},"last":4.4109999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":2.2349999999999999,"15min":1.9930000000000001},"min":{"1min":0.432,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6809999999999992,"5min":25.931000000000001,"15min":100.309},"last":4.3470000000000004},{"interface":"front","average":{"1min":1.51,"5min":1.7,"15min":1.7589999999999999},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":6.915,"5min":13.446,"15min":100.143},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:14:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":2.234,"15min":1.9790000000000001},"min":{"1min":0.441,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.7349999999999994,"5min":25.984999999999999,"15min":100.214},"last":4.4560000000000004},{"interface":"front","average":{"1min":1.675,"5min":1.76,"15min":1.7869999999999999},"min":{"1min":0.45100000000000001,"5min":0.374,"15min":0.317},"max":{"1min":6.8739999999999997,"5min":13.471,"15min":100.113},"last":0.48699999999999999}]}]},{"osd":0,"up_from":84,"seq":360777253020,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2424676,"kb_used_data":2068444,"kb_used_omap":28,"kb_used_meta":356195,"kb_avail":91320476,"statfs":{"total":95995035648,"available":93512167424,"internally_reserved":0,"allocated":2118086656,"data_stored":2117033067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":364744528},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.347,"15min":2.3959999999999999},"min":{"1min":0.187,"5min":0.184,"15min":0.17000000000000001},"max":{"1min":6.1210000000000004,"5min":37.764000000000003,"15min":118.48},"last":0.373},{"interface":"front","average":{"1min":2.8849999999999998,"5min":3.2349999999999999,"15min":2.8690000000000002},"min":{"1min":0.26100000000000001,"5min":0.25,"15min":0.214},"max":{"1min":10.968,"5min":38.128999999999998,"15min":118.384},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:14:29 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":2.5019999999999998,"15min":2.5019999999999998},"min":{"1min":0.308,"5min":0.22500000000000001,"15min":0.221},"max":{"1min":11.012,"5min":13.108000000000001,"15min":118.54000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.3479999999999999,"15min":2.5950000000000002},"min":{"1min":0.21099999999999999,"5min":0.20799999999999999,"15min":0.20300000000000001},"max":{"1min":2.4199999999999999,"5min":12.821,"15min":118.727},"last":0.39500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":5.0549999999999997,"15min":4.3600000000000003},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":10.813000000000001,"5min":156.488,"15min":156.488},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.8940000000000001,"5min":5.2039999999999997,"15min":4.452},"min":{"1min":0.39500000000000002,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":11.045999999999999,"5min":156.50700000000001,"15min":156.50700000000001},"last":0.51200000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":1.845,"15min":1.927},"min":{"1min":0.34100000000000003,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.702,"5min":9.7270000000000003,"15min":118.633},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.6180000000000001,"15min":2.1619999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.1870000000000003,"5min":10.492000000000001,"15min":118.913},"last":0.433}]},{"osd":5,"last update":"Mon Oct 9 04:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.6739999999999999,"15min":2.6789999999999998},"min":{"1min":0.31,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.1210000000000004,"5min":10.445,"15min":182.68899999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.571,"15min":2.4700000000000002},"min":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0369999999999999,"5min":9.8919999999999995,"15min":182.66300000000001},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 04:14:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":3.2410000000000001,"15min":2.3740000000000001},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0659999999999998,"5min":137.36099999999999,"15min":137.36099999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.7330000000000001,"5min":3.6259999999999999,"15min":2.387},"min":{"1min":0.42399999999999999,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.6319999999999997,"5min":136.88999999999999,"15min":136.88999999999999},"last":0.59099999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":2.6280000000000001,"15min":3.0150000000000001},"min":{"1min":0.39200000000000002,"5min":0.33800000000000002,"15min":0.32100000000000001},"max":{"1min":6.0949999999999998,"5min":121.324,"15min":185.024},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.0979999999999999,"15min":2.6459999999999999},"min":{"1min":0.53600000000000003,"5min":0.38900000000000001,"15min":0.32700000000000001},"max":{"1min":6.298,"5min":12.417,"15min":184.98599999999999},"last":0.64000000000000001}]}]},{"osd":2,"up_from":94,"seq":403726925968,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2266056,"kb_used_data":1878336,"kb_used_omap":81,"kb_used_meta":387630,"kb_avail":91479096,"statfs":{"total":95995035648,"available":93674594304,"internally_reserved":0,"allocated":1923416064,"data_stored":1922393416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":396933338},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:14:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":3.0800000000000001,"15min":1.8600000000000001},"min":{"1min":0.224,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":11.066000000000001,"5min":46.750999999999998,"15min":46.750999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.1930000000000001,"5min":2.8479999999999999,"15min":2.1200000000000001},"min":{"1min":0.26500000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":8.9949999999999992,"5min":48.350999999999999,"15min":48.350999999999999},"last":1.4850000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:14:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":2.1619999999999999,"15min":2.1869999999999998},"min":{"1min":0.28100000000000003,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":11.726000000000001,"5min":14.99,"15min":68.009},"last":2.3220000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":1.748,"15min":2.198},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.5720000000000001,"5min":16.597000000000001,"15min":66.975999999999999},"last":0.33600000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:14:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.4790000000000001,"15min":1.788},"min":{"1min":0.26000000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.403,"5min":20.242999999999999,"15min":27.562000000000001},"last":2.4769999999999999},{"interface":"front","average":{"1min":1.4670000000000001,"5min":2.3940000000000001,"15min":1.986},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.773999999999999,"5min":17.076000000000001,"15min":46.286999999999999},"last":6.4779999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:14:23 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.1200000000000001,"15min":1.458},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.149,"5min":7.5279999999999996,"15min":46.075000000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.712,"15min":1.3080000000000001},"min":{"1min":0.32800000000000001,"5min":0.28899999999999998,"15min":0.27200000000000002},"max":{"1min":4.2539999999999996,"5min":20.134,"15min":20.134},"last":0.629}]},{"osd":5,"last update":"Mon Oct 9 04:14:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.776,"15min":1.591},"min":{"1min":0.40799999999999997,"5min":0.33400000000000002,"15min":0.27600000000000002},"max":{"1min":8.4339999999999993,"5min":26.667000000000002,"15min":46.265000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.4410000000000001,"15min":1.6180000000000001},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.0039999999999996,"5min":26.654,"15min":46.106000000000002},"last":0.38400000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:14:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":3.0369999999999999,"15min":1.744},"min":{"1min":0.40600000000000003,"5min":0.34200000000000003,"15min":0.311},"max":{"1min":4.2169999999999996,"5min":114.14,"15min":114.14},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.353,"5min":2.8799999999999999,"15min":1.96},"min":{"1min":0.47299999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.5050000000000008,"5min":114.123,"15min":114.123},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:14:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":3.2330000000000001,"15min":1.9399999999999999},"min":{"1min":0.46700000000000003,"5min":0.36299999999999999,"15min":0.34300000000000003},"max":{"1min":4.367,"5min":117.32599999999999,"15min":117.32599999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.3540000000000001,"5min":2.992,"15min":2.0830000000000002},"min":{"1min":0.44800000000000001,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":8.4779999999999998,"5min":117.28700000000001,"15min":117.28700000000001},"last":0.51300000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762442,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2529260,"kb_used_data":1993636,"kb_used_omap":7,"kb_used_meta":535608,"kb_avail":91215892,"statfs":{"total":95995035648,"available":93405073408,"internally_reserved":0,"allocated":2041483264,"data_stored":2040482124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":548463208},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":4.6239999999999997,"15min":3.7330000000000001},"min":{"1min":0.22500000000000001,"5min":0.19400000000000001,"15min":0.185},"max":{"1min":15.000999999999999,"5min":100.512,"15min":106.22},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.878,"5min":5.0140000000000002,"15min":3.8639999999999999},"min":{"1min":0.23799999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":6.3769999999999998,"5min":101.733,"15min":106.986},"last":0.45700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":3.577,"15min":2.581},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":14.949999999999999,"5min":57.055999999999997,"15min":106.121},"last":0.314},{"interface":"front","average":{"1min":1.859,"5min":3.633,"15min":2.4340000000000002},"min":{"1min":0.255,"5min":0.193,"15min":0.193},"max":{"1min":15.129,"5min":57.088999999999999,"15min":65.015000000000001},"last":0.27700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":2.4209999999999998,"15min":2.4209999999999998},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":15.103,"5min":23.405999999999999,"15min":106.105},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":3.4590000000000001,"15min":2.8220000000000001},"min":{"1min":0.28599999999999998,"5min":0.217,"15min":0.217},"max":{"1min":6.0049999999999999,"5min":98.765000000000001,"15min":106.70399999999999},"last":3.7749999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.859,"15min":2.097},"min":{"1min":0.33100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.2639999999999998,"5min":13.536,"15min":106.85899999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.6779999999999999,"15min":1.5009999999999999},"min":{"1min":0.33200000000000002,"5min":0.29899999999999999,"15min":0.26900000000000002},"max":{"1min":10.366,"5min":10.366,"15min":37.442},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.6830000000000001,"15min":1.413},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.340999999999999,"5min":10.340999999999999,"15min":12.819000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.367,"5min":1.7989999999999999,"15min":1.6479999999999999},"min":{"1min":0.34000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":14.932,"5min":14.932,"15min":14.932},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.7250000000000001,"15min":2.0190000000000001},"min":{"1min":0.41499999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":14.976000000000001,"5min":14.976000000000001,"15min":106.036},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.70899999999999996,"5min":1.9650000000000001,"15min":1.966},"min":{"1min":0.44400000000000001,"5min":0.318,"15min":0.318},"max":{"1min":1.5509999999999999,"5min":13.571,"15min":106.741},"last":0.52200000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.9570000000000001,"15min":2.0259999999999998},"min":{"1min":0.39700000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.4320000000000004,"5min":13.494999999999999,"15min":106.82599999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.177,"5min":1.857,"15min":1.897},"min":{"1min":0.433,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":10.298,"5min":10.298,"15min":106.084},"last":0.438}]}]},{"osd":4,"up_from":0,"seq":133143986458,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724822,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2626092,"kb_used_data":2328804,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91119060,"statfs":{"total":95995035648,"available":93305917440,"internally_reserved":0,"allocated":2384695296,"data_stored":2383986582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:14:27 2023","interfaces":[{"interface":"back","average":{"1min":5.633,"5min":7.9180000000000001,"15min":3.6070000000000002},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":108.58199999999999,"5min":151.23599999999999,"15min":151.23599999999999},"last":0.23400000000000001},{"interface":"front","average":{"1min":5.6929999999999996,"5min":8.4640000000000004,"15min":3.8540000000000001},"min":{"1min":0.33500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":109.742,"5min":151.566,"15min":151.566},"last":1.042}]},{"osd":1,"last update":"Mon Oct 9 04:14:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":3.6339999999999999,"15min":2.4580000000000002},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":9.5310000000000006,"5min":72.734999999999999,"15min":91.212999999999994},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.4159999999999999,"5min":3.7879999999999998,"15min":2.6699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.23300000000000001},"max":{"1min":4.7619999999999996,"5min":72.762,"15min":91.254000000000005},"last":1.0069999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:14:27 2023","interfaces":[{"interface":"back","average":{"1min":6.0010000000000003,"5min":3.702,"15min":2.508},"min":{"1min":0.36699999999999999,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":106.508,"5min":106.508,"15min":106.508},"last":0.36299999999999999},{"interface":"front","average":{"1min":5.7549999999999999,"5min":3.8220000000000001,"15min":2.4940000000000002},"min":{"1min":0.34499999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":106.462,"5min":106.462,"15min":106.462},"last":1.1459999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:14:55 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.1259999999999999,"15min":2.1070000000000002},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":27.925000000000001,"5min":27.925000000000001,"15min":91.834999999999994},"last":1.9399999999999999},{"interface":"front","average":{"1min":3.2309999999999999,"5min":2.1549999999999998,"15min":2.145},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":27.939,"5min":27.939,"15min":91.802000000000007},"last":1.929}]},{"osd":4,"last update":"Mon Oct 9 04:14:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.57999999999999996,"15min":0.67900000000000005},"min":{"1min":0.29599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":3.2109999999999999,"5min":3.2440000000000002,"15min":5.7430000000000003},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.59299999999999997,"15min":0.72599999999999998},"min":{"1min":0.32300000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.944,"5min":4.944,"15min":6.7709999999999999},"last":0.93000000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:14:55 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.917,"15min":1.1279999999999999},"min":{"1min":0.312,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":2.6930000000000001,"5min":123.239,"15min":123.239},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.056,"5min":2.0430000000000001,"15min":1.1739999999999999},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.7119999999999997,"5min":123.014,"15min":123.014},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:14:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.60699999999999998,"15min":0.79200000000000004},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":2.6760000000000002,"5min":3.25,"15min":9.4930000000000003},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.64500000000000002,"15min":0.82199999999999995},"min":{"1min":0.41499999999999998,"5min":0.307,"15min":0.307},"max":{"1min":3.3130000000000002,"5min":3.3130000000000002,"15min":8.5329999999999995},"last":0.63800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2133000192,"data_stored":2133000192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2185756672,"data_stored":2185756672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1923612672,"data_stored":1923612672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2055471104,"data_stored":2055471104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":938475520,"data_stored":938475520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2405302272,"data_stored":2405302272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1618477056,"data_stored":1618477056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1641807872,"data_stored":1641807872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:15:02.292 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:15:02.292 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-10-09T04:15:02.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:02.317 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 78905 78889 68.451 44.5625 0.0223389 0.0146037 2023-10-09T04:15:02.450 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:02.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:02 smithi163 ceph-mon[221067]: pgmap v696: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 5779/231882 objects degraded (2.492%); 3369/231882 objects misplaced (1.453%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:15:02.743 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/231882 objects degraded (2.492%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:02.743 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:02 smithi163 ceph-mon[213841]: pgmap v696: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 5779/231882 objects degraded (2.492%); 3369/231882 objects misplaced (1.453%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:15:02.743 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/231882 objects degraded (2.492%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:02.782 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:02.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:02.853 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:02.862 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:02.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:03.009 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:03.009 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:02 smithi181 ceph-mon[196979]: pgmap v696: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 5779/231882 objects degraded (2.492%); 3369/231882 objects misplaced (1.453%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:15:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/231882 objects degraded (2.492%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:03.024 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:03.254 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:03.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:03.320 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 79566 79550 68.0758 41.3125 0.0169291 0.0146834 2023-10-09T04:15:03.368 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:03.457 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:04.325 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 80213 80197 67.6986 40.4375 0.043054 0.014766 2023-10-09T04:15:04.326 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:04.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:04.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:04.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:04.533 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:04.641 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:04.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:15:04.774 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:04.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:04.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:04 smithi163 ceph-mon[213841]: pgmap v697: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 5779/231882 objects degraded (2.492%); 3369/231882 objects misplaced (1.453%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T04:15:04.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:04.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:04 smithi163 ceph-mon[221067]: pgmap v697: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 5779/231882 objects degraded (2.492%); 3369/231882 objects misplaced (1.453%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T04:15:04.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:04.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:05.018 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:04 smithi181 ceph-mon[196979]: pgmap v697: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 681 op/s; 5779/231882 objects degraded (2.492%); 3369/231882 objects misplaced (1.453%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T04:15:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:05.327 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 80880 80864 67.3506 41.6875 0.00392375 0.0148306 2023-10-09T04:15:05.769 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:05.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:05.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:05.918 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:05.966 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:05.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:06.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:06.133 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:06.326 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 81697 81681 67.1363 51.0625 0.0176213 0.0148894 2023-10-09T04:15:06.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:06 smithi163 ceph-mon[221067]: pgmap v698: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 5779/236637 objects degraded (2.442%); 3369/236637 objects misplaced (1.424%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:15:06.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:06 smithi163 ceph-mon[213841]: pgmap v698: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 5779/236637 objects degraded (2.442%); 3369/236637 objects misplaced (1.424%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:15:07.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:06 smithi181 ceph-mon[196979]: pgmap v698: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 5779/236637 objects degraded (2.442%); 3369/236637 objects misplaced (1.424%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:15:07.331 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 82453 82437 66.8781 47.25 0.0132485 0.0149476 2023-10-09T04:15:07.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:07.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:07.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:07.777 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:07.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:07.901 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/236637 objects degraded (2.442%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:07.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:07.902 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/236637 objects degraded (2.442%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:07.902 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:07.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:08.003 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/236637 objects degraded (2.442%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:08.040 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:08.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:08.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:08.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:08.230 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:08.281 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:08.330 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 83129 83113 66.5596 42.25 0.0260386 0.0150188 2023-10-09T04:15:08.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:08.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:08.376 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:08.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:08.620 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:08.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:08 smithi163 ceph-mon[221067]: pgmap v699: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 5779/242415 objects degraded (2.384%); 2899/242415 objects misplaced (1.196%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:15:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:08 smithi163 ceph-mon[213841]: pgmap v699: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 5779/242415 objects degraded (2.384%); 2899/242415 objects misplaced (1.196%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:15:08.925 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:15:08.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:15:09.014 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:15:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:08 smithi181 ceph-mon[196979]: pgmap v699: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 5779/242415 objects degraded (2.384%); 2899/242415 objects misplaced (1.196%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:15:09.062 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:15:09.094 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:15:09.141 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:15:09.161 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:15:09.179 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:15:09.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.180 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:15:09.330 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 83689 83673 66.1603 35 0.0030089 0.015083 2023-10-09T04:15:09.397 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:09.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:09.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:09.638 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:09.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1291224729' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:09.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1908240229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:09.767 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:09.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1291224729' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:09.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1908240229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:09.767 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:09.841 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:15:10.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1291224729' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1908240229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:10.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:10.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:10.332 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:15:10.332332+0000 min lat: 0.0010792 max lat: 6.1654 avg lat: 0.0151854 2023-10-09T04:15:10.332 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:15:10.332 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 84325 84309 65.8289 39.75 0.0167529 0.0151854 2023-10-09T04:15:10.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:10.390 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:10.510 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:10.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:10.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:10.643 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:10.732 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 1 pg_num to 11 2023-10-09T04:15:10.740 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:10.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:10.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:10.848 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:10.882 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:10.904 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:15:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:15:10] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: pgmap v700: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 5779/245550 objects degraded (2.353%); 2899/245550 objects misplaced (1.181%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: failed to encode map e258 with expected crc 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: failed to encode map e258 with expected crc 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3969788005' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:15:10.907 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: pgmap v700: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 5779/245550 objects degraded (2.353%); 2899/245550 objects misplaced (1.181%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: failed to encode map e258 with expected crc 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: failed to encode map e258 with expected crc 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3969788005' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:15:10.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:15:10.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:11.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:11.004 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: pgmap v700: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 5779/245550 objects degraded (2.353%); 2899/245550 objects misplaced (1.181%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:15:11.004 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:15:11.004 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: failed to encode map e258 with expected crc 2023-10-09T04:15:11.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: failed to encode map e258 with expected crc 2023-10-09T04:15:11.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:11.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T04:15:11.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3969788005' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:15:11.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:15:11.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:11.140 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:11.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:11.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:11.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:11.332 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 85021 85005 65.5533 43.5 0.00905138 0.0152483 2023-10-09T04:15:11.362 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:11.429 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:11.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:11.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:11.576 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:11.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:11.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:11.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:11.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:11.837 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: failed to encode map e259 with expected crc 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: failed to encode map e259 with expected crc 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T04:15:12.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: failed to encode map e259 with expected crc 2023-10-09T04:15:12.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: failed to encode map e259 with expected crc 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: failed to encode map e259 with expected crc 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T04:15:12.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T04:15:12.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: failed to encode map e259 with expected crc 2023-10-09T04:15:12.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:12.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:12.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T04:15:12.332 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 85589 85573 65.1869 35.5 0.0211317 0.0153357 2023-10-09T04:15:12.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:12.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:12.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:12.688 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:12.747 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[221067]: pgmap v703: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 5779/252564 objects degraded (2.288%); 2346/252564 objects misplaced (0.929%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[221067]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[221067]: failed to encode map e260 with expected crc 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/252564 objects degraded (2.288%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[213841]: pgmap v703: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 5779/252564 objects degraded (2.288%); 2346/252564 objects misplaced (0.929%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[213841]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T04:15:12.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[213841]: failed to encode map e260 with expected crc 2023-10-09T04:15:12.749 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/252564 objects degraded (2.288%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:12.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:12.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:12.847 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:15:12.847 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:15:12.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:15:12.847 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49817 6.c 6:3ffecb2a:::benchmark_data_smithi163_287034_object49816:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 586.15624634200003, 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21672543999999999, 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:12.848 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49817 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687660+0000", 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687659+0000", 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687682+0000", 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3472000000000002e-05 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687694+0000", 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1080999999999999e-05 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687701+0000", 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9000000000000006e-06 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687750+0000", 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8877e-05 2023-10-09T04:15:12.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.687874+0000", 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000123812 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.688008+0000", 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134256 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.690036+0000", 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020273660000000001 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692667+0000", 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026314289999999998 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904254+0000", 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21158665099999999 2023-10-09T04:15:12.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904341+0000", 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7026000000000003e-05 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904386+0000", 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4781999999999999e-05 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.854 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:49825 6.c 6:38f6223e:::benchmark_data_smithi163_287034_object49824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 586.15133397600005, 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.214667942, 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 49825 2023-10-09T04:15:12.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692573+0000", 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692571+0000", 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692600+0000", 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9017999999999999e-05 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692612+0000", 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.171e-05 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692620+0000", 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1240000000000005e-06 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692694+0000", 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4127000000000004e-05 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692812+0000", 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011761599999999999 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.692939+0000", 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012688700000000001 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904425+0000", 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.211486216 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.904454+0000", 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9196999999999998e-05 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907121+0000", 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002666982 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907194+0000", 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3281999999999994e-05 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:05:26.907241+0000", 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6106999999999997e-05 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.44666.0:117683 6.b 6:df64dfc0:::benchmark_data_smithi163_287034_object9420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 511.282085177, 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21390703, 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.44666", 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 117683 2023-10-09T04:15:12.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561821+0000", 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561824+0000", 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.525e-06 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561832+0000", 2023-10-09T04:15:12.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6060000000000003e-06 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561837+0000", 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5090000000000004e-06 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.561900+0000", 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2890000000000003e-05 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.562018+0000", 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011803699999999999 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.562119+0000", 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101275 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.565635+0000", 2023-10-09T04:15:12.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035162940000000001 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.568886+0000", 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032510759999999999 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775646+0000", 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.206759888 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.867 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775692+0000", 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5757999999999998e-05 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:06:41.775728+0000", 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6587000000000001e-05 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.868 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:644 7.c 7:3fcf2acf:::benchmark_data_smithi163_346082_object643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 344.25224082300002, 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21846595499999999, 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 644 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.869 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591664+0000", 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591688+0000", 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3495000000000001e-05 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591699+0000", 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1834000000000001e-05 2023-10-09T04:15:12.871 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591707+0000", 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6029999999999999e-06 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591761+0000", 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4044e-05 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591884+0000", 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012263899999999999 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592028+0000", 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014450300000000001 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809992+0000", 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21796359500000001 2023-10-09T04:15:12.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810008+0000", 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6293000000000001e-05 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810044+0000", 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5468e-05 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.874 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810090+0000", 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6409999999999998e-05 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810132+0000", 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1647999999999998e-05 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.875 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:645 7.c 7:3c7dda99:::benchmark_data_smithi163_346082_object644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 344.25218583100002, 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21961103900000001, 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 645 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.876 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591720+0000", 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591740+0000", 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9584000000000001e-05 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591748+0000", 2023-10-09T04:15:12.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591755+0000", 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7190000000000001e-06 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809375+0000", 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.217619379 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809503+0000", 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012796899999999999 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809689+0000", 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000186577 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810749+0000", 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010602400000000001 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810776+0000", 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6995000000000002e-05 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811241+0000", 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046449799999999998 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811284+0000", 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3325999999999998e-05 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811332+0000", 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7512999999999997e-05 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.882 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:647 7.2 e171/167 7:4b12a746:::benchmark_data_smithi163_346082_object646:head v 171'43, mlcod=171'39)", 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 344.25193292199998, 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21782781300000001, 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:15:12.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591973+0000", 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.884 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591996+0000", 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2821000000000001e-05 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592006+0000", 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0288999999999999e-05 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.885 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592016+0000", 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0186e-05 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592076+0000", 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9373000000000002e-05 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808728+0000", 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21665226300000001 2023-10-09T04:15:12.886 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809757+0000", 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001029409 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809801+0000", 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4079000000000003e-05 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.887 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:644 7.c e171/167 ondisk, result = 0)", 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 344.25116687899998, 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.217269874, 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592739+0000", 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592743+0000", 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.442e-06 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808711+0000", 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21596727399999999 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.890 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808728+0000", 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7541999999999999e-05 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809978+0000", 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001249687 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809985+0000", 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.498e-06 2023-10-09T04:15:12.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810010+0000", 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4077999999999999e-05 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:644 7.c e171/167 ondisk, result = 0)", 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:15:12.892 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 344.25106843200001, 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21730428800000001, 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:15:12.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592842+0000", 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1080000000000003e-06 2023-10-09T04:15:12.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809055+0000", 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.216213023 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809069+0000", 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3861e-05 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.895 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810034+0000", 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000965032 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810039+0000", 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5489999999999997e-06 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810142+0000", 2023-10-09T04:15:12.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010331999999999999 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:38946 7.c 7:3abe6352:::benchmark_data_smithi163_346082_object38945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.87453341600002, 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23044379800000001, 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.897 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38946 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969372+0000", 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969391+0000", 2023-10-09T04:15:12.899 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9732999999999999e-05 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969404+0000", 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2989999999999999e-05 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969412+0000", 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9889999999999992e-06 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969455+0000", 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3009000000000001e-05 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969584+0000", 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012886500000000001 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:15:12.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969717+0000", 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013274599999999999 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972017+0000", 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023001100000000002 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199698+0000", 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22768131599999999 2023-10-09T04:15:12.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199742+0000", 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.375e-05 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199779+0000", 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6940999999999997e-05 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199817+0000", 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7861000000000002e-05 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38943 7.16 e175/174 7:6e66cd39:::benchmark_data_smithi163_346082_object38942:head v 175'2310, mlcod=175'2307)", 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.87254608400002, 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22686308999999999, 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.904 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971382+0000", 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1302e-05 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971388+0000", 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0599999999999996e-06 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971393+0000", 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3909999999999997e-06 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972116+0000", 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072317400000000004 2023-10-09T04:15:12.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972130+0000", 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3424999999999999e-05 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972810+0000", 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068082899999999998 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198224+0000", 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.225413118 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:38951 7.c 7:39c83fcb:::benchmark_data_smithi163_346082_object38950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.87162388399997, 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22769973700000001, 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.909 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38951 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972281+0000", 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972307+0000", 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5286000000000002e-05 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972319+0000", 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2325e-05 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972341+0000", 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1837000000000001e-05 2023-10-09T04:15:12.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972386+0000", 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5516000000000003e-05 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972524+0000", 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013795999999999999 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972659+0000", 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013506 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199867+0000", 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22720789899999999 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199914+0000", 2023-10-09T04:15:12.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6495999999999997e-05 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199923+0000", 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7970000000000006e-06 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199950+0000", 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7152999999999999e-05 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199982+0000", 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2765000000000002e-05 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38945 7.a e175/167 7:5f8231b3:::benchmark_data_smithi163_346082_object38944:head v 175'2413, mlcod=175'2410)", 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.87111132400003, 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.228137689, 2023-10-09T04:15:12.916 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972816+0000", 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1577999999999999e-05 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.918 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198218+0000", 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22540215399999999 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198227+0000", 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9449999999999999e-06 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198279+0000", 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1304999999999998e-05 2023-10-09T04:15:12.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198298+0000", 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9219999999999999e-05 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200904+0000", 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002605797 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200933+0000", 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9247e-05 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:38946 7.c e175/167 ondisk, result = 0)", 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.87108390999998, 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22688220200000001, 2023-10-09T04:15:12.921 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972822+0000", 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972826+0000", 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1230000000000002e-06 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198187+0000", 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22536124799999999 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198204+0000", 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7017999999999999e-05 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199684+0000", 2023-10-09T04:15:12.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001479209 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199692+0000", 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1899999999999995e-06 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199705+0000", 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2989999999999999e-05 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.925 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38948 7.11 e175/174 7:88c099c1:::benchmark_data_smithi163_346082_object38947:head v 175'2351, mlcod=175'2349)", 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.87052875699999, 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22730851799999999, 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973377+0000", 2023-10-09T04:15:12.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973404+0000", 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7041000000000001e-05 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198189+0000", 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22478490800000001 2023-10-09T04:15:12.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198205+0000", 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6181e-05 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198251+0000", 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6136e-05 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198278+0000", 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7257e-05 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200653+0000", 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023747289999999999 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200686+0000", 2023-10-09T04:15:12.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3191999999999997e-05 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80498 7.13 7:ce654277:::benchmark_data_smithi163_346082_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 274.52606582499999, 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1130738259999999, 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.931 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80498 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317838+0000", 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317881+0000", 2023-10-09T04:15:12.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3013000000000003e-05 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317895+0000", 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3732e-05 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317903+0000", 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5209999999999997e-06 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318091+0000", 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188829 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318101+0000", 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.55e-06 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323224+0000", 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051229439999999999 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323236+0000", 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992e-05 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.423747+0000", 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.100511362 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.423904+0000", 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015625099999999999 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.424075+0000", 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000170977 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430599+0000", 2023-10-09T04:15:12.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065249189999999997 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430661+0000", 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1789000000000001e-05 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430676+0000", 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4297999999999999e-05 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430753+0000", 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7937999999999998e-05 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430805+0000", 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1829000000000001e-05 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430915+0000", 2023-10-09T04:15:12.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109257 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80508 7.13 7:cf9b83a2:::benchmark_data_smithi163_346082_object80507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 274.52598971200001, 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1132838570000001, 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.940 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80508 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317916+0000", 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317944+0000", 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7642000000000001e-05 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317952+0000", 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3070000000000003e-06 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317958+0000", 2023-10-09T04:15:12.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1290000000000001e-06 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318152+0000", 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019322999999999999 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318158+0000", 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7700000000000004e-06 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323259+0000", 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051002950000000003 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323278+0000", 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814999999999999e-05 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425771+0000", 2023-10-09T04:15:12.945 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.102493095 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425915+0000", 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000144451 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.426078+0000", 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000162595 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.946 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430682+0000", 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046044320000000003 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430721+0000", 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9020000000000002e-05 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.947 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430993+0000", 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027199400000000001 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431048+0000", 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4808000000000001e-05 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431097+0000", 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9586000000000003e-05 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.948 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431201+0000", 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103231 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80516 7.13 7:c9a4c09d:::benchmark_data_smithi163_346082_object80515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 274.52594295599999, 2023-10-09T04:15:12.949 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.116727963, 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80516 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:15:12.950 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317963+0000", 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.951 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317983+0000", 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9938000000000001e-05 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317990+0000", 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7850000000000003e-06 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.952 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317993+0000", 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3680000000000002e-06 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318180+0000", 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018708599999999999 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318187+0000", 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2589999999999997e-06 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.953 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323302+0000", 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051155979999999998 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323310+0000", 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0660000000000004e-06 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.954 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427478+0000", 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1041673599999999 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427613+0000", 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000135888 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427760+0000", 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014688799999999999 2023-10-09T04:15:12.955 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430688+0000", 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029275270000000001 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431261+0000", 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00057298199999999998 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.956 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434445+0000", 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031837290000000002 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434514+0000", 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8955999999999995e-05 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434567+0000", 2023-10-09T04:15:12.957 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3599e-05 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434692+0000", 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012444600000000001 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80551 7.13 7:c9852ceb:::benchmark_data_smithi163_346082_object80550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:15:12.958 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 274.37343252800002, 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96918742400000002, 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80551 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.959 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470472+0000", 2023-10-09T04:15:12.960 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470498+0000", 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5672000000000001e-05 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470514+0000", 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5846999999999999e-05 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.961 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470522+0000", 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9629999999999998e-06 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470576+0000", 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4360999999999997e-05 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:15:12.962 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470594+0000", 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7754999999999999e-05 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.428996+0000", 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95840238700000002 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429120+0000", 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012304300000000001 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.963 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429313+0000", 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019363099999999999 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431216+0000", 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019029240000000001 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.964 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434407+0000", 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031906230000000001 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434794+0000", 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00038685899999999998 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439487+0000", 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046938939999999997 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.965 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439541+0000", 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3309000000000001e-05 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439661+0000", 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120699 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.966 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80553 7.3 7:c0606c4f:::benchmark_data_smithi163_346082_object80552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 274.370242405, 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95719071, 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.967 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80553 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.968 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473663+0000", 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473684+0000", 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1585000000000001e-05 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.969 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473699+0000", 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4456e-05 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473705+0000", 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6630000000000004e-06 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473767+0000", 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1607999999999997e-05 2023-10-09T04:15:12.970 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473779+0000", 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2330999999999999e-05 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.419942+0000", 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94616239800000002 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.971 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.420106+0000", 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016377699999999999 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.420277+0000", 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017136500000000001 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425713+0000", 2023-10-09T04:15:12.972 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005435601 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425748+0000", 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5747999999999997e-05 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430649+0000", 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049003709999999997 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.973 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430704+0000", 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5140999999999999e-05 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430751+0000", 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7691999999999997e-05 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.974 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430855+0000", 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103417 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80558 7.13 7:c83af463:::benchmark_data_smithi163_346082_object80557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:15:12.975 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 274.36434653200001, 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96028118200000001, 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80558 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.976 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.977 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479559+0000", 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479582+0000", 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2878999999999999e-05 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:15:12.978 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479594+0000", 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1328000000000001e-05 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479601+0000", 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3690000000000001e-06 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479661+0000", 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0053000000000003e-05 2023-10-09T04:15:12.979 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479673+0000", 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1877e-05 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431306+0000", 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95163314200000004 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.980 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431474+0000", 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016787400000000001 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431652+0000", 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017823200000000001 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434479+0000", 2023-10-09T04:15:12.981 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028264900000000001 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434839+0000", 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00036005299999999999 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434892+0000", 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3001000000000003e-05 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.982 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439712+0000", 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048203079999999997 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439752+0000", 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9496999999999998e-05 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:15:12.983 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439841+0000", 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9649999999999997e-05 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:15:12.984 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:15:12.984 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:12.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:15:13.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:12 smithi181 ceph-mon[196979]: pgmap v703: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 5779/252564 objects degraded (2.288%); 2346/252564 objects misplaced (0.929%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:15:13.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:12 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T04:15:13.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:12 smithi181 ceph-mon[196979]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T04:15:13.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:12 smithi181 ceph-mon[196979]: failed to encode map e260 with expected crc 2023-10-09T04:15:13.013 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/252564 objects degraded (2.288%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:13.015 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:13.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:13.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:13.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:13.265 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:13.336 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 86262 86246 64.9055 42.0625 0.0155688 0.0154013 2023-10-09T04:15:13.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:13.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:13.911 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:13.950 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:14.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:14.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:14.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:14.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:15:14.222 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:14.338 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 86972 86956 64.6597 44.375 0.0193823 0.0154593 2023-10-09T04:15:14.370 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:14.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:14.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[213841]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T04:15:14.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[213841]: failed to encode map e261 with expected crc 2023-10-09T04:15:14.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[213841]: failed to encode map e261 with expected crc 2023-10-09T04:15:14.417 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[213841]: pgmap v706: 99 pgs: 10 unknown, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 584 op/s; 5779/252564 objects degraded (2.288%); 2346/252564 objects misplaced (0.929%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T04:15:14.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:14.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:14.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:15:14.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:14.617 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:14.635 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:14.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[221067]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T04:15:14.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[221067]: failed to encode map e261 with expected crc 2023-10-09T04:15:14.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[221067]: failed to encode map e261 with expected crc 2023-10-09T04:15:14.723 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:14 smithi163 ceph-mon[221067]: pgmap v706: 99 pgs: 10 unknown, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 584 op/s; 5779/252564 objects degraded (2.288%); 2346/252564 objects misplaced (0.929%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T04:15:14.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:14 smithi181 ceph-mon[196979]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T04:15:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:14 smithi181 ceph-mon[196979]: failed to encode map e261 with expected crc 2023-10-09T04:15:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:14 smithi181 ceph-mon[196979]: failed to encode map e261 with expected crc 2023-10-09T04:15:14.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:14 smithi181 ceph-mon[196979]: pgmap v706: 99 pgs: 10 unknown, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 584 op/s; 5779/252564 objects degraded (2.288%); 2346/252564 objects misplaced (0.929%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T04:15:14.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:14.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:14.952 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:15.037 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:15.340 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 87682 87666 64.4192 44.375 0.00886147 0.0155179 2023-10-09T04:15:16.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:16.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:16.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:16.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:16.332 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:16.340 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 88450 88434 64.2283 48 0.0195627 0.015562 2023-10-09T04:15:16.474 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:16.489 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:16.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:16.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:16.643 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:16.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:16.686 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:16 smithi181 ceph-mon[196979]: pgmap v707: 99 pgs: 10 unknown, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 5779/256836 objects degraded (2.250%); 2346/256836 objects misplaced (0.913%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:15:16.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:16.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:16.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:16.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:16 smithi163 ceph-mon[221067]: pgmap v707: 99 pgs: 10 unknown, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 5779/256836 objects degraded (2.250%); 2346/256836 objects misplaced (0.913%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:15:16.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:16 smithi163 ceph-mon[213841]: pgmap v707: 99 pgs: 10 unknown, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 5779/256836 objects degraded (2.250%); 2346/256836 objects misplaced (0.913%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:15:16.830 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:17.026 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:15:17.055 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:15:17.089 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:15:17.108 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:17.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:17.128 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:15:17.156 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:15:17.179 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:15:17.205 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:15:17.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:17.231 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.236 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.237 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:15:17.264 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:17.341 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 88870 88854 63.792 26.25 0.00459832 0.0155962 2023-10-09T04:15:17.586 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:15:17.586 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:15:17.586 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T04:15:17.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T04:15:17.601 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:17.601 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:17 smithi181 ceph-mon[196979]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:17 smithi181 ceph-mon[196979]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:17 smithi181 ceph-mon[196979]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T04:15:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:17 smithi181 ceph-mon[196979]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1564552316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/256836 objects degraded (2.250%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:18.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[213841]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[213841]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[213841]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T04:15:18.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[213841]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1564552316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/256836 objects degraded (2.250%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[221067]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[221067]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[221067]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[221067]: failed to encode map e262 with expected crc 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1564552316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:18.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/256836 objects degraded (2.250%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:18.341 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 89302 89286 63.3741 27 0.0355174 0.0157746 2023-10-09T04:15:18.383 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:15:18.883 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:18.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:18.953 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[221067]: pgmap v709: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 5779/262650 objects degraded (2.200%); 1768/262650 objects misplaced (0.673%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:15:18.953 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[221067]: failed to encode map e263 with expected crc 2023-10-09T04:15:18.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[221067]: failed to encode map e263 with expected crc 2023-10-09T04:15:18.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[221067]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T04:15:18.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[221067]: failed to encode map e263 with expected crc 2023-10-09T04:15:18.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:18.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:18.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[213841]: pgmap v709: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 5779/262650 objects degraded (2.200%); 1768/262650 objects misplaced (0.673%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:15:18.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[213841]: failed to encode map e263 with expected crc 2023-10-09T04:15:18.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[213841]: failed to encode map e263 with expected crc 2023-10-09T04:15:18.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[213841]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T04:15:18.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:18 smithi163 ceph-mon[213841]: failed to encode map e263 with expected crc 2023-10-09T04:15:19.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:18 smithi181 ceph-mon[196979]: pgmap v709: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 5779/262650 objects degraded (2.200%); 1768/262650 objects misplaced (0.673%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:15:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:18 smithi181 ceph-mon[196979]: failed to encode map e263 with expected crc 2023-10-09T04:15:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:18 smithi181 ceph-mon[196979]: failed to encode map e263 with expected crc 2023-10-09T04:15:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:18 smithi181 ceph-mon[196979]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T04:15:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:18 smithi181 ceph-mon[196979]: failed to encode map e263 with expected crc 2023-10-09T04:15:19.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:19.089 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:19.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:19.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:19.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:19.341 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 89978 89962 63.1365 42.25 0.0269474 0.0158306 2023-10-09T04:15:19.355 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:19.364 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:19.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:19.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:19.639 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:19.682 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:19.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:19.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:19.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:19.871 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:19.928 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:19.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:20.035 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:20.065 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:20.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:20.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:20.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:20.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:20.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[213841]: pgmap v711: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 5779/265476 objects degraded (2.177%); 1768/265476 objects misplaced (0.666%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T04:15:20.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1011323884' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:20.311 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:15:20.329 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:20.341 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0137 2023-10-09T04:15:20.341 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 90581 2023-10-09T04:15:20.341 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 62.8939 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 40.8134 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 206.688 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1006 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 653.033 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3307 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0158954 2023-10-09T04:15:20.342 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0698007 2023-10-09T04:15:20.343 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 6.1654 2023-10-09T04:15:20.343 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.0010792 2023-10-09T04:15:20.347 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:15:20.367 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:20.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:20.482 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:20.483 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:20 smithi181 ceph-mon[196979]: pgmap v711: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 5779/265476 objects degraded (2.177%); 1768/265476 objects misplaced (0.666%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T04:15:20.483 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1011323884' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:20.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:20.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:20.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:20.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:20.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[221067]: pgmap v711: 99 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 5779/265476 objects degraded (2.177%); 1768/265476 objects misplaced (0.666%); 5.9 MiB/s, 93 objects/s recovering 2023-10-09T04:15:20.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1011323884' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:20.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:20.687 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:20.744 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:20.745 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:20.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:20.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:20.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:20.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:20.964 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:15:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:15:20] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:15:20.968 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:21.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:15:21.407 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:21.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:21.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:21.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:21.564 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:21 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:21.564 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:21 smithi181 ceph-mon[196979]: failed to encode map e264 with expected crc 2023-10-09T04:15:21.564 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:21 smithi181 ceph-mon[196979]: failed to encode map e264 with expected crc 2023-10-09T04:15:21.564 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1011323884' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:15:21.565 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:21 smithi181 ceph-mon[196979]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T04:15:21.567 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[221067]: failed to encode map e264 with expected crc 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[221067]: failed to encode map e264 with expected crc 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1011323884' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[221067]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[213841]: failed to encode map e264 with expected crc 2023-10-09T04:15:21.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[213841]: failed to encode map e264 with expected crc 2023-10-09T04:15:21.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1011323884' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:15:21.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:21 smithi163 ceph-mon[213841]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T04:15:21.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:15:21.639 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:21.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:21.744 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:21.792 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:22.058 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:22.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:22.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:22.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:22.517 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:22.568 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:22 smithi181 ceph-mon[196979]: pgmap v713: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 49 MiB/s wr, 811 op/s; 5779/270261 objects degraded (2.138%); 1369/270261 objects misplaced (0.507%); 10 MiB/s, 163 objects/s recovering 2023-10-09T04:15:22.573 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:22.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:22 smithi163 ceph-mon[221067]: pgmap v713: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 49 MiB/s wr, 811 op/s; 5779/270261 objects degraded (2.138%); 1369/270261 objects misplaced (0.507%); 10 MiB/s, 163 objects/s recovering 2023-10-09T04:15:22.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:22 smithi163 ceph-mon[213841]: pgmap v713: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 170 B/s rd, 49 MiB/s wr, 811 op/s; 5779/270261 objects degraded (2.138%); 1369/270261 objects misplaced (0.507%); 10 MiB/s, 163 objects/s recovering 2023-10-09T04:15:22.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:22.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:22.848 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:22.874 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:22.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:22.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:22.933 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:23.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:23.192 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:23.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:23.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:23.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:23 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/270261 objects degraded (2.138%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:15:23.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:23.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:23.445 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:23 smithi163 conmon[213817]: 2023-10-09T04:15:23.441+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e264 definitely_dead 0 2023-10-09T04:15:23.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:23.448 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:23 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/270261 objects degraded (2.138%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:15:23.448 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:23.448 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:23.498 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:23.652 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:23 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/270261 objects degraded (2.138%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:15:23.652 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:23.652 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:23.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:23.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:23.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:23.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:23.896 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:23.955 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:23.955 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:23.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:23.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:23.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:24.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:24.141 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:24.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:24.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:24.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:24.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:15:24.319 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:24.366 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:24.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:24.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:24.417 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.4. 2023-10-09T04:15:24.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:24.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:24.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:24.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4266648873' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:15:24.438 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[213841]: pgmap v714: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 147 B/s rd, 25 MiB/s wr, 421 op/s; 5779/270261 objects degraded (2.138%); 1369/270261 objects misplaced (0.507%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:15:24.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:24.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:24.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:24.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4266648873' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:15:24.439 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:24 smithi163 ceph-mon[221067]: pgmap v714: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 147 B/s rd, 25 MiB/s wr, 421 op/s; 5779/270261 objects degraded (2.138%); 1369/270261 objects misplaced (0.507%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:15:24.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:24.510 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:24 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:24 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:24 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4266648873' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:15:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:24 smithi181 ceph-mon[196979]: pgmap v714: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 147 B/s rd, 25 MiB/s wr, 421 op/s; 5779/270261 objects degraded (2.138%); 1369/270261 objects misplaced (0.507%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:15:24.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:15:25.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:25.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:25.154 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:25.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:25.246 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:25.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:25.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:25.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:25.443 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T04:15:25.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:25.492 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:25 smithi181 ceph-mon[196979]: failed to encode map e265 with expected crc 2023-10-09T04:15:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4266648873' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:15:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:25 smithi181 ceph-mon[196979]: osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:15:25.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:25 smithi181 ceph-mon[196979]: failed to encode map e265 with expected crc 2023-10-09T04:15:25.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[221067]: failed to encode map e265 with expected crc 2023-10-09T04:15:25.642 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4266648873' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:15:25.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[221067]: osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:15:25.643 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[221067]: failed to encode map e265 with expected crc 2023-10-09T04:15:25.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[213841]: failed to encode map e265 with expected crc 2023-10-09T04:15:25.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4266648873' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:15:25.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[213841]: osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:15:25.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:25 smithi163 ceph-mon[213841]: failed to encode map e265 with expected crc 2023-10-09T04:15:25.674 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:25.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:25.718 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:25.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:25.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:25.830 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:15:25.850 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:15:25.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:25.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:25.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:25.874 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:15:25.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:15:25.921 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:15:25.948 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:15:25.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:25.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:15:25.992 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:15:25.993 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.993 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.993 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.993 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.994 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.994 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.994 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.994 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:15:25.994 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:15:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.627 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: pgmap v717: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 78 op/s; 5779/259386 objects degraded (2.228%); 1369/259386 objects misplaced (0.528%) 2023-10-09T04:15:26.628 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4156534877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:15:26.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: pgmap v717: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 78 op/s; 5779/259386 objects degraded (2.228%); 1369/259386 objects misplaced (0.528%) 2023-10-09T04:15:26.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4156534877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:26.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: failed to encode map e266 with expected crc 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: pgmap v717: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 78 op/s; 5779/259386 objects degraded (2.228%); 1369/259386 objects misplaced (0.528%) 2023-10-09T04:15:26.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4156534877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:26.948 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:93827 8.0 8:06641cc4:::benchmark_data_smithi163_404719_object3243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:15:26.942622+0000", 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0040999280000000001, 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041380150000000001, 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:26.949 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93827 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942622+0000", 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942622+0000", 2023-10-09T04:15:26.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942620+0000", 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942628+0000", 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3119999999999996e-06 2023-10-09T04:15:26.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942640+0000", 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.222e-05 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942647+0000", 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.861e-06 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944692+0000", 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002044779 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944820+0000", 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000128515 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944927+0000", 2023-10-09T04:15:26.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010680599999999999 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:93830 8.9 8:944a1bd5:::benchmark_data_smithi163_404719_object3246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:15:26.942662+0000", 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0040594200000000002, 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041795540000000003, 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:26.954 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93830 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942662+0000", 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942662+0000", 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942662+0000", 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942665+0000", 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8499999999999998e-06 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942672+0000", 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7519999999999999e-06 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942677+0000", 2023-10-09T04:15:26.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3669999999999996e-06 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942720+0000", 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3047999999999998e-05 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942869+0000", 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014947699999999999 2023-10-09T04:15:26.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.943004+0000", 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013455100000000001 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.946553+0000", 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035484890000000002 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:26.959 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:93831 8.0 8.41e8a630 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:15:26.942993+0000", 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0037286530000000002, 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039242369999999997, 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:26.960 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93831 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942993+0000", 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942993+0000", 2023-10-09T04:15:26.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942993+0000", 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.942997+0000", 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0949999999999998e-06 2023-10-09T04:15:26.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.943007+0000", 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0703999999999999e-05 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.943013+0000", 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2979999999999999e-06 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:26.963 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:93835 8.0 8.bb708620 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:15:26.944755+0000", 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0019672380000000001, 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022145210000000001, 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:26.964 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 93835 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944755+0000", 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944755+0000", 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:26.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944754+0000", 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944759+0000", 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7019999999999998e-06 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944775+0000", 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5977e-05 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:15:26.944782+0000", 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9600000000000003e-06 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.967 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:26.968 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:15:26.968 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 4 2023-10-09T04:15:26.968 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:15:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:15:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:15:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: 8.2 starting backfill to osd.4 from (0'0,248'2905] MIN to 264'5963 2023-10-09T04:15:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: 8.5 starting backfill to osd.4 from (0'0,248'2867] MIN to 264'5828 2023-10-09T04:15:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5779/259386 objects degraded (2.228%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: 8.2 starting backfill to osd.4 from (0'0,248'2905] MIN to 264'5963 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: 8.5 starting backfill to osd.4 from (0'0,248'2867] MIN to 264'5828 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5779/259386 objects degraded (2.228%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:15:27.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:27.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:27.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:15:27.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:15:27.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: 8.2 starting backfill to osd.4 from (0'0,248'2905] MIN to 264'5963 2023-10-09T04:15:27.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: 8.5 starting backfill to osd.4 from (0'0,248'2867] MIN to 264'5828 2023-10-09T04:15:27.709 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: failed to encode map e267 with expected crc 2023-10-09T04:15:27.710 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5779/259386 objects degraded (2.228%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:15:27.716 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:27.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:15:27.791 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:28.005 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:28.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:28.115 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:28.174 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:28.261 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:28.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:28.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:28.319 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:28.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:28.441 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:28 smithi181 ceph-mon[196979]: pgmap v719: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 76 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5823/255468 objects degraded (2.279%); 2794/255468 objects misplaced (1.094%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:15:28.449 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:28.583 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:28.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:28 smithi163 ceph-mon[221067]: pgmap v719: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 76 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5823/255468 objects degraded (2.279%); 2794/255468 objects misplaced (1.094%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:15:28.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:28 smithi163 ceph-mon[213841]: pgmap v719: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 76 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5823/255468 objects degraded (2.279%); 2794/255468 objects misplaced (1.094%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:15:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:28.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:28.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:28.742 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:29.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:29.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:29.141 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:29.168 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:29.211 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:29.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:29.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:29.370 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:29.492 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:15:29.510 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:29.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:29.619 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:29.691 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:29.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3007689479' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:29.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:29.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3007689479' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:29.703 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:29.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3007689479' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:29.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:29.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:29.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:29.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:15:29.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:29.971 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:30.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:30.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:30.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:30.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:30.233 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:30.314 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:15:30.314 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:15:30.314 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:15:30.314 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:15:30.314 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:15:30.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:30.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:30.444 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:15:30.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:15:30.444 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T04:15:30.444 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:15:30.444 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:15:30.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T04:15:30.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[213841]: pgmap v721: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 76 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5881/253500 objects degraded (2.320%); 2794/253500 objects misplaced (1.102%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:15:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:15:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.496 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:30 smithi163 ceph-mon[221067]: pgmap v721: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 76 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5881/253500 objects degraded (2.320%); 2794/253500 objects misplaced (1.102%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:15:30.524 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:30.592 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: failed to encode map e268 with expected crc 2023-10-09T04:15:30.595 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:30 smithi181 ceph-mon[196979]: pgmap v721: 99 pgs: 18 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 76 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 5881/253500 objects degraded (2.320%); 2794/253500 objects misplaced (1.102%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:15:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:30.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:30.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:30.727 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:30.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:15:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:15:30] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:15:31.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:31.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:31.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:31.194 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:31.241 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:15:31.268 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:31.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:31.303 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:31.369 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:31.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:31.433 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:31.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:31.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:31.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:31.641 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:31.696 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:31.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:31.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:31 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:15:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:31.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:31.866 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:31.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:31.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:31 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:15:31.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:31 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:15:31.988 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:32.496 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:32.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:32.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:32.661 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:32 smithi181 ceph-mon[196979]: pgmap v722: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 1.16k op/s; 5881/255285 objects degraded (2.304%); 13219/255285 objects misplaced (5.178%); 35 MiB/s, 1 keys/s, 562 objects/s recovering 2023-10-09T04:15:32.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/255285 objects degraded (2.304%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:32 smithi163 ceph-mon[221067]: pgmap v722: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 1.16k op/s; 5881/255285 objects degraded (2.304%); 13219/255285 objects misplaced (5.178%); 35 MiB/s, 1 keys/s, 562 objects/s recovering 2023-10-09T04:15:32.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/255285 objects degraded (2.304%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:32 smithi163 ceph-mon[213841]: pgmap v722: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 1.16k op/s; 5881/255285 objects degraded (2.304%); 13219/255285 objects misplaced (5.178%); 35 MiB/s, 1 keys/s, 562 objects/s recovering 2023-10-09T04:15:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/255285 objects degraded (2.304%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:33.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118877 6.a 6:5646758f:::benchmark_data_smithi163_287034_object10614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 530.81911652700001, 2023-10-09T04:15:33.833 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24332414399999999, 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118877 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.834 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010796+0000", 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010794+0000", 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.835 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010802+0000", 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3140000000000004e-06 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010818+0000", 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5945000000000001e-05 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.836 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010825+0000", 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1659999999999997e-06 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.010868+0000", 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2886999999999998e-05 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148662+0000", 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.137794378 2023-10-09T04:15:33.837 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.148810+0000", 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000147527 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.152530+0000", 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003719703 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.838 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.153038+0000", 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050862599999999998 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254050+0000", 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.101012064 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254090+0000", 2023-10-09T04:15:33.839 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9533999999999998e-05 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:43.254120+0000", 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9903000000000002e-05 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151856 6.d 6:ba33a9cc:::benchmark_data_smithi163_287034_object43593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:15:33.840 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 486.20294782500002, 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23517902700000001, 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151856 2023-10-09T04:15:33.841 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.842 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626964+0000", 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626966+0000", 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.877e-06 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.843 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626972+0000", 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7089999999999999e-06 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.626975+0000", 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1319999999999998e-06 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627008+0000", 2023-10-09T04:15:33.844 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3343999999999998e-05 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627097+0000", 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8706000000000001e-05 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.627179+0000", 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2310000000000003e-05 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.845 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.631260+0000", 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040813139999999999 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794861+0000", 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16360079 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862048+0000", 2023-10-09T04:15:33.846 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067187075999999998 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862105+0000", 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6997000000000003e-05 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862143+0000", 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8050999999999999e-05 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.847 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:151866 6.d 6:b4107b04:::benchmark_data_smithi163_287034_object43603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 486.19299103399999, 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22535977800000001, 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:33.848 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 151866 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636921+0000", 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.849 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636920+0000", 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636926+0000", 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.850 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636939+0000", 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2904e-05 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636944+0000", 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8879999999999996e-06 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.851 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.636983+0000", 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8402000000000003e-05 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637122+0000", 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000139307 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.637246+0000", 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012406599999999999 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.852 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.641603+0000", 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043568410000000002 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.794991+0000", 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15338833599999999 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.853 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862183+0000", 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.067192247999999996 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862248+0000", 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4317000000000001e-05 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:27.862281+0000", 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3144e-05 2023-10-09T04:15:33.854 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169261 6.d 6:b3c8e3b4:::benchmark_data_smithi163_287034_object60998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.81907029299998, 2023-10-09T04:15:33.855 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22831853499999999, 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169261 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:15:33.856 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010842+0000", 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010841+0000", 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.857 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.858 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010846+0000", 2023-10-09T04:15:33.858 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1510000000000003e-06 2023-10-09T04:15:33.858 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.858 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.858 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.858 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010858+0000", 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1446e-05 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010864+0000", 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3620000000000002e-06 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.010914+0000", 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9552000000000003e-05 2023-10-09T04:15:33.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011016+0000", 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102262 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.011120+0000", 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104622 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020057+0000", 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089365069999999998 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179106+0000", 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15904936 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239056+0000", 2023-10-09T04:15:33.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.059950160000000002 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239115+0000", 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8446000000000003e-05 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.239160+0000", 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5503999999999997e-05 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.862 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:169262 6.d 6:b9ed22f3:::benchmark_data_smithi163_287034_object60999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.81405339499997, 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23726620200000001, 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:33.863 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 169262 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015859+0000", 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015858+0000", 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015863+0000", 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.87e-06 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015871+0000", 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7810000000000005e-06 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015874+0000", 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6440000000000002e-06 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.015992+0000", 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117414 2023-10-09T04:15:33.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016078+0000", 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6527999999999996e-05 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.016159+0000", 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0648000000000004e-05 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.020242+0000", 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040829209999999998 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.179222+0000", 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15898009699999999 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253062+0000", 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.073840684000000004 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253100+0000", 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7886000000000003e-05 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:07:51.253125+0000", 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4675999999999999e-05 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.869 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:205752 6.a 6:5755c8f4:::benchmark_data_smithi163_287034_object97489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 407.95768153300003, 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22375711200000001, 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:33.870 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 205752 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872231+0000", 2023-10-09T04:15:33.871 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872230+0000", 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872233+0000", 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8949999999999998e-06 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.872 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872238+0000", 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2449999999999998e-06 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872241+0000", 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8600000000000001e-06 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.873 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872286+0000", 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5157999999999997e-05 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872387+0000", 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010013699999999999 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.872470+0000", 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3599999999999999e-05 2023-10-09T04:15:33.874 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881422+0000", 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089518930000000007 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:45.881460+0000", 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7812000000000003e-05 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.875 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095901+0000", 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.214440778 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095945+0000", 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4332000000000002e-05 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:08:46.095988+0000", 2023-10-09T04:15:33.876 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2778000000000003e-05 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36612 7.0 7:006250d9:::benchmark_data_smithi163_346082_object36611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 345.53989809699999, 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97418880500000005, 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.877 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36612 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:15:33.878 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290012+0000", 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290059+0000", 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6390000000000001e-05 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.879 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290079+0000", 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0370999999999999e-05 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290088+0000", 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.183e-06 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.880 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291399+0000", 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013106369999999999 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291468+0000", 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9116999999999996e-05 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264113+0000", 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97264483000000002 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.881 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264203+0000", 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0143999999999996e-05 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:50550 7.0 7:02154344:::benchmark_data_smithi163_346082_object50549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 327.55097128699998, 2023-10-09T04:15:33.882 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25363291799999998, 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 50550 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.883 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278941+0000", 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278940+0000", 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:33.884 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.278990+0000", 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0781000000000001e-05 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279008+0000", 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7193999999999998e-05 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.885 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279016+0000", 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2830000000000001e-06 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279063+0000", 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7596000000000003e-05 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279201+0000", 2023-10-09T04:15:33.886 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137634 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.279310+0000", 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109231 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.280343+0000", 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001032989 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.887 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.280822+0000", 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047909999999999999 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532453+0000", 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25163097600000001 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532526+0000", 2023-10-09T04:15:33.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2867000000000006e-05 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532574+0000", 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7599000000000003e-05 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:50558 7.0 7:042b1ccf:::benchmark_data_smithi163_346082_object50557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-10-09T04:15:33.889 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 327.54533249600001, 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24840627900000001, 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 50558 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.890 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284580+0000", 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284578+0000", 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:33.891 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284632+0000", 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4066e-05 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284645+0000", 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2251e-05 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284650+0000", 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2100000000000001e-06 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284690+0000", 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9845000000000002e-05 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284837+0000", 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014754199999999999 2023-10-09T04:15:33.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.284947+0000", 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109949 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.285939+0000", 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00099183299999999991 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.894 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.286366+0000", 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042700700000000003 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532907+0000", 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24654063800000001 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532954+0000", 2023-10-09T04:15:33.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6845999999999998e-05 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:06.532986+0000", 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2450000000000003e-05 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80455 7.0 7:03584563:::benchmark_data_smithi163_346082_object80454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:15:33.896 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 296.69436822599999, 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.31383416800000002, 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80455 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135544+0000", 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135542+0000", 2023-10-09T04:15:33.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135577+0000", 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5281000000000003e-05 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135599+0000", 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2073e-05 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135607+0000", 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.174e-06 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135662+0000", 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4206999999999998e-05 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135784+0000", 2023-10-09T04:15:33.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121916 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.135908+0000", 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124819 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.137658+0000", 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017491220000000001 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.139918+0000", 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022599949999999999 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449290+0000", 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30937226000000001 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449346+0000", 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6529e-05 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449378+0000", 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.18e-05 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.903 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80493 7.0 7:04f35a7b:::benchmark_data_smithi163_346082_object80492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 296.64891225500003, 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27444662600000003, 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80493 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181000+0000", 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.180999+0000", 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181023+0000", 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4460999999999998e-05 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181036+0000", 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.277e-05 2023-10-09T04:15:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181043+0000", 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6320000000000002e-06 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181091+0000", 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8309000000000001e-05 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181190+0000", 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9109000000000003e-05 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.181286+0000", 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5846000000000004e-05 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.182553+0000", 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001267191 2023-10-09T04:15:33.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.182801+0000", 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024814299999999999 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455353+0000", 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27255158600000001 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455406+0000", 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3074e-05 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.455447+0000", 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0441000000000001e-05 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.910 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80494 7.e 7:79ff37ea:::benchmark_data_smithi163_346082_object80493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 296.38028461599998, 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.00624114, 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80494 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.911 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449627+0000", 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449685+0000", 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8378000000000002e-05 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449697+0000", 2023-10-09T04:15:33.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2427e-05 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449703+0000", 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449751+0000", 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8476999999999997e-05 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449761+0000", 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0378e-05 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441447+0000", 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99168547399999996 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441622+0000", 2023-10-09T04:15:33.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017500600000000001 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441827+0000", 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020516999999999999 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451473+0000", 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096461649999999999 2023-10-09T04:15:33.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452805+0000", 2023-10-09T04:15:33.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013312770000000001 2023-10-09T04:15:33.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454609+0000", 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018042959999999999 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454649+0000", 2023-10-09T04:15:33.918 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0256999999999998e-05 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.455869+0000", 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001219649 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80509 7.e 7:77548b6a:::benchmark_data_smithi163_346082_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:15:33.919 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 296.36144664900002, 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0024772179999999, 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80509 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468464+0000", 2023-10-09T04:15:33.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468512+0000", 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615000000000001e-05 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468527+0000", 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4778e-05 2023-10-09T04:15:33.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468534+0000", 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2049999999999996e-06 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468595+0000", 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0865000000000001e-05 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468603+0000", 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2439999999999996e-06 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445422+0000", 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97681920499999997 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445635+0000", 2023-10-09T04:15:33.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000213382 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445836+0000", 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020043 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451507+0000", 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056710220000000004 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452890+0000", 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013838699999999999 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469631+0000", 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01674059 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469678+0000", 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6551999999999997e-05 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.470943+0000", 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00126515 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80511 7.e 7:74ae5f43:::benchmark_data_smithi163_346082_object80510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:15:33.927 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 296.359884346, 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002226042, 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80511 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470027+0000", 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470082+0000", 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5106999999999999e-05 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470093+0000", 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0738e-05 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470098+0000", 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5820000000000003e-06 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470142+0000", 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4261000000000002e-05 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470148+0000", 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0170000000000002e-06 2023-10-09T04:15:33.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.447968+0000", 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97782021900000005 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448099+0000", 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013048700000000001 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448204+0000", 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105266 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451513+0000", 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033096240000000002 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.456169+0000", 2023-10-09T04:15:33.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046552060000000003 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471004+0000", 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014834926999999999 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471062+0000", 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8873999999999999e-05 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472254+0000", 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011913939999999999 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80513 7.e 7:768b3fe4:::benchmark_data_smithi163_346082_object80512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 296.35795863599998, 2023-10-09T04:15:33.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0014613109999999, 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80513 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471952+0000", 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472006+0000", 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3981999999999999e-05 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472018+0000", 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1924e-05 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472023+0000", 2023-10-09T04:15:33.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6999999999999999e-06 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472070+0000", 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7218000000000001e-05 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472078+0000", 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8830000000000006e-06 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449809+0000", 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97773131000000002 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449917+0000", 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107543 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.450027+0000", 2023-10-09T04:15:33.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010959399999999999 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451621+0000", 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015948360000000001 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.466881+0000", 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015260029 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472336+0000", 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054546960000000002 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472393+0000", 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6829000000000001e-05 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.473415+0000", 2023-10-09T04:15:33.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001021852 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:130222 7.e 7:72845360:::benchmark_data_smithi163_346082_object36952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 225.46337577599999, 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.231189378, 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.943 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 130222 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366536+0000", 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366539+0000", 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0939999999999999e-06 2023-10-09T04:15:33.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366546+0000", 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5060000000000001e-06 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366550+0000", 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2880000000000003e-06 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.370042+0000", 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034921779999999999 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.370190+0000", 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000148338 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.370348+0000", 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015750399999999999 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.380035+0000", 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096872339999999994 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.387136+0000", 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0071007370000000002 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.596137+0000", 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20900122300000001 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.596197+0000", 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0238999999999997e-05 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.597726+0000", 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015284649999999999 2023-10-09T04:15:33.949 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:130225 7.e 7:70682ac4:::benchmark_data_smithi163_346082_object36955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:48.366579+0000", 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 225.46333362499999, 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.232392707, 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.950 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 130225 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366579+0000", 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366579+0000", 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366578+0000", 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366581+0000", 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.32e-06 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366588+0000", 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6059999999999999e-06 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.366592+0000", 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1740000000000002e-06 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.373524+0000", 2023-10-09T04:15:33.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069322630000000001 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.373652+0000", 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127868 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.373767+0000", 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011438 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.381882+0000", 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081150849999999993 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.387171+0000", 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052897389999999999 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.597799+0000", 2023-10-09T04:15:33.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.21062740799999999 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.597852+0000", 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3316999999999998e-05 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.598971+0000", 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011191230000000001 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.958 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:130230 7.e 7:795e8c02:::benchmark_data_smithi163_346082_object36960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 225.457530115, 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.22737015799999999, 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:15:33.959 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 130230 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372382+0000", 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372384+0000", 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4710000000000001e-06 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372396+0000", 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1887e-05 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.372400+0000", 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1729999999999997e-06 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.376814+0000", 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004414187 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.376922+0000", 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010727500000000001 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.377006+0000", 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4342e-05 2023-10-09T04:15:33.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.381892+0000", 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004886189 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.387197+0000", 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053050759999999997 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.599035+0000", 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.211838046 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.599075+0000", 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9289000000000003e-05 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:11:48.599752+0000", 2023-10-09T04:15:33.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00067761399999999999 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:66350 8.9 8:90950650:::benchmark_data_smithi163_404719_object66349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:44.690816+0000", 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 49.139095986999997, 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28791419600000001, 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.966 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 66350 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690816+0000", 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690816+0000", 2023-10-09T04:15:33.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690815+0000", 2023-10-09T04:15:33.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:33.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690888+0000", 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2810000000000003e-05 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690899+0000", 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1848000000000001e-05 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690906+0000", 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1979999999999998e-06 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.690950+0000", 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4799999999999998e-05 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.691097+0000", 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014667200000000001 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.691248+0000", 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015087600000000001 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.695818+0000", 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004569699 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.697294+0000", 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014759840000000001 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.978658+0000", 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28136455700000002 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.978697+0000", 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9067999999999999e-05 2023-10-09T04:15:33.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.978730+0000", 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3074999999999998e-05 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:66352 8.9 8:9d93a67d:::benchmark_data_smithi163_404719_object66351:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:44.692747+0000", 2023-10-09T04:15:33.974 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 49.137165318000001, 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.29346293499999998, 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 66352 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:33.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.692747+0000", 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.692747+0000", 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.692746+0000", 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.692808+0000", 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1947999999999997e-05 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.692817+0000", 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7849999999999997e-06 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:33.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.692822+0000", 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9320000000000004e-06 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.693580+0000", 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00075849800000000003 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.693695+0000", 2023-10-09T04:15:33.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114916 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.693815+0000", 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119739 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.696023+0000", 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022084769999999999 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.697667+0000", 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0016437069999999999 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.986110+0000", 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28844303700000001 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.986168+0000", 2023-10-09T04:15:33.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7432000000000002e-05 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:44.986210+0000", 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2154999999999999e-05 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:33.981 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:15:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:33.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:33.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:34.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:34.156 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:34.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:34.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:34.413 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:34.428 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:34.469 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:34.562 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:15:34.562 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:15:34.562 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:15:34.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:15:34.635 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:34.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:34.735 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:15:34.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:34 smithi181 ceph-mon[196979]: pgmap v723: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23 MiB/s wr, 894 op/s; 5881/255285 objects degraded (2.304%); 13219/255285 objects misplaced (5.178%); 27 MiB/s, 1 keys/s, 434 objects/s recovering 2023-10-09T04:15:34.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:34.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:34.784 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:15:34.792 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:34.823 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:15:34.863 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:15:34.882 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:34.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:34.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:34.895 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:15:34.961 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:15:34.995 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:15:34.995 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:35.035 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.056 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.057 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:15:35.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:34 smithi163 ceph-mon[221067]: pgmap v723: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23 MiB/s wr, 894 op/s; 5881/255285 objects degraded (2.304%); 13219/255285 objects misplaced (5.178%); 27 MiB/s, 1 keys/s, 434 objects/s recovering 2023-10-09T04:15:35.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:35.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:35.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:34 smithi163 ceph-mon[213841]: pgmap v723: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23 MiB/s wr, 894 op/s; 5881/255285 objects degraded (2.304%); 13219/255285 objects misplaced (5.178%); 27 MiB/s, 1 keys/s, 434 objects/s recovering 2023-10-09T04:15:35.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:15:35.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:35.065 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:35.173 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:35.718 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:35 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1672210142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:36.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1672210142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:36.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:35 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1672210142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:36.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:36.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:36.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:36.282 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:36.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:36.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:36.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:36.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:36 smithi181 ceph-mon[196979]: pgmap v724: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 20 MiB/s wr, 814 op/s; 5881/250398 objects degraded (2.349%); 12991/250398 objects misplaced (5.188%); 22 MiB/s, 345 objects/s recovering 2023-10-09T04:15:36.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:37.046 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:37.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:36 smithi163 ceph-mon[213841]: pgmap v724: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 20 MiB/s wr, 814 op/s; 5881/250398 objects degraded (2.349%); 12991/250398 objects misplaced (5.188%); 22 MiB/s, 345 objects/s recovering 2023-10-09T04:15:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:36 smithi163 ceph-mon[221067]: pgmap v724: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 20 MiB/s wr, 814 op/s; 5881/250398 objects degraded (2.349%); 12991/250398 objects misplaced (5.188%); 22 MiB/s, 345 objects/s recovering 2023-10-09T04:15:37.156 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:37.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:37.206 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:37.206 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:37.259 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:37.312 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:37.503 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:37.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:37.611 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:37.653 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:37.724 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/250398 objects degraded (2.349%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:37.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:37.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:37.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:37.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/250398 objects degraded (2.349%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:37.838 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:37.917 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:37.974 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/250398 objects degraded (2.349%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:38.044 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:38.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:38.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:38.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:15:38.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:38.230 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:38.368 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:38.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:38.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:38.547 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:38.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:38.690 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:38.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:38.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:38.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:38 smithi181 ceph-mon[196979]: pgmap v725: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 949 op/s; 5881/244347 objects degraded (2.407%); 12538/244347 objects misplaced (5.131%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:15:38.735 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3517867299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:15:38.796 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:38 smithi163 ceph-mon[213841]: pgmap v725: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 949 op/s; 5881/244347 objects degraded (2.407%); 12538/244347 objects misplaced (5.131%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:15:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3517867299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:15:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:38 smithi163 ceph-mon[221067]: pgmap v725: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 949 op/s; 5881/244347 objects degraded (2.407%); 12538/244347 objects misplaced (5.131%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:15:38.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3517867299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:15:38.835 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:38.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:38.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:38.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:39.057 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:39.120 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:39.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:39.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:39.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:39.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:39.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:15:39.402 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:39.430 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:15:39.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:39.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:39.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:39.632 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:39.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:39.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:39 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:39.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:39 smithi181 ceph-mon[196979]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.741 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3517867299' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:15:39.742 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:39 smithi181 ceph-mon[196979]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:15:39.742 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:39 smithi181 ceph-mon[196979]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.742 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:39 smithi181 ceph-mon[196979]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:39.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:39.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:39.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[213841]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3517867299' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:15:39.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[213841]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:15:39.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[213841]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.744 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[213841]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:39.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[221067]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.745 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3517867299' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:15:39.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[221067]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:15:39.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[221067]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.746 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:39 smithi163 ceph-mon[221067]: failed to encode map e269 with expected crc 2023-10-09T04:15:39.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:39.899 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:40.740 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:40.740 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:40.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:40 smithi181 ceph-mon[196979]: failed to encode map e269 with expected crc 2023-10-09T04:15:40.751 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:40 smithi181 ceph-mon[196979]: pgmap v727: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 996 op/s; 5881/240966 objects degraded (2.441%); 12538/240966 objects misplaced (5.203%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:15:40.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:40.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:40.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:40.923 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:40 smithi163 ceph-mon[213841]: failed to encode map e269 with expected crc 2023-10-09T04:15:40.923 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:40 smithi163 ceph-mon[213841]: pgmap v727: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 996 op/s; 5881/240966 objects degraded (2.441%); 12538/240966 objects misplaced (5.203%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:15:40.924 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:15:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:15:40] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-10-09T04:15:40.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:40 smithi163 ceph-mon[221067]: failed to encode map e269 with expected crc 2023-10-09T04:15:40.924 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:40 smithi163 ceph-mon[221067]: pgmap v727: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 996 op/s; 5881/240966 objects degraded (2.441%); 12538/240966 objects misplaced (5.203%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:15:40.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:40.994 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:41.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:41.063 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:15:41.063 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:15:41.063 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T04:15:41.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T04:15:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:41.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:41.114 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:41.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:41.195 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:41.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:41.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:41.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:41.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:15:41.656 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:41.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:41.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:41.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:41.992 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:42.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:42.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:42.184 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:42.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:42.259 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:42.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:42.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:42.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:42.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:42.473 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:42.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:42.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:42.618 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:42.683 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:42.762 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:42 smithi163 ceph-mon[221067]: pgmap v728: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5881/233463 objects degraded (2.519%); 11896/233463 objects misplaced (5.095%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:42.765 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/233463 objects degraded (2.519%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:42.766 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:42 smithi181 ceph-mon[196979]: pgmap v728: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5881/233463 objects degraded (2.519%); 11896/233463 objects misplaced (5.095%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:42.766 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/233463 objects degraded (2.519%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:42.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:42.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:42.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:15:42.961 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:43.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:43.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:42 smithi163 ceph-mon[213841]: pgmap v728: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5881/233463 objects degraded (2.519%); 11896/233463 objects misplaced (5.095%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:43.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/233463 objects degraded (2.519%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:43.176 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:43.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:43.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:43.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:43.498 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:43.819 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:15:43.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:43.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:43.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:43.864 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:15:43.915 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:15:43.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:43.966 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:15:43.993 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:44.000 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:15:44.054 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:15:44.093 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:15:44.137 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:15:44.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.149 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:15:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:44.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:44.247 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:15:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:15:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118953 6.c 6:3024d817:::benchmark_data_smithi163_287034_object10690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 539.70514301799994, 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96349797500000001, 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118953 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535427+0000", 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535426+0000", 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535432+0000", 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0039999999999996e-06 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535441+0000", 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0080000000000008e-06 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535448+0000", 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.737e-06 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535513+0000", 2023-10-09T04:15:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5147000000000005e-05 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.535523+0000", 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0067e-05 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.481396+0000", 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94587314700000003 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.483600+0000", 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022032530000000001 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.483670+0000", 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0501999999999997e-05 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.497950+0000", 2023-10-09T04:15:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014279490000000001 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498056+0000", 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106544 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498844+0000", 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00078815099999999996 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498889+0000", 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4614999999999999e-05 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498925+0000", 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6409000000000003e-05 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118965 6.c 6:37b70bb5:::benchmark_data_smithi163_287034_object10702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 539.65493443399998, 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91336464399999995, 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118965 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585636+0000", 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585634+0000", 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585641+0000", 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0720000000000002e-06 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585655+0000", 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3332e-05 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585661+0000", 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4740000000000002e-06 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585709+0000", 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7892999999999997e-05 2023-10-09T04:15:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585715+0000", 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4180000000000002e-06 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486249+0000", 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90053330399999998 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486335+0000", 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6322000000000005e-05 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.486411+0000", 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5829999999999997e-05 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498155+0000", 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011743709999999999 2023-10-09T04:15:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498809+0000", 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00065421399999999997 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498938+0000", 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012885899999999999 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498971+0000", 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3025999999999999e-05 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499001+0000", 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9971000000000002e-05 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118966 6.c 6:3491ce90:::benchmark_data_smithi163_287034_object10703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 539.65489571700004, 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91791875999999994, 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118966 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585675+0000", 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585674+0000", 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585677+0000", 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6410000000000002e-06 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585683+0000", 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1410000000000002e-06 2023-10-09T04:15:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585687+0000", 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9369999999999997e-06 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585735+0000", 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7969000000000001e-05 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.585741+0000", 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7339999999999996e-06 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489228+0000", 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90348698199999999 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.268 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489334+0000", 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106357 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.489437+0000", 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103119 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.498838+0000", 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00940123 2023-10-09T04:15:44.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499006+0000", 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016752499999999999 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503505+0000", 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044992570000000004 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503553+0000", 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8050000000000002e-05 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503594+0000", 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0227e-05 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.271 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:118982 6.c 6:3e1a0c35:::benchmark_data_smithi163_287034_object10719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 539.63505532800002, 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90241945000000001, 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 118982 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605515+0000", 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605514+0000", 2023-10-09T04:15:44.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605521+0000", 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8539999999999999e-06 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605532+0000", 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.114e-05 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605537+0000", 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.118e-06 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605601+0000", 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4351999999999996e-05 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.605612+0000", 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0536e-05 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492104+0000", 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88649252599999995 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492184+0000", 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9084000000000004e-05 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.492265+0000", 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1630000000000003e-05 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499011+0000", 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067454790000000004 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503639+0000", 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046281070000000002 2023-10-09T04:15:44.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507860+0000", 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042216689999999999 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507899+0000", 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.841e-05 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507935+0000", 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5747000000000002e-05 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.44666.0:119013 6.c 6:3a6a5865:::benchmark_data_smithi163_287034_object10750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 539.49200764800003, 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75951261400000003, 2023-10-09T04:15:44.279 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.44666", 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/3911853756", 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 119013 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748563+0000", 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748561+0000", 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748576+0000", 2023-10-09T04:15:44.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4914e-05 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748593+0000", 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6529999999999999e-05 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748600+0000", 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6789999999999999e-06 2023-10-09T04:15:44.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748669+0000", 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9541000000000004e-05 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:44.748682+0000", 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3251000000000001e-05 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494591+0000", 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.74590919700000002 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494688+0000", 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6761000000000004e-05 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.494788+0000", 2023-10-09T04:15:44.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9431000000000004e-05 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.499015+0000", 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042273709999999997 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.503667+0000", 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046520449999999996 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.507982+0000", 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043144380000000003 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.508033+0000", 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1585000000000002e-05 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:06:45.508075+0000", 2023-10-09T04:15:44.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2299999999999998e-05 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:70984 8.5 8:a6ee6452:::benchmark_data_smithi163_404719_object70983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.698687+0000", 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.541883884999997, 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.04172793, 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.287 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 70984 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698687+0000", 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698687+0000", 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698685+0000", 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698722+0000", 2023-10-09T04:15:44.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7101999999999998e-05 2023-10-09T04:15:44.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698736+0000", 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4136e-05 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698744+0000", 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8310000000000001e-06 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698839+0000", 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4968000000000003e-05 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698852+0000", 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3225e-05 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.714871+0000", 2023-10-09T04:15:44.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0160188429999999 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.714980+0000", 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109119 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.715089+0000", 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010936499999999999 2023-10-09T04:15:44.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722342+0000", 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0072527260000000001 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740274+0000", 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017932251999999999 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740329+0000", 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4815000000000003e-05 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740389+0000", 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9481999999999997e-05 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740415+0000", 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5911e-05 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:70992 8.5 8:ad8cc80d:::benchmark_data_smithi163_404719_object70991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.709201+0000", 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.531369108, 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.031350838, 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 70992 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709201+0000", 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709201+0000", 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709200+0000", 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709245+0000", 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5425e-05 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709259+0000", 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3689999999999999e-05 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709264+0000", 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0710000000000002e-06 2023-10-09T04:15:44.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709315+0000", 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1563000000000001e-05 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709322+0000", 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1890000000000004e-06 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.716865+0000", 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.007543503 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.717004+0000", 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013937699999999999 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.717178+0000", 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017350699999999999 2023-10-09T04:15:44.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725150+0000", 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079721570000000005 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740456+0000", 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015305536999999999 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740498+0000", 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2141999999999997e-05 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740530+0000", 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1856000000000001e-05 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740552+0000", 2023-10-09T04:15:44.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2501999999999999e-05 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:70994 8.5 8:ac3f1792:::benchmark_data_smithi163_404719_object70993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.711696+0000", 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.528874188000003, 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.029056387, 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.304 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 70994 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711696+0000", 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711696+0000", 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711695+0000", 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711742+0000", 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7809999999999998e-05 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711757+0000", 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4721e-05 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711763+0000", 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7080000000000002e-06 2023-10-09T04:15:44.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711829+0000", 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.567e-05 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711840+0000", 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1269e-05 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.719011+0000", 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0071713149999999 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.719112+0000", 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101012 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.719209+0000", 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6763999999999997e-05 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725161+0000", 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059519120000000002 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740595+0000", 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015433733999999999 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740711+0000", 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011607699999999999 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740734+0000", 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3340999999999999e-05 2023-10-09T04:15:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740753+0000", 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8683999999999999e-05 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71006 8.5 8:a0595674:::benchmark_data_smithi163_404719_object71005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.312 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.721327+0000", 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.519243764000002, 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.019527788, 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71006 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721327+0000", 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721327+0000", 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721325+0000", 2023-10-09T04:15:44.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721358+0000", 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2787999999999998e-05 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721368+0000", 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.747e-06 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721373+0000", 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.972e-06 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721444+0000", 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1616e-05 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721450+0000", 2023-10-09T04:15:44.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7810000000000002e-06 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.720425+0000", 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99897466800000001 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.720564+0000", 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138923 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.720733+0000", 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016904999999999999 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726337+0000", 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005604263 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740663+0000", 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014326271999999999 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740787+0000", 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012382499999999999 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740821+0000", 2023-10-09T04:15:44.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3371999999999998e-05 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740854+0000", 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3926e-05 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71026 8.5 8:ae6fe286:::benchmark_data_smithi163_404719_object71025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.320 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.745442+0000", 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.495128020999999, 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99553465500000005, 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71026 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745442+0000", 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745442+0000", 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745441+0000", 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745496+0000", 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5269999999999998e-05 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745515+0000", 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8793000000000001e-05 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745524+0000", 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2459999999999992e-06 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745623+0000", 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8352999999999998e-05 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745638+0000", 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5029e-05 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722389+0000", 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97675139099999997 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722520+0000", 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013034200000000001 2023-10-09T04:15:44.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722669+0000", 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014938199999999999 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726350+0000", 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036812239999999999 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740688+0000", 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014338138 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740917+0000", 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022848700000000001 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740945+0000", 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8660999999999999e-05 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740977+0000", 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1699000000000003e-05 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.328 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71032 8.5 8:a57fa36f:::benchmark_data_smithi163_404719_object71031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.755072+0000", 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.485498958999997, 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98602111000000003, 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71032 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755072+0000", 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755072+0000", 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755070+0000", 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755156+0000", 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5607999999999998e-05 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755176+0000", 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9936e-05 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755183+0000", 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4549999999999998e-06 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755257+0000", 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4425000000000007e-05 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755268+0000", 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0352e-05 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.723798+0000", 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96853069899999999 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.723932+0000", 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013312900000000001 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.724093+0000", 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016132000000000001 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.729917+0000", 2023-10-09T04:15:44.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0058235860000000004 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740889+0000", 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010972812 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741018+0000", 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012877900000000001 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741054+0000", 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6196999999999999e-05 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741093+0000", 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8302e-05 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71064 8.5 8:a22115a5:::benchmark_data_smithi163_404719_object71063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.795912+0000", 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.444658474999997, 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94542275099999995, 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71064 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.337 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795912+0000", 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795912+0000", 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795910+0000", 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795983+0000", 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3238999999999995e-05 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796003+0000", 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9522999999999999e-05 2023-10-09T04:15:44.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796012+0000", 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6719999999999993e-06 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796072+0000", 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9982999999999998e-05 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796081+0000", 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.611e-06 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725178+0000", 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92909712700000002 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725247+0000", 2023-10-09T04:15:44.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8887999999999995e-05 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725329+0000", 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2138999999999994e-05 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733054+0000", 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077248730000000002 2023-10-09T04:15:44.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741159+0000", 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081048920000000007 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741241+0000", 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2134999999999999e-05 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741297+0000", 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6091999999999998e-05 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741335+0000", 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7395999999999999e-05 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.344 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71093 8.5 8:aad31fa0:::benchmark_data_smithi163_404719_object71092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.832717+0000", 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.407853711999998, 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.908734229, 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71093 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832717+0000", 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832717+0000", 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832715+0000", 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832756+0000", 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0484000000000001e-05 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832777+0000", 2023-10-09T04:15:44.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1484999999999998e-05 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832785+0000", 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6380000000000004e-06 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832872+0000", 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6444000000000001e-05 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832879+0000", 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1500000000000002e-06 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726395+0000", 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89351601599999997 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726520+0000", 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125439 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726687+0000", 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166446 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733064+0000", 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0063777189999999996 2023-10-09T04:15:44.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741213+0000", 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081485700000000008 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741367+0000", 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015453900000000001 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741419+0000", 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1666999999999998e-05 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741451+0000", 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1931000000000003e-05 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71111 8.5 8:a84d3fc2:::benchmark_data_smithi163_404719_object71110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.852904+0000", 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.387665982999998, 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88869594600000001, 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71111 2023-10-09T04:15:44.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852904+0000", 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852904+0000", 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852896+0000", 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999914 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852939+0000", 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3155000000000003e-05 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852956+0000", 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6302000000000001e-05 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852962+0000", 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6309999999999997e-06 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.853035+0000", 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3224000000000002e-05 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.853047+0000", 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1993e-05 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.728640+0000", 2023-10-09T04:15:44.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87559244700000005 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.728728+0000", 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8270999999999996e-05 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.728824+0000", 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6330999999999996e-05 2023-10-09T04:15:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736127+0000", 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0073024980000000001 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741496+0000", 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053695189999999997 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741531+0000", 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4558999999999999e-05 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741571+0000", 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9604000000000002e-05 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741600+0000", 2023-10-09T04:15:44.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9832000000000002e-05 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71156 8.5 8:af4bd142:::benchmark_data_smithi163_404719_object71155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.932452+0000", 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.308118794000002, 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80928035600000003, 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.361 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71156 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932452+0000", 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932452+0000", 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932450+0000", 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932530+0000", 2023-10-09T04:15:44.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0531000000000005e-05 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932547+0000", 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.679e-05 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932554+0000", 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5010000000000001e-06 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932616+0000", 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2164999999999999e-05 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932622+0000", 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5529999999999998e-06 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.729963+0000", 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79734061300000003 2023-10-09T04:15:44.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.730088+0000", 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012476699999999999 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.730195+0000", 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010767 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736133+0000", 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059373910000000002 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741630+0000", 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054977200000000002 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741682+0000", 2023-10-09T04:15:44.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1603e-05 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741710+0000", 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7708999999999998e-05 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741732+0000", 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2283999999999999e-05 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.368 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71164 8.5 8:a61e31e7:::benchmark_data_smithi163_404719_object71163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.938922+0000", 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.301648262999997, 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80291156200000002, 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.369 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71164 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938922+0000", 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938922+0000", 2023-10-09T04:15:44.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938921+0000", 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938963+0000", 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2756999999999997e-05 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938981+0000", 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7232e-05 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938988+0000", 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5429999999999996e-06 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.939059+0000", 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0704000000000006e-05 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.939071+0000", 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1817000000000001e-05 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.731693+0000", 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79262240100000003 2023-10-09T04:15:44.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.731779+0000", 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5703000000000003e-05 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.731887+0000", 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108614 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737400+0000", 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055126380000000003 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741663+0000", 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042629709999999999 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741767+0000", 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103897 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741805+0000", 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8052000000000001e-05 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741834+0000", 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8707000000000001e-05 2023-10-09T04:15:44.376 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71188 8.5 8:a7ef5618:::benchmark_data_smithi163_404719_object71187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.981878+0000", 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.258692955000001, 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.76008435900000004, 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.377 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71188 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981878+0000", 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981878+0000", 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981876+0000", 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981907+0000", 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0701999999999998e-05 2023-10-09T04:15:44.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981923+0000", 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5843000000000001e-05 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981929+0000", 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981979+0000", 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0614e-05 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981987+0000", 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5399999999999998e-06 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733108+0000", 2023-10-09T04:15:44.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.751121025 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733221+0000", 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112824 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733319+0000", 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8542999999999995e-05 2023-10-09T04:15:44.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737429+0000", 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041098580000000001 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741863+0000", 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044344629999999996 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741896+0000", 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2789000000000001e-05 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741933+0000", 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6572999999999997e-05 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741962+0000", 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9087000000000001e-05 2023-10-09T04:15:44.384 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71212 8.5 8:a8a8c0bd:::benchmark_data_smithi163_404719_object71211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:51.023176+0000", 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.217394024999997, 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.71901980099999996, 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.385 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71212 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023176+0000", 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023176+0000", 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023175+0000", 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023241+0000", 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6810000000000006e-05 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023258+0000", 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6670000000000001e-05 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023265+0000", 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.939e-06 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023330+0000", 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4519999999999999e-05 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023340+0000", 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0241000000000001e-05 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.734849+0000", 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.71150922500000002 2023-10-09T04:15:44.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.734932+0000", 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2621999999999996e-05 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.735027+0000", 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5351000000000002e-05 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.738674+0000", 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036465289999999999 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742078+0000", 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034042859999999999 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742149+0000", 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1057999999999996e-05 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742177+0000", 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7776999999999998e-05 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742196+0000", 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9616e-05 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.392 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71222 8.5 8:a9184906:::benchmark_data_smithi163_404719_object71221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:51.037493+0000", 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.203077374999999, 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.70486150800000003, 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71222 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037493+0000", 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037493+0000", 2023-10-09T04:15:44.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037491+0000", 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037563+0000", 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1388000000000002e-05 2023-10-09T04:15:44.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037578+0000", 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5000999999999999e-05 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037585+0000", 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7549999999999997e-06 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037641+0000", 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6799999999999998e-05 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037650+0000", 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3629999999999994e-06 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736153+0000", 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.69850320200000005 2023-10-09T04:15:44.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736236+0000", 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2647999999999999e-05 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736354+0000", 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118429 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.739932+0000", 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003577724 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742128+0000", 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021960819999999998 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742281+0000", 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152984 2023-10-09T04:15:44.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742324+0000", 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3430999999999998e-05 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742355+0000", 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0292999999999999e-05 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.400 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71234 8.5 8:acb150b2:::benchmark_data_smithi163_404719_object71233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:51.067468+0000", 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 53.173102925000002, 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.67751995300000001, 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:15:44.401 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71234 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067468+0000", 2023-10-09T04:15:44.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067468+0000", 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067466+0000", 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067516+0000", 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9421e-05 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067528+0000", 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2018000000000001e-05 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067533+0000", 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7140000000000001e-06 2023-10-09T04:15:44.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067589+0000", 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6066000000000002e-05 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067598+0000", 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0299999999999999e-06 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737452+0000", 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.66985455400000005 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737525+0000", 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3143000000000001e-05 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737618+0000", 2023-10-09T04:15:44.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3046000000000003e-05 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740018+0000", 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002400063 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742236+0000", 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022179439999999999 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.744908+0000", 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026718050000000002 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.744946+0000", 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7642999999999999e-05 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:15:44.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.744987+0000", 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1659000000000002e-05 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:15:44.413 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:15:44.414 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:44.476 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:44.778 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:44 smithi163 ceph-mon[221067]: pgmap v729: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5881/233463 objects degraded (2.519%); 11896/233463 objects misplaced (5.095%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:44.778 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2354639946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:45.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:44 smithi163 ceph-mon[213841]: pgmap v729: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5881/233463 objects degraded (2.519%); 11896/233463 objects misplaced (5.095%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:45.054 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2354639946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:45.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:44 smithi181 ceph-mon[196979]: pgmap v729: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 5881/233463 objects degraded (2.519%); 11896/233463 objects misplaced (5.095%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:45.065 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2354639946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:45.084 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:15:45.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:45.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:45.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:45.594 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:45.630 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:45.638 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:45.648 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:45.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:45.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:45.809 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:45.929 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:45.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:46.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:46.086 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:46.111 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:45 smithi163 ceph-mon[213841]: pgmap v730: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 5881/228891 objects degraded (2.569%); 11647/228891 objects misplaced (5.088%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:46.111 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:45 smithi163 ceph-mon[221067]: pgmap v730: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 5881/228891 objects degraded (2.569%); 11647/228891 objects misplaced (5.088%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:46.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:45 smithi181 ceph-mon[196979]: pgmap v730: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 5881/228891 objects degraded (2.569%); 11647/228891 objects misplaced (5.088%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:15:46.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:46.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:46.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:46.513 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:46.596 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:46.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:46.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:46.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:46.786 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:moved 0 / 297 (0%) 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:avg 37.125 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 6.39213 -> 6.39213 (expected baseline 5.69951) 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 25 -> 25 pgs (0.673401 -> 0.673401 * mean) 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.7 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:oload 1000 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 0.05 2023-10-09T04:15:46.866 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:15:46.867 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0242 2023-10-09T04:15:46.867 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.2419 2023-10-09T04:15:46.892 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:46.893 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:15:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:46.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:47.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:47.086 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:47.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:47.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:47.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:47.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:47.295 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:47.382 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:47.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:47.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:47.577 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:47.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:15:47.685 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:15:47.686 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:15:47.686 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:15:47.686 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:15:47.711 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/228891 objects degraded (2.569%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:47.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/228891 objects degraded (2.569%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:47.840 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/228891 objects degraded (2.569%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:48.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:48.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:48.211 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:48.244 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:48.275 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:15:48.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-09T04:15:48.439 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:48.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:48 smithi181 ceph-mon[196979]: from='client.45491 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:48.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:48 smithi181 ceph-mon[196979]: pgmap v731: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5881/223047 objects degraded (2.637%); 11216/223047 objects misplaced (5.029%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:48.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:15:48.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:48.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:48.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:48.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:48.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:48 smithi163 ceph-mon[213841]: from='client.45491 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:48.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:48 smithi163 ceph-mon[213841]: pgmap v731: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5881/223047 objects degraded (2.637%); 11216/223047 objects misplaced (5.029%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:48.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:48 smithi163 ceph-mon[221067]: from='client.45491 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:48.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:48 smithi163 ceph-mon[221067]: pgmap v731: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5881/223047 objects degraded (2.637%); 11216/223047 objects misplaced (5.029%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:48.738 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:48.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:48.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:48.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:49.058 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:49.059 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:49.275 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:49.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:49.386 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:49.471 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:49.511 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:15:49.537 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:49.537 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4149240903' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:49.538 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:49 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:49.538 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:49.538 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4149240903' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:49.538 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:49 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:49.572 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:49.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:49.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:49.716 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:49.716 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4149240903' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:49.716 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:49 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:15:49.750 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:49.844 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:49.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:49.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:49.993 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:50.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:50.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:50.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:50.185 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:50.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:50.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:15:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:15:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[221067]: pgmap v733: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5881/219741 objects degraded (2.676%); 11216/219741 objects misplaced (5.104%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:50 smithi163 ceph-mon[213841]: pgmap v733: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5881/219741 objects degraded (2.676%); 11216/219741 objects misplaced (5.104%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:50.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:50.664 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: failed to encode map e270 with expected crc 2023-10-09T04:15:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:50 smithi181 ceph-mon[196979]: pgmap v733: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5881/219741 objects degraded (2.676%); 11216/219741 objects misplaced (5.104%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:50.789 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:50.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:50.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:50.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:51.041 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:15:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:15:50] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-10-09T04:15:51.045 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:51.109 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:15:51.110 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:15:51.110 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:15:51.110 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:15:51.110 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:15:51.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:51.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:51.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:51.389 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:51.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:51.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:51.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:51.565 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:15:51.612 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:15:51.622 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/222661986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:51.629 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/222661986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:51.641 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:51.669 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:15:51.713 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:15:51.751 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:51.755 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:15:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:51.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:51.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/222661986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:15:51.799 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:15:51.847 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:15:51.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:51.885 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.887 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:15:51.908 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:51.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:15:52.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:52.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:52.115 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:52.174 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:52.268 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:52.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:52.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:52.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:52.403 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:52.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:52.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:52.455 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:15:52.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:15:52.638 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:52.726 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:52 smithi181 ceph-mon[196979]: pgmap v734: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 5881/212421 objects degraded (2.769%); 10538/212421 objects misplaced (4.961%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:52.726 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/212421 objects degraded (2.769%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:52.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:15:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:52.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:15:52.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:15:52.871 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:52.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:52 smithi163 ceph-mon[221067]: pgmap v734: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 5881/212421 objects degraded (2.769%); 10538/212421 objects misplaced (4.961%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:52.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/212421 objects degraded (2.769%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:52.899 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:52 smithi163 ceph-mon[213841]: pgmap v734: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 5881/212421 objects degraded (2.769%); 10538/212421 objects misplaced (4.961%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:52.899 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/212421 objects degraded (2.769%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:52.901 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:53.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:53.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:53.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:15:53.617 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:53.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:15:53.748 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:53.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:53.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:15:53.861 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:53.974 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:54.106 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:54.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:54.219 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:54.317 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:54.340 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:15:54.340 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:15:54.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:54.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:moved 0 / 297 (0%) 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:avg 37.125 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 6.39213 -> 6.39213 (expected baseline 5.69951) 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 25 -> 25 pgs (0.673401 -> 0.673401 * mean) 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.7 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:oload 1000 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 3 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:15:54.583 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0228 2023-10-09T04:15:54.584 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.2278 2023-10-09T04:15:54.602 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:15:54.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:15:54.674 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:54.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:15:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:54.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:15:54.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:54 smithi163 ceph-mon[221067]: pgmap v735: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 5881/212421 objects degraded (2.769%); 10538/212421 objects misplaced (4.961%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:54.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:54 smithi163 ceph-mon[213841]: pgmap v735: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 5881/212421 objects degraded (2.769%); 10538/212421 objects misplaced (4.961%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:54.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:15:54.889 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:55.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:54 smithi181 ceph-mon[196979]: pgmap v735: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 5881/212421 objects degraded (2.769%); 10538/212421 objects misplaced (4.961%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:55.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:15:55.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:15:56.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:15:56.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:55 smithi163 ceph-mon[221067]: from='client.55312 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:56.005 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:55 smithi163 ceph-mon[213841]: from='client.55312 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:56.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:15:56.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:55 smithi181 ceph-mon[196979]: from='client.55312 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:56.107 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:15:56.142 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:56.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:15:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:56.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:15:56.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:15:56.391 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:15:56.391 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T04:15:56.468 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:56.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:15:56.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:56.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:15:56.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:56.831 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:56.843 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:56.959 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:15:56.964 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:56 smithi163 ceph-mon[213841]: pgmap v736: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 5881/207975 objects degraded (2.828%); 10341/207975 objects misplaced (4.972%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:56.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:56 smithi163 ceph-mon[221067]: pgmap v736: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 5881/207975 objects degraded (2.828%); 10341/207975 objects misplaced (4.972%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:56.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:15:56.977 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:57.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:56 smithi181 ceph-mon[196979]: pgmap v736: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 5881/207975 objects degraded (2.828%); 10341/207975 objects misplaced (4.972%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:15:57.067 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:57.128 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:57.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:57.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:57.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:57.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:15:57.478 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:57.485 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:57.737 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:15:58.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2035335544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:58.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:57 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:58.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/207975 objects degraded (2.828%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:58.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2035335544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:58.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:57 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:58.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/207975 objects degraded (2.828%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2035335544' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:57 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:15:58.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/207975 objects degraded (2.828%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:15:58.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:15:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:15:58.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:15:58.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:15:58.986 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:15:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:58 smithi181 ceph-mon[196979]: pgmap v737: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5881/202488 objects degraded (2.904%); 9867/202488 objects misplaced (4.873%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:15:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:58 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:58 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:15:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:58 smithi181 ceph-mon[196979]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:15:59.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:58 smithi181 ceph-mon[196979]: failed to encode map e271 with expected crc 2023-10-09T04:15:59.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:15:58 smithi181 ceph-mon[196979]: failed to encode map e271 with expected crc 2023-10-09T04:15:59.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[213841]: pgmap v737: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5881/202488 objects degraded (2.904%); 9867/202488 objects misplaced (4.873%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:15:59.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:59.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[213841]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[213841]: failed to encode map e271 with expected crc 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[213841]: failed to encode map e271 with expected crc 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[221067]: pgmap v737: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5881/202488 objects degraded (2.904%); 9867/202488 objects misplaced (4.873%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[221067]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[221067]: failed to encode map e271 with expected crc 2023-10-09T04:15:59.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:15:58 smithi163 ceph-mon[221067]: failed to encode map e271 with expected crc 2023-10-09T04:15:59.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:00.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:00.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:00.147 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:00.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:00.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:00.322 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:00.391 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:00.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:00.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:00.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:00.667 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:00.839 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:16:00.839 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:16:00.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:00 smithi163 ceph-mon[221067]: pgmap v739: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5881/199365 objects degraded (2.950%); 9867/199365 objects misplaced (4.949%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:16:00.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:00 smithi163 ceph-mon[213841]: pgmap v739: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5881/199365 objects degraded (2.950%); 9867/199365 objects misplaced (4.949%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:16:00.876 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:16:00] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-10-09T04:16:00.982 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:16:01.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:00 smithi181 ceph-mon[196979]: pgmap v739: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5881/199365 objects degraded (2.950%); 9867/199365 objects misplaced (4.949%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:16:01.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:16:01.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:01.085 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:16:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:01.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:01.124 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:16:01.157 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:16:01.187 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:01.195 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:16:01.232 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:01.236 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:16:01.274 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:16:01.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.308 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:16:01.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:01.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:01.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:01.452 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:01.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:01.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:01.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:01.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:01.722 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:01.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:01.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:01.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:01.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:01.958 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3082438555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3082438555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:02.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3082438555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:02.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:16:03.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:02 smithi181 ceph-mon[196979]: pgmap v740: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/192567 objects degraded (3.054%); 9179/192567 objects misplaced (4.767%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/192567 objects degraded (3.054%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:03.070 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:02 smithi163 ceph-mon[221067]: pgmap v740: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/192567 objects degraded (3.054%); 9179/192567 objects misplaced (4.767%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:03.070 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/192567 objects degraded (3.054%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:03.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:02 smithi163 ceph-mon[213841]: pgmap v740: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/192567 objects degraded (3.054%); 9179/192567 objects misplaced (4.767%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:03.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/192567 objects degraded (3.054%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:03.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:03.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:03.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:16:03.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:03.305 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:03.515 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:03.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:03.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:03.662 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:03.721 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:03.741 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:03.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:03.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:03.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:03.898 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:03.955 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:03.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:03.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:04.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:04.115 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:04.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:04.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:04.239 INFO:teuthology.orchestra.run.smithi163.stdout:moved 0 / 297 (0%) 2023-10-09T04:16:04.239 INFO:teuthology.orchestra.run.smithi163.stdout:avg 37.125 2023-10-09T04:16:04.239 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 6.39213 -> 6.39213 (expected baseline 5.69951) 2023-10-09T04:16:04.239 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 25 -> 25 pgs (0.673401 -> 0.673401 * mean) 2023-10-09T04:16:04.239 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.7 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2023-10-09T04:16:04.239 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:16:04.240 INFO:teuthology.orchestra.run.smithi163.stdout:oload 1000 2023-10-09T04:16:04.240 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 1 2023-10-09T04:16:04.240 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:16:04.240 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0212 2023-10-09T04:16:04.240 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.2116 2023-10-09T04:16:04.263 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:16:04.320 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:04.357 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:04.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:04.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:04.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:04.585 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:04.668 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:04.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:04.784 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:04.854 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:04 smithi163 ceph-mon[221067]: pgmap v741: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/192567 objects degraded (3.054%); 9179/192567 objects misplaced (4.767%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:04.855 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:04.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:04 smithi163 ceph-mon[213841]: pgmap v741: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/192567 objects degraded (3.054%); 9179/192567 objects misplaced (4.767%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:04.857 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:04.860 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:04 smithi181 ceph-mon[196979]: pgmap v741: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/192567 objects degraded (3.054%); 9179/192567 objects misplaced (4.767%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:05.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:16:06.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:05 smithi163 ceph-mon[221067]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:06.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:05 smithi163 ceph-mon[213841]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:06.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:05 smithi181 ceph-mon[196979]: from='client.35619 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:06.310 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:06.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:06.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:06.456 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:06.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:06.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:06.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:06.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:06.668 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:06.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:06.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:06.835 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:16:06.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T04:16:06.862 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:06 smithi181 ceph-mon[196979]: pgmap v742: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/188121 objects degraded (3.126%); 8955/188121 objects misplaced (4.760%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:07.027 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:07.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:06 smithi163 ceph-mon[213841]: pgmap v742: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/188121 objects degraded (3.126%); 8955/188121 objects misplaced (4.760%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:07.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:06 smithi163 ceph-mon[221067]: pgmap v742: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5881/188121 objects degraded (3.126%); 8955/188121 objects misplaced (4.760%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:16:07.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:07.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:07.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:07.416 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:07.610 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:07.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:07.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:07.761 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:07.798 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:07.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:07.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:07.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/188121 objects degraded (3.126%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:07.963 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:07 smithi163 ceph-mon[213841]: pgmap v743: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5881/182313 objects degraded (3.226%); 8435/182313 objects misplaced (4.627%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:16:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/188121 objects degraded (3.126%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:07.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:07 smithi163 ceph-mon[221067]: pgmap v743: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5881/182313 objects degraded (3.226%); 8435/182313 objects misplaced (4.627%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:16:08.062 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:16:08.062 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:16:08.062 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:16:08.062 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:16:08.062 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:16:08.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:08.148 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:08.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/188121 objects degraded (3.126%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:08.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:07 smithi181 ceph-mon[196979]: pgmap v743: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5881/182313 objects degraded (3.226%); 8435/182313 objects misplaced (4.627%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:16:08.869 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:16:08.881 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:08.912 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1777457049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:08.912 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:08.915 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1777457049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:08.915 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:09.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1777457049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:09.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:09.620 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:09.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:09.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:09.802 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[221067]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[221067]: failed to encode map e272 with expected crc 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[221067]: failed to encode map e272 with expected crc 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[221067]: pgmap v745: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 5881/179211 objects degraded (3.282%); 8435/179211 objects misplaced (4.707%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:16:10.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:10.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:10.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[213841]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:16:10.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[213841]: failed to encode map e272 with expected crc 2023-10-09T04:16:10.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[213841]: failed to encode map e272 with expected crc 2023-10-09T04:16:10.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:09 smithi163 ceph-mon[213841]: pgmap v745: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 5881/179211 objects degraded (3.282%); 8435/179211 objects misplaced (4.707%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:16:10.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:16:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:09 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:09 smithi181 ceph-mon[196979]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:16:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:09 smithi181 ceph-mon[196979]: failed to encode map e272 with expected crc 2023-10-09T04:16:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:09 smithi181 ceph-mon[196979]: failed to encode map e272 with expected crc 2023-10-09T04:16:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:09 smithi181 ceph-mon[196979]: pgmap v745: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 5881/179211 objects degraded (3.282%); 8435/179211 objects misplaced (4.707%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:16:10.901 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:16:10] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-10-09T04:16:10.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:10.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:10.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:11.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:11.059 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:11.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:11.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:11.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:11.299 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:11.340 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:11.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:11.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:11.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:11.604 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:11.672 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:16:11.726 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:11.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:11.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:11.848 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4261390254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:11.848 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4261390254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:11.861 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:11.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:16:11.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:16:11.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:16:11.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:16:11.901 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:16:11.901 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:16:11.901 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:16:11.901 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.901 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.901 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.902 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.902 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.902 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.903 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.903 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:16:11.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:11.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:11.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:12.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4261390254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:12.029 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:12.110 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:12.748 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:12.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:16:12.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:12.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:12.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:13.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:12 smithi181 ceph-mon[196979]: pgmap v746: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5881/172557 objects degraded (3.408%); 7753/172557 objects misplaced (4.493%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/172557 objects degraded (3.408%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:13.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:12 smithi163 ceph-mon[213841]: pgmap v746: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5881/172557 objects degraded (3.408%); 7753/172557 objects misplaced (4.493%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:13.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/172557 objects degraded (3.408%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:13.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:12 smithi163 ceph-mon[221067]: pgmap v746: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5881/172557 objects degraded (3.408%); 7753/172557 objects misplaced (4.493%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:13.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/172557 objects degraded (3.408%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:13.106 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:13.303 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:13.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:13.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:13.499 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:14.065 INFO:teuthology.orchestra.run.smithi163.stdout:moved 14 / 297 (4.7138%) 2023-10-09T04:16:14.065 INFO:teuthology.orchestra.run.smithi163.stdout:avg 37.125 2023-10-09T04:16:14.065 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 6.39213 -> 6.46988 (expected baseline 5.69951) 2023-10-09T04:16:14.065 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 25 -> 29 pgs (0.673401 -> 0.781145 * mean) 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.7 with 45 -> 46 pgs (1.21212 -> 1.23906 * mean) 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:oload 110 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 1 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0196 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.0216 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 weight 1.0000 -> 0.8339 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 weight 1.0000 -> 0.8599 2023-10-09T04:16:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 weight 1.0000 -> 0.8940 2023-10-09T04:16:14.118 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:14.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:14.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:14.223 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:16:14.293 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:14.350 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:14 smithi163 conmon[200567]: 2023-10-09T04:16:14.050+0000 7f40871b1700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T04:16:14.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:14 smithi163 ceph-mon[213841]: pgmap v747: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5881/172557 objects degraded (3.408%); 7753/172557 objects misplaced (4.493%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:14.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:14 smithi163 ceph-mon[221067]: pgmap v747: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5881/172557 objects degraded (3.408%); 7753/172557 objects misplaced (4.493%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:14.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:14.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:14.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:15.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:14 smithi181 ceph-mon[196979]: pgmap v747: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5881/172557 objects degraded (3.408%); 7753/172557 objects misplaced (4.493%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:15.021 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:15.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:16:15.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:15.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:15.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:15.304 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T04:16:15.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T04:16:15.352 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:15.389 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:15.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:15.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:15.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:15.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:15.659 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:15.717 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:15.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:15.725 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:15 smithi163 ceph-mon[213841]: from='client.45530 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:15.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:15 smithi163 ceph-mon[221067]: from='client.45530 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:15.823 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:15.865 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:15.922 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:15.925 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:15 smithi181 ceph-mon[196979]: from='client.45530 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:15.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:15.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:16.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:16.068 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:16.077 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:16:16.077 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:16:16.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:16.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:16.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:16.320 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:16.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:16.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:16.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:16.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:16.569 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:16.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:16.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:16.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:16.817 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:16.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:16 smithi163 ceph-mon[221067]: pgmap v748: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5881/168219 objects degraded (3.496%); 7556/168219 objects misplaced (4.492%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:16 smithi163 ceph-mon[213841]: pgmap v748: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5881/168219 objects degraded (3.496%); 7556/168219 objects misplaced (4.492%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:17.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:16 smithi181 ceph-mon[196979]: pgmap v748: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5881/168219 objects degraded (3.496%); 7556/168219 objects misplaced (4.492%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:16:17.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:17.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:17.373 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:17.399 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:17.834 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:17.835 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5881/168219 objects degraded (3.496%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:17.842 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5881/168219 objects degraded (3.496%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:17.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:17.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:16:17.912 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:17.947 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:17.975 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:18.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5881/168219 objects degraded (3.496%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:18.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:18.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:18.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:18.197 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:18.262 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:18.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:18.330 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:18.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:18.462 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:18.800 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:18.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:18.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:18.952 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:18.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:18 smithi163 ceph-mon[213841]: pgmap v749: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 5881/162948 objects degraded (3.609%); 6994/162948 objects misplaced (4.292%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:16:18.966 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:18 smithi163 ceph-mon[221067]: pgmap v749: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 5881/162948 objects degraded (3.609%); 6994/162948 objects misplaced (4.292%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:16:19.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:18 smithi181 ceph-mon[196979]: pgmap v749: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 5881/162948 objects degraded (3.609%); 6994/162948 objects misplaced (4.292%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:16:19.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:19.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:19.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:19.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:19.238 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:19.344 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:19.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:19.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:19.532 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:19.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1745302145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:19.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:19.764 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:19.786 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1745302145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:19.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:19.787 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:19.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1745302145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:19.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:19.797 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:19.807 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:16:19.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:19.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:19.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:19.951 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:20.009 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:20.031 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:20.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:20.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:20.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:20.291 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:20.820 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:20.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:20.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:20.959 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:21.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:21.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:21.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: pgmap v750: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 5881/159993 objects degraded (3.676%); 6994/159993 objects misplaced (4.371%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[221067]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: pgmap v750: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 5881/159993 objects degraded (3.676%); 6994/159993 objects misplaced (4.371%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:20 smithi163 ceph-mon[213841]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.084 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:16:20] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:16:21.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:21.136 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: pgmap v750: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 92 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 5881/159993 objects degraded (3.676%); 6994/159993 objects misplaced (4.371%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:16:21.136 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:21.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:16:21.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:16:21.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:20 smithi181 ceph-mon[196979]: failed to encode map e273 with expected crc 2023-10-09T04:16:21.139 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:21.189 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:21.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:21.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:21.323 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:21.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:21.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:21.577 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:21.606 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:21.749 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:16:21.771 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:16:21.798 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:16:21.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:21.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:21.830 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:16:21.867 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:16:21.901 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:16:21.912 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:21.954 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:16:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[221067]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:16:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[221067]: failed to encode map e274 with expected crc 2023-10-09T04:16:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[221067]: failed to encode map e274 with expected crc 2023-10-09T04:16:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[221067]: failed to encode map e274 with expected crc 2023-10-09T04:16:21.969 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/599394572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:21.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[213841]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:16:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[213841]: failed to encode map e274 with expected crc 2023-10-09T04:16:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[213841]: failed to encode map e274 with expected crc 2023-10-09T04:16:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[213841]: failed to encode map e274 with expected crc 2023-10-09T04:16:21.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/599394572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:21.989 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:21.989 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:16:22.008 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.008 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.008 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.008 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.008 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.009 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.009 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.009 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:16:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:21 smithi181 ceph-mon[196979]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:16:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:21 smithi181 ceph-mon[196979]: failed to encode map e274 with expected crc 2023-10-09T04:16:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:21 smithi181 ceph-mon[196979]: failed to encode map e274 with expected crc 2023-10-09T04:16:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:21 smithi181 ceph-mon[196979]: failed to encode map e274 with expected crc 2023-10-09T04:16:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/599394572' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:22.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[221067]: pgmap v753: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 5878/153315 objects degraded (3.834%); 6428/153315 objects misplaced (4.193%); 4.0 MiB/s, 87 objects/s recovering 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[221067]: failed to encode map e275 with expected crc 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[221067]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[221067]: failed to encode map e275 with expected crc 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[213841]: pgmap v753: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 5878/153315 objects degraded (3.834%); 6428/153315 objects misplaced (4.193%); 4.0 MiB/s, 87 objects/s recovering 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[213841]: failed to encode map e275 with expected crc 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[213841]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[213841]: failed to encode map e275 with expected crc 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5878/153315 objects degraded (3.834%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:22.795 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'59323','3':'54518','5':'57529'}"}]: dispatch 2023-10-09T04:16:22.995 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:23.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:23.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5878/153315 objects degraded (3.834%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'59323','3':'54518','5':'57529'}"}]: dispatch 2023-10-09T04:16:23.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:23.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:22 smithi181 ceph-mon[196979]: pgmap v753: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 5878/153315 objects degraded (3.834%); 6428/153315 objects misplaced (4.193%); 4.0 MiB/s, 87 objects/s recovering 2023-10-09T04:16:23.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:22 smithi181 ceph-mon[196979]: failed to encode map e275 with expected crc 2023-10-09T04:16:23.155 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:22 smithi181 ceph-mon[196979]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:16:23.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:22 smithi181 ceph-mon[196979]: failed to encode map e275 with expected crc 2023-10-09T04:16:23.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5878/153315 objects degraded (3.834%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:16:23.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'59323','3':'54518','5':'57529'}"}]: dispatch 2023-10-09T04:16:23.158 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:23.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:23.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:23.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:23.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:16:23.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:moved 14 / 297 (4.7138%) 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:avg 37.125 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 6.39213 -> 6.46988 (expected baseline 5.69951) 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 25 -> 29 pgs (0.673401 -> 0.781145 * mean) 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.7 with 45 -> 46 pgs (1.21212 -> 1.23906 * mean) 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:oload 110 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 3 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:16:23.464 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0180 2023-10-09T04:16:23.465 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.0198 2023-10-09T04:16:23.465 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 weight 1.0000 -> 0.8319 2023-10-09T04:16:23.465 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5 weight 1.0000 -> 0.8778 2023-10-09T04:16:23.465 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1 weight 1.0000 -> 0.9052 2023-10-09T04:16:23.498 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:24.065 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[213841]: from='client.55354 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[213841]: 1.0 scrub starts 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'59323','3':'54518','5':'57529'}"}]': finished 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[213841]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[213841]: failed to encode map e276 with expected crc 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:24.068 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[221067]: from='client.55354 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:24.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[221067]: 1.0 scrub starts 2023-10-09T04:16:24.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'59323','3':'54518','5':'57529'}"}]': finished 2023-10-09T04:16:24.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[221067]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:16:24.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[221067]: failed to encode map e276 with expected crc 2023-10-09T04:16:24.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:24.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:24.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:24.210 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:24.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:23 smithi181 ceph-mon[196979]: from='client.55354 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:23 smithi181 ceph-mon[196979]: 1.0 scrub starts 2023-10-09T04:16:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'59323','3':'54518','5':'57529'}"}]': finished 2023-10-09T04:16:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:23 smithi181 ceph-mon[196979]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:16:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:23 smithi181 ceph-mon[196979]: failed to encode map e276 with expected crc 2023-10-09T04:16:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:24.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:24.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:24.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:24.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:24.696 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:24.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:24.763 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:24.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:24.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:24.818 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:16:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:16:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:16:24.818 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:16:24.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:24.926 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:24.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:24.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: failed to encode map e276 with expected crc 2023-10-09T04:16:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: pgmap v756: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5878/153315 objects degraded (3.834%); 6428/153315 objects misplaced (4.193%); 2.4 MiB/s, 84 objects/s recovering 2023-10-09T04:16:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:16:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: failed to encode map e276 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: pgmap v756: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5878/153315 objects degraded (3.834%); 6428/153315 objects misplaced (4.193%); 2.4 MiB/s, 84 objects/s recovering 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:24 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:25.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: failed to encode map e276 with expected crc 2023-10-09T04:16:25.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: pgmap v756: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5878/153315 objects degraded (3.834%); 6428/153315 objects misplaced (4.193%); 2.4 MiB/s, 84 objects/s recovering 2023-10-09T04:16:25.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:16:25.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.121 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:24 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.123 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:25.181 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:25.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:25.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:25.315 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:25.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:25.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:25.469 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:25.510 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:25.563 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:25.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:25.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:25.748 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:25.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:16:25.909 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:25.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:25.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: failed to encode map e277 with expected crc 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: failed to encode map e278 with expected crc 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:16:25.988 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[221067]: pgmap v759: 99 pgs: 1 remapped+peering, 3 peering, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 2933/150672 objects degraded (1.947%); 6306/150672 objects misplaced (4.185%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:16:26.001 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.001 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.001 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: failed to encode map e278 with expected crc 2023-10-09T04:16:26.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:16:26.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:25 smithi163 ceph-mon[213841]: pgmap v759: 99 pgs: 1 remapped+peering, 3 peering, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 2933/150672 objects degraded (1.947%); 6306/150672 objects misplaced (4.185%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:16:26.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:26.130 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:26.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: failed to encode map e277 with expected crc 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: failed to encode map e278 with expected crc 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:16:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:25 smithi181 ceph-mon[196979]: pgmap v759: 99 pgs: 1 remapped+peering, 3 peering, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 2933/150672 objects degraded (1.947%); 6306/150672 objects misplaced (4.185%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:16:27.014 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:27.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:27.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:27.241 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[221067]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.241 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[221067]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 275'8219 2023-10-09T04:16:27.242 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[221067]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.242 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[221067]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.242 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:16:27.246 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:27.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[213841]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[213841]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 275'8219 2023-10-09T04:16:27.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[213841]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[213841]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:26 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:16:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:26 smithi181 ceph-mon[196979]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:26 smithi181 ceph-mon[196979]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 275'8219 2023-10-09T04:16:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:26 smithi181 ceph-mon[196979]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:26 smithi181 ceph-mon[196979]: failed to encode map e278 with expected crc 2023-10-09T04:16:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:26 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:16:27.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:27.349 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:27.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:27.449 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:27.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:27.457 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:27.555 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:27.705 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:27.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:27.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:27.921 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:28.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:28.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:28.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2933/150672 objects degraded (1.947%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[221067]: pgmap v760: 99 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 11759/147381 objects misplaced (7.979%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2933/150672 objects degraded (1.947%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:28.107 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:27 smithi163 ceph-mon[213841]: pgmap v760: 99 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 11759/147381 objects misplaced (7.979%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:16:28.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:28.259 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:28.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2933/150672 objects degraded (1.947%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:28.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:27 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:27 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:27 smithi181 ceph-mon[196979]: pgmap v760: 99 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 11759/147381 objects misplaced (7.979%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:16:28.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:28.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:28.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:28.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:28.632 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:28.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:28.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:28.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:28.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:28.895 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:16:28.919 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/241460887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:28.919 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:28 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:28.920 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/241460887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:28.920 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:28 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:28.934 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:29.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/241460887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:28 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:29.493 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:16:29.493 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:16:29.494 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:16:29.494 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:16:29.969 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:29.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:30.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:30.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.104 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:29 smithi181 ceph-mon[196979]: pgmap v762: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 2782/145776 objects degraded (1.908%); 14646/145776 objects misplaced (10.047%); 3.7 MiB/s, 60 objects/s recovering 2023-10-09T04:16:30.105 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:30.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:30.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:30.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:30.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:16:30.318 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[221067]: pgmap v762: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 2782/145776 objects degraded (1.908%); 14646/145776 objects misplaced (10.047%); 3.7 MiB/s, 60 objects/s recovering 2023-10-09T04:16:30.319 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: failed to encode map e279 with expected crc 2023-10-09T04:16:30.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:29 smithi163 ceph-mon[213841]: pgmap v762: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 2782/145776 objects degraded (1.908%); 14646/145776 objects misplaced (10.047%); 3.7 MiB/s, 60 objects/s recovering 2023-10-09T04:16:30.323 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:30.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:30.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:30.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:30.831 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:30.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:30 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:16:30.863 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:16:30] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:16:31.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:31.117 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:30 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:16:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:31.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:31.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:31.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:30 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:16:31.327 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:31.347 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:31.436 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:16:31.457 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:31.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:31.481 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:16:31.494 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:16:31.495 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:16:31.521 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:16:31.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:31.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:16:31.615 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:16:31.636 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:31.670 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:16:31.712 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:16:31.749 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:16:31.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:16:32.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1130500138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:32.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:31 smithi163 ceph-mon[213841]: pgmap v763: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 2782/139338 objects degraded (1.997%); 13824/139338 objects misplaced (9.921%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:32.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1130500138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:32.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:31 smithi163 ceph-mon[221067]: pgmap v763: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 2782/139338 objects degraded (1.997%); 13824/139338 objects misplaced (9.921%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:32.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1130500138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:32.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:31 smithi181 ceph-mon[196979]: pgmap v763: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 2782/139338 objects degraded (1.997%); 13824/139338 objects misplaced (9.921%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:32.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:16:32.974 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:32.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:32.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:32 smithi163 ceph-mon[221067]: 1.8 scrub starts 2023-10-09T04:16:32.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2782/139338 objects degraded (1.997%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:32.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:32 smithi163 ceph-mon[213841]: 1.8 scrub starts 2023-10-09T04:16:32.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2782/139338 objects degraded (1.997%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:33.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:33.131 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:33.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:33.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:32 smithi181 ceph-mon[196979]: 1.8 scrub starts 2023-10-09T04:16:33.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2782/139338 objects degraded (1.997%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:33.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:33.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:33.330 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:33.403 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:33.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:33.507 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:16:33.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:33.561 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:33.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:33.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:33.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:33.934 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:34.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:33 smithi181 ceph-mon[196979]: pgmap v764: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 2782/139338 objects degraded (1.997%); 13824/139338 objects misplaced (9.921%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:16:34.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:34.309 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:33 smithi163 ceph-mon[213841]: pgmap v764: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 2782/139338 objects degraded (1.997%); 13824/139338 objects misplaced (9.921%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:16:34.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:34.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:33 smithi163 ceph-mon[221067]: pgmap v764: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 2782/139338 objects degraded (1.997%); 13824/139338 objects misplaced (9.921%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:16:34.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:34.443 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:34.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:34.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:34.594 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:34.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:34.652 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:34.712 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:34.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:34 smithi163 ceph-mon[221067]: 1.4 scrub starts 2023-10-09T04:16:34.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:34.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:34.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:34 smithi163 ceph-mon[213841]: 1.4 scrub starts 2023-10-09T04:16:34.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:34.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:34.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:35.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:35.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:35.153 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:35.238 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:35.250 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:34 smithi181 ceph-mon[196979]: 1.4 scrub starts 2023-10-09T04:16:35.250 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:35.250 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:35.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:35.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:35.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:35.474 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:35.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:16:36.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:36.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:36.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:36.240 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:16:36.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[213841]: 1.1 deep-scrub starts 2023-10-09T04:16:36.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:36.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:36.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[213841]: pgmap v765: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 2782/136038 objects degraded (2.045%); 13655/136038 objects misplaced (10.038%); 3.2 MiB/s, 52 objects/s recovering 2023-10-09T04:16:36.257 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:16:36.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[221067]: 1.1 deep-scrub starts 2023-10-09T04:16:36.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:36.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:36.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:35 smithi163 ceph-mon[221067]: pgmap v765: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 2782/136038 objects degraded (2.045%); 13655/136038 objects misplaced (10.038%); 3.2 MiB/s, 52 objects/s recovering 2023-10-09T04:16:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:35 smithi181 ceph-mon[196979]: 1.1 deep-scrub starts 2023-10-09T04:16:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:35 smithi181 ceph-mon[196979]: pgmap v765: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 2782/136038 objects degraded (2.045%); 13655/136038 objects misplaced (10.038%); 3.2 MiB/s, 52 objects/s recovering 2023-10-09T04:16:36.301 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:37.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[221067]: 1.3 scrub starts 2023-10-09T04:16:37.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[221067]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:16:37.181 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[213841]: 1.3 scrub starts 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[213841]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:37.182 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:37.183 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":765,"stamp":"2023-10-09T04:16:35.612167+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2970752952,"num_objects":45346,"num_object_clones":0,"num_object_copies":136038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2782,"num_objects_misplaced":13655,"num_objects_unfound":0,"num_objects_dirty":45346,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":136142,"num_write_kb":5799630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11291,"num_bytes_recovered":727195222,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126622,"ondisk_log_size":126622,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12471616,"kb_used_data":9372736,"kb_used_omap":126,"kb_used_meta":3098625,"kb_avail":737489600,"statfs":{"total":767960285184,"available":755189350400,"internally_reserved":0,"allocated":9597681664,"data_stored":9589756259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3172992635},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,2,4,5,6,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-319684608,"num_objects":-4878,"num_object_clones":0,"num_object_copies":-14634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-151,"num_objects_misplaced":7349,"num_objects_unfound":0,"num_objects_dirty":-4878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4878,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":522,"num_bytes_recovered":33947648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004302"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472020+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:16:25.472020+0000","last_peered":"2023-10-09T04:16:25.472020+0000","last_clean":"2023-10-09T04:16:25.472020+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:16:25.472020+0000","last_undegraded":"2023-10-09T04:16:25.472020+0000","last_fullsized":"2023-10-09T04:16:25.472020+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":234,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472472+0000","last_change":"2023-10-09T04:15:25.414174+0000","last_active":"2023-10-09T04:16:25.472472+0000","last_peered":"2023-10-09T04:16:25.472472+0000","last_clean":"2023-10-09T04:16:25.472472+0000","last_became_active":"2023-10-09T04:15:25.413807+0000","last_became_peered":"2023-10-09T04:15:25.413807+0000","last_unstale":"2023-10-09T04:16:25.472472+0000","last_undegraded":"2023-10-09T04:16:25.472472+0000","last_fullsized":"2023-10-09T04:16:25.472472+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:01.968150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":249,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472137+0000","last_change":"2023-10-09T04:15:25.414811+0000","last_active":"2023-10-09T04:16:25.472137+0000","last_peered":"2023-10-09T04:16:25.472137+0000","last_clean":"2023-10-09T04:16:25.472137+0000","last_became_active":"2023-10-09T04:15:25.414562+0000","last_became_peered":"2023-10-09T04:15:25.414562+0000","last_unstale":"2023-10-09T04:16:25.472137+0000","last_undegraded":"2023-10-09T04:16:25.472137+0000","last_fullsized":"2023-10-09T04:16:25.472137+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:10.270201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":246,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467872+0000","last_change":"2023-10-09T04:16:24.469036+0000","last_active":"2023-10-09T04:16:25.467872+0000","last_peered":"2023-10-09T04:16:25.467872+0000","last_clean":"2023-10-09T04:16:25.467872+0000","last_became_active":"2023-10-09T04:16:24.468649+0000","last_became_peered":"2023-10-09T04:16:24.468649+0000","last_unstale":"2023-10-09T04:16:25.467872+0000","last_undegraded":"2023-10-09T04:16:25.467872+0000","last_fullsized":"2023-10-09T04:16:25.467872+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:34:53.505550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:28.868024+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:16:28.868024+0000","last_peered":"2023-10-09T04:16:28.868024+0000","last_clean":"2023-10-09T04:16:28.868024+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:16:28.868024+0000","last_undegraded":"2023-10-09T04:16:28.868024+0000","last_fullsized":"2023-10-09T04:16:28.868024+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465370+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:16:25.465370+0000","last_peered":"2023-10-09T04:16:25.465370+0000","last_clean":"2023-10-09T04:16:25.465370+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:16:25.465370+0000","last_undegraded":"2023-10-09T04:16:25.465370+0000","last_fullsized":"2023-10-09T04:16:25.465370+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":235,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467658+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:16:25.467658+0000","last_peered":"2023-10-09T04:16:25.467658+0000","last_clean":"2023-10-09T04:16:25.467658+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:16:25.467658+0000","last_undegraded":"2023-10-09T04:16:25.467658+0000","last_fullsized":"2023-10-09T04:16:25.467658+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":259,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.460761+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:16:25.460761+0000","last_peered":"2023-10-09T04:16:25.460761+0000","last_clean":"2023-10-09T04:16:25.460761+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:16:25.460761+0000","last_undegraded":"2023-10-09T04:16:25.460761+0000","last_fullsized":"2023-10-09T04:16:25.460761+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":238,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465368+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:16:25.465368+0000","last_peered":"2023-10-09T04:16:25.465368+0000","last_clean":"2023-10-09T04:16:25.465368+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:16:25.465368+0000","last_undegraded":"2023-10-09T04:16:25.465368+0000","last_fullsized":"2023-10-09T04:16:25.465368+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":97,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.460826+0000","last_change":"2023-10-09T04:13:46.101209+0000","last_active":"2023-10-09T04:16:25.460826+0000","last_peered":"2023-10-09T04:16:25.460826+0000","last_clean":"2023-10-09T04:16:25.460826+0000","last_became_active":"2023-10-09T04:13:46.100828+0000","last_became_peered":"2023-10-09T04:13:46.100828+0000","last_unstale":"2023-10-09T04:16:25.460826+0000","last_undegraded":"2023-10-09T04:16:25.460826+0000","last_fullsized":"2023-10-09T04:16:25.460826+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:59.155026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":273,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498415+0000","last_change":"2023-10-09T04:15:25.545630+0000","last_active":"2023-10-09T04:16:25.498415+0000","last_peered":"2023-10-09T04:16:25.498415+0000","last_clean":"2023-10-09T04:16:25.498415+0000","last_became_active":"2023-10-09T04:15:25.410551+0000","last_became_peered":"2023-10-09T04:15:25.410551+0000","last_unstale":"2023-10-09T04:16:25.498415+0000","last_undegraded":"2023-10-09T04:16:25.498415+0000","last_fullsized":"2023-10-09T04:16:25.498415+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:15.300784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.463472+0000","last_change":"2023-10-09T04:16:24.482108+0000","last_active":"2023-10-09T04:16:25.463472+0000","last_peered":"2023-10-09T04:16:25.463472+0000","last_clean":"2023-10-09T04:16:25.463472+0000","last_became_active":"2023-10-09T04:16:24.481813+0000","last_became_peered":"2023-10-09T04:16:24.481813+0000","last_unstale":"2023-10-09T04:16:25.463472+0000","last_undegraded":"2023-10-09T04:16:25.463472+0000","last_fullsized":"2023-10-09T04:16:25.463472+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:51.848459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":33,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472393+0000","last_change":"2023-10-09T04:15:25.523555+0000","last_active":"2023-10-09T04:16:25.472393+0000","last_peered":"2023-10-09T04:16:25.472393+0000","last_clean":"2023-10-09T04:16:25.472393+0000","last_became_active":"2023-10-09T04:15:25.408994+0000","last_became_peered":"2023-10-09T04:15:25.408994+0000","last_unstale":"2023-10-09T04:16:25.472393+0000","last_undegraded":"2023-10-09T04:16:25.472393+0000","last_fullsized":"2023-10-09T04:16:25.472393+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:39.516242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472079+0000","last_change":"2023-10-09T04:15:25.476381+0000","last_active":"2023-10-09T04:16:25.472079+0000","last_peered":"2023-10-09T04:16:25.472079+0000","last_clean":"2023-10-09T04:16:25.472079+0000","last_became_active":"2023-10-09T04:15:25.475863+0000","last_became_peered":"2023-10-09T04:15:25.475863+0000","last_unstale":"2023-10-09T04:16:25.472079+0000","last_undegraded":"2023-10-09T04:16:25.472079+0000","last_fullsized":"2023-10-09T04:16:25.472079+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:46.820781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":272,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497980+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:16:25.497980+0000","last_peered":"2023-10-09T04:16:25.497980+0000","last_clean":"2023-10-09T04:16:25.497980+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:16:25.497980+0000","last_undegraded":"2023-10-09T04:16:25.497980+0000","last_fullsized":"2023-10-09T04:16:25.497980+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498905+0000","last_change":"2023-10-09T04:16:24.616676+0000","last_active":"2023-10-09T04:16:25.498905+0000","last_peered":"2023-10-09T04:16:25.498905+0000","last_clean":"2023-10-09T04:16:25.498905+0000","last_became_active":"2023-10-09T04:16:24.616320+0000","last_became_peered":"2023-10-09T04:16:24.616320+0000","last_unstale":"2023-10-09T04:16:25.498905+0000","last_undegraded":"2023-10-09T04:16:25.498905+0000","last_fullsized":"2023-10-09T04:16:25.498905+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:32.174105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":236,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.468632+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:16:25.468632+0000","last_peered":"2023-10-09T04:16:25.468632+0000","last_clean":"2023-10-09T04:16:25.468632+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:16:25.468632+0000","last_undegraded":"2023-10-09T04:16:25.468632+0000","last_fullsized":"2023-10-09T04:16:25.468632+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":237,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465390+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:16:25.465390+0000","last_peered":"2023-10-09T04:16:25.465390+0000","last_clean":"2023-10-09T04:16:25.465390+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:16:25.465390+0000","last_undegraded":"2023-10-09T04:16:25.465390+0000","last_fullsized":"2023-10-09T04:16:25.465390+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.471785+0000","last_change":"2023-10-09T04:15:25.409247+0000","last_active":"2023-10-09T04:16:25.471785+0000","last_peered":"2023-10-09T04:16:25.471785+0000","last_clean":"2023-10-09T04:16:25.471785+0000","last_became_active":"2023-10-09T04:15:25.408950+0000","last_became_peered":"2023-10-09T04:15:25.408950+0000","last_unstale":"2023-10-09T04:16:25.471785+0000","last_undegraded":"2023-10-09T04:16:25.471785+0000","last_fullsized":"2023-10-09T04:16:25.471785+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:46.455335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":236,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:28.868703+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:16:28.868703+0000","last_peered":"2023-10-09T04:16:28.868703+0000","last_clean":"2023-10-09T04:16:28.868703+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:16:28.868703+0000","last_undegraded":"2023-10-09T04:16:28.868703+0000","last_fullsized":"2023-10-09T04:16:28.868703+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":191,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467090+0000","last_change":"2023-10-09T04:16:24.625253+0000","last_active":"2023-10-09T04:16:25.467090+0000","last_peered":"2023-10-09T04:16:25.467090+0000","last_clean":"2023-10-09T04:16:25.467090+0000","last_became_active":"2023-10-09T04:16:24.617700+0000","last_became_peered":"2023-10-09T04:16:24.617700+0000","last_unstale":"2023-10-09T04:16:25.467090+0000","last_undegraded":"2023-10-09T04:16:25.467090+0000","last_fullsized":"2023-10-09T04:16:25.467090+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:29.757285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472799+0000","last_change":"2023-10-09T04:16:24.474334+0000","last_active":"2023-10-09T04:16:25.472799+0000","last_peered":"2023-10-09T04:16:25.472799+0000","last_clean":"2023-10-09T04:16:25.472799+0000","last_became_active":"2023-10-09T04:16:24.474003+0000","last_became_peered":"2023-10-09T04:16:24.474003+0000","last_unstale":"2023-10-09T04:16:25.472799+0000","last_undegraded":"2023-10-09T04:16:25.472799+0000","last_fullsized":"2023-10-09T04:16:25.472799+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:38.410877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"279'8442","reported_seq":9553,"reported_epoch":279,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:16:30.725147+0000","last_change":"2023-10-09T04:16:23.478439+0000","last_active":"2023-10-09T04:16:30.725147+0000","last_peered":"2023-10-09T04:16:30.725147+0000","last_clean":"2023-10-09T04:14:14.612673+0000","last_became_active":"2023-10-09T04:15:25.527686+0000","last_became_peered":"2023-10-09T04:15:25.527686+0000","last_unstale":"2023-10-09T04:16:30.725147+0000","last_undegraded":"2023-10-09T04:16:30.725147+0000","last_fullsized":"2023-10-09T04:15:25.403875+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8442,"log_dups_size":0,"ondisk_log_size":8442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:36.920282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2230,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8442,"num_write_kb":361152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":614},{"shards":"2,5,6","objects":2230}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":267,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498725+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:16:25.498725+0000","last_peered":"2023-10-09T04:16:25.498725+0000","last_clean":"2023-10-09T04:16:25.498725+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:16:25.498725+0000","last_undegraded":"2023-10-09T04:16:25.498725+0000","last_fullsized":"2023-10-09T04:16:25.498725+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":220,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.464728+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:16:25.464728+0000","last_peered":"2023-10-09T04:16:25.464728+0000","last_clean":"2023-10-09T04:16:25.464728+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:16:25.464728+0000","last_undegraded":"2023-10-09T04:16:25.464728+0000","last_fullsized":"2023-10-09T04:16:25.464728+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"279'8591","reported_seq":14405,"reported_epoch":279,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:16:35.314787+0000","last_change":"2023-10-09T04:15:26.504654+0000","last_active":"2023-10-09T04:16:35.314787+0000","last_peered":"2023-10-09T04:16:35.314787+0000","last_clean":"2023-10-09T04:15:24.138637+0000","last_became_active":"2023-10-09T04:15:26.489098+0000","last_became_peered":"2023-10-09T04:15:26.489098+0000","last_unstale":"2023-10-09T04:16:35.314787+0000","last_undegraded":"2023-10-09T04:16:35.314787+0000","last_fullsized":"2023-10-09T04:16:35.314787+0000","mapping_epoch":266,"log_start":"247'1746","ondisk_log_start":"247'1746","created":241,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":6845,"log_dups_size":0,"ondisk_log_size":6845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:37.061888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179240960,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2735,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8591,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2867,"num_bytes_recovered":187891712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,7,0],"avail_no_missing":["6","0","7"],"object_location_counts":[{"shards":"0,6,7","objects":2735}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":271,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497835+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:16:25.497835+0000","last_peered":"2023-10-09T04:16:25.497835+0000","last_clean":"2023-10-09T04:16:25.497835+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:16:25.497835+0000","last_undegraded":"2023-10-09T04:16:25.497835+0000","last_fullsized":"2023-10-09T04:16:25.497835+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":33,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.471871+0000","last_change":"2023-10-09T04:16:24.476123+0000","last_active":"2023-10-09T04:16:25.471871+0000","last_peered":"2023-10-09T04:16:25.471871+0000","last_clean":"2023-10-09T04:16:25.471871+0000","last_became_active":"2023-10-09T04:16:24.475818+0000","last_became_peered":"2023-10-09T04:16:24.475818+0000","last_unstale":"2023-10-09T04:16:25.471871+0000","last_undegraded":"2023-10-09T04:16:25.471871+0000","last_fullsized":"2023-10-09T04:16:25.471871+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:57.051534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"279'8447","reported_seq":8494,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:31.096170+0000","last_change":"2023-10-09T04:13:42.646173+0000","last_active":"2023-10-09T04:16:31.096170+0000","last_peered":"2023-10-09T04:16:31.096170+0000","last_clean":"2023-10-09T04:16:31.096170+0000","last_became_active":"2023-10-09T04:13:42.645934+0000","last_became_peered":"2023-10-09T04:13:42.645934+0000","last_unstale":"2023-10-09T04:16:31.096170+0000","last_undegraded":"2023-10-09T04:16:31.096170+0000","last_fullsized":"2023-10-09T04:16:31.096170+0000","mapping_epoch":241,"log_start":"247'1700","ondisk_log_start":"247'1700","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.323104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190251008,"num_objects":2903,"num_object_clones":0,"num_object_copies":8709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8447,"num_write_kb":363200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":222,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498210+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:16:25.498210+0000","last_peered":"2023-10-09T04:16:25.498210+0000","last_clean":"2023-10-09T04:16:25.498210+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:16:25.498210+0000","last_undegraded":"2023-10-09T04:16:25.498210+0000","last_fullsized":"2023-10-09T04:16:25.498210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":200,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.474401+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:16:25.474401+0000","last_peered":"2023-10-09T04:16:25.474401+0000","last_clean":"2023-10-09T04:16:25.474401+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:16:25.474401+0000","last_undegraded":"2023-10-09T04:16:25.474401+0000","last_fullsized":"2023-10-09T04:16:25.474401+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"279'8582","reported_seq":8630,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:33.803200+0000","last_change":"2023-10-09T04:13:42.847572+0000","last_active":"2023-10-09T04:16:33.803200+0000","last_peered":"2023-10-09T04:16:33.803200+0000","last_clean":"2023-10-09T04:16:33.803200+0000","last_became_active":"2023-10-09T04:13:42.847257+0000","last_became_peered":"2023-10-09T04:13:42.847257+0000","last_unstale":"2023-10-09T04:16:33.803200+0000","last_undegraded":"2023-10-09T04:16:33.803200+0000","last_fullsized":"2023-10-09T04:16:33.803200+0000","mapping_epoch":241,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8082,"log_dups_size":0,"ondisk_log_size":8082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:19.215045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183894016,"num_objects":2806,"num_object_clones":0,"num_object_copies":8418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8582,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":238,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.475273+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:16:25.475273+0000","last_peered":"2023-10-09T04:16:25.475273+0000","last_clean":"2023-10-09T04:16:25.475273+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:16:25.475273+0000","last_undegraded":"2023-10-09T04:16:25.475273+0000","last_fullsized":"2023-10-09T04:16:25.475273+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":254,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467490+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:16:25.467490+0000","last_peered":"2023-10-09T04:16:25.467490+0000","last_clean":"2023-10-09T04:16:25.467490+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:16:25.467490+0000","last_undegraded":"2023-10-09T04:16:25.467490+0000","last_fullsized":"2023-10-09T04:16:25.467490+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498219+0000","last_change":"2023-10-09T04:15:13.426686+0000","last_active":"2023-10-09T04:16:25.498219+0000","last_peered":"2023-10-09T04:16:25.498219+0000","last_clean":"2023-10-09T04:16:25.498219+0000","last_became_active":"2023-10-09T04:15:13.426360+0000","last_became_peered":"2023-10-09T04:15:13.426360+0000","last_unstale":"2023-10-09T04:16:25.498219+0000","last_undegraded":"2023-10-09T04:16:25.498219+0000","last_fullsized":"2023-10-09T04:16:25.498219+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.0","version":"279'8406","reported_seq":8452,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:30.629826+0000","last_change":"2023-10-09T04:13:42.847429+0000","last_active":"2023-10-09T04:16:30.629826+0000","last_peered":"2023-10-09T04:16:30.629826+0000","last_clean":"2023-10-09T04:16:30.629826+0000","last_became_active":"2023-10-09T04:13:42.847130+0000","last_became_peered":"2023-10-09T04:13:42.847130+0000","last_unstale":"2023-10-09T04:16:30.629826+0000","last_undegraded":"2023-10-09T04:16:30.629826+0000","last_fullsized":"2023-10-09T04:16:30.629826+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8406,"log_dups_size":0,"ondisk_log_size":8406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.587257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8406,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":211,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467337+0000","last_change":"2023-10-09T04:15:25.538809+0000","last_active":"2023-10-09T04:16:25.467337+0000","last_peered":"2023-10-09T04:16:25.467337+0000","last_clean":"2023-10-09T04:16:25.467337+0000","last_became_active":"2023-10-09T04:15:25.419652+0000","last_became_peered":"2023-10-09T04:15:25.419652+0000","last_unstale":"2023-10-09T04:16:25.467337+0000","last_undegraded":"2023-10-09T04:16:25.467337+0000","last_fullsized":"2023-10-09T04:16:25.467337+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:02.875159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472344+0000","last_change":"2023-10-09T04:15:25.427175+0000","last_active":"2023-10-09T04:16:25.472344+0000","last_peered":"2023-10-09T04:16:25.472344+0000","last_clean":"2023-10-09T04:16:25.472344+0000","last_became_active":"2023-10-09T04:15:25.426875+0000","last_became_peered":"2023-10-09T04:15:25.426875+0000","last_unstale":"2023-10-09T04:16:25.472344+0000","last_undegraded":"2023-10-09T04:16:25.472344+0000","last_fullsized":"2023-10-09T04:16:25.472344+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:46.697878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497360+0000","last_change":"2023-10-09T04:15:13.421294+0000","last_active":"2023-10-09T04:16:25.497360+0000","last_peered":"2023-10-09T04:16:25.497360+0000","last_clean":"2023-10-09T04:16:25.497360+0000","last_became_active":"2023-10-09T04:15:13.421001+0000","last_became_peered":"2023-10-09T04:15:13.421001+0000","last_unstale":"2023-10-09T04:16:25.497360+0000","last_undegraded":"2023-10-09T04:16:25.497360+0000","last_fullsized":"2023-10-09T04:16:25.497360+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1","version":"279'8693","reported_seq":8741,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:33.710464+0000","last_change":"2023-10-09T04:13:42.633796+0000","last_active":"2023-10-09T04:16:33.710464+0000","last_peered":"2023-10-09T04:16:33.710464+0000","last_clean":"2023-10-09T04:16:33.710464+0000","last_became_active":"2023-10-09T04:13:42.633490+0000","last_became_peered":"2023-10-09T04:13:42.633490+0000","last_unstale":"2023-10-09T04:16:33.710464+0000","last_undegraded":"2023-10-09T04:16:33.710464+0000","last_fullsized":"2023-10-09T04:16:33.710464+0000","mapping_epoch":241,"log_start":"247'600","ondisk_log_start":"247'600","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8093,"log_dups_size":0,"ondisk_log_size":8093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:46.137500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8693,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472164+0000","last_change":"2023-10-09T04:15:25.432206+0000","last_active":"2023-10-09T04:16:25.472164+0000","last_peered":"2023-10-09T04:16:25.472164+0000","last_clean":"2023-10-09T04:16:25.472164+0000","last_became_active":"2023-10-09T04:15:25.430422+0000","last_became_peered":"2023-10-09T04:15:25.430422+0000","last_unstale":"2023-10-09T04:16:25.472164+0000","last_undegraded":"2023-10-09T04:16:25.472164+0000","last_fullsized":"2023-10-09T04:16:25.472164+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.755818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472074+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:16:25.472074+0000","last_peered":"2023-10-09T04:16:25.472074+0000","last_clean":"2023-10-09T04:16:25.472074+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:16:25.472074+0000","last_undegraded":"2023-10-09T04:16:25.472074+0000","last_fullsized":"2023-10-09T04:16:25.472074+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"279'8630","reported_seq":14521,"reported_epoch":279,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:16:31.069374+0000","last_change":"2023-10-09T04:15:26.506798+0000","last_active":"2023-10-09T04:16:31.069374+0000","last_peered":"2023-10-09T04:16:31.069374+0000","last_clean":"2023-10-09T04:15:24.385046+0000","last_became_active":"2023-10-09T04:15:26.497446+0000","last_became_peered":"2023-10-09T04:15:26.497446+0000","last_unstale":"2023-10-09T04:16:31.069374+0000","last_undegraded":"2023-10-09T04:16:31.069374+0000","last_fullsized":"2023-10-09T04:16:31.069374+0000","mapping_epoch":266,"log_start":"247'1900","ondisk_log_start":"247'1900","created":241,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:01.043194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192937984,"num_objects":2944,"num_object_clones":0,"num_object_copies":8832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2944,"num_objects_unfound":0,"num_objects_dirty":2944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8630,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2907,"num_bytes_recovered":190513152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[0,5,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":2944}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.471651+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:16:25.471651+0000","last_peered":"2023-10-09T04:16:25.471651+0000","last_clean":"2023-10-09T04:16:25.471651+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:16:25.471651+0000","last_undegraded":"2023-10-09T04:16:25.471651+0000","last_fullsized":"2023-10-09T04:16:25.471651+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":33,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472459+0000","last_change":"2023-10-09T04:16:24.617626+0000","last_active":"2023-10-09T04:16:25.472459+0000","last_peered":"2023-10-09T04:16:25.472459+0000","last_clean":"2023-10-09T04:16:25.472459+0000","last_became_active":"2023-10-09T04:16:24.617298+0000","last_became_peered":"2023-10-09T04:16:24.617298+0000","last_unstale":"2023-10-09T04:16:25.472459+0000","last_undegraded":"2023-10-09T04:16:25.472459+0000","last_fullsized":"2023-10-09T04:16:25.472459+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:28.718470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497602+0000","last_change":"2023-10-09T04:15:13.428903+0000","last_active":"2023-10-09T04:16:25.497602+0000","last_peered":"2023-10-09T04:16:25.497602+0000","last_clean":"2023-10-09T04:16:25.497602+0000","last_became_active":"2023-10-09T04:15:13.428029+0000","last_became_peered":"2023-10-09T04:15:13.428029+0000","last_unstale":"2023-10-09T04:16:25.497602+0000","last_undegraded":"2023-10-09T04:16:25.497602+0000","last_fullsized":"2023-10-09T04:16:25.497602+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.3","version":"279'8413","reported_seq":8470,"reported_epoch":279,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:16:30.681807+0000","last_change":"2023-10-09T04:16:25.591418+0000","last_active":"2023-10-09T04:16:30.681807+0000","last_peered":"2023-10-09T04:16:30.681807+0000","last_clean":"2023-10-09T04:16:23.412068+0000","last_became_active":"2023-10-09T04:16:25.564970+0000","last_became_peered":"2023-10-09T04:16:25.564970+0000","last_unstale":"2023-10-09T04:16:30.681807+0000","last_undegraded":"2023-10-09T04:16:30.681807+0000","last_fullsized":"2023-10-09T04:16:30.681807+0000","mapping_epoch":277,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":7913,"log_dups_size":0,"ondisk_log_size":7913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:44.646528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187367424,"num_objects":2859,"num_object_clones":0,"num_object_copies":8577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2859,"num_objects_unfound":0,"num_objects_dirty":2859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8413,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":2859}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472084+0000","last_change":"2023-10-09T04:16:24.471755+0000","last_active":"2023-10-09T04:16:25.472084+0000","last_peered":"2023-10-09T04:16:25.472084+0000","last_clean":"2023-10-09T04:16:25.472084+0000","last_became_active":"2023-10-09T04:16:24.471509+0000","last_became_peered":"2023-10-09T04:16:24.471509+0000","last_unstale":"2023-10-09T04:16:25.472084+0000","last_undegraded":"2023-10-09T04:16:25.472084+0000","last_fullsized":"2023-10-09T04:16:25.472084+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:57.228210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":251,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472860+0000","last_change":"2023-10-09T04:15:25.418196+0000","last_active":"2023-10-09T04:16:25.472860+0000","last_peered":"2023-10-09T04:16:25.472860+0000","last_clean":"2023-10-09T04:16:25.472860+0000","last_became_active":"2023-10-09T04:15:25.417900+0000","last_became_peered":"2023-10-09T04:15:25.417900+0000","last_unstale":"2023-10-09T04:16:25.472860+0000","last_undegraded":"2023-10-09T04:16:25.472860+0000","last_fullsized":"2023-10-09T04:16:25.472860+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:19.256359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498565+0000","last_change":"2023-10-09T04:15:13.426845+0000","last_active":"2023-10-09T04:16:25.498565+0000","last_peered":"2023-10-09T04:16:25.498565+0000","last_clean":"2023-10-09T04:16:25.498565+0000","last_became_active":"2023-10-09T04:15:13.426510+0000","last_became_peered":"2023-10-09T04:15:13.426510+0000","last_unstale":"2023-10-09T04:16:25.498565+0000","last_undegraded":"2023-10-09T04:16:25.498565+0000","last_fullsized":"2023-10-09T04:16:25.498565+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.e","version":"279'8509","reported_seq":18170,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:31.014706+0000","last_change":"2023-10-09T04:16:21.813363+0000","last_active":"2023-10-09T04:16:31.014706+0000","last_peered":"2023-10-09T04:16:31.014706+0000","last_clean":"2023-10-09T04:16:31.014706+0000","last_became_active":"2023-10-09T04:16:21.793534+0000","last_became_peered":"2023-10-09T04:16:21.793534+0000","last_unstale":"2023-10-09T04:16:31.014706+0000","last_undegraded":"2023-10-09T04:16:31.014706+0000","last_fullsized":"2023-10-09T04:16:31.014706+0000","mapping_epoch":274,"log_start":"247'1800","ondisk_log_start":"247'1800","created":241,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:38.188955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190119936,"num_objects":2901,"num_object_clones":0,"num_object_copies":8703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":365120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4794,"num_bytes_recovered":302186496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":256,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472307+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:16:25.472307+0000","last_peered":"2023-10-09T04:16:25.472307+0000","last_clean":"2023-10-09T04:16:25.472307+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:16:25.472307+0000","last_undegraded":"2023-10-09T04:16:25.472307+0000","last_fullsized":"2023-10-09T04:16:25.472307+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":255,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465362+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:16:25.465362+0000","last_peered":"2023-10-09T04:16:25.465362+0000","last_clean":"2023-10-09T04:16:25.465362+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:16:25.465362+0000","last_undegraded":"2023-10-09T04:16:25.465362+0000","last_fullsized":"2023-10-09T04:16:25.465362+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":229,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.464909+0000","last_change":"2023-10-09T04:16:24.473165+0000","last_active":"2023-10-09T04:16:25.464909+0000","last_peered":"2023-10-09T04:16:25.464909+0000","last_clean":"2023-10-09T04:16:25.464909+0000","last_became_active":"2023-10-09T04:16:24.472923+0000","last_became_peered":"2023-10-09T04:16:24.472923+0000","last_unstale":"2023-10-09T04:16:25.464909+0000","last_undegraded":"2023-10-09T04:16:25.464909+0000","last_fullsized":"2023-10-09T04:16:25.464909+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:01.846352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":628,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498294+0000","last_change":"2023-10-09T04:15:13.421254+0000","last_active":"2023-10-09T04:16:25.498294+0000","last_peered":"2023-10-09T04:16:25.498294+0000","last_clean":"2023-10-09T04:16:25.498294+0000","last_became_active":"2023-10-09T04:15:13.420889+0000","last_became_peered":"2023-10-09T04:15:13.420889+0000","last_unstale":"2023-10-09T04:16:25.498294+0000","last_undegraded":"2023-10-09T04:16:25.498294+0000","last_fullsized":"2023-10-09T04:16:25.498294+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.8","version":"279'8414","reported_seq":8460,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:30.710361+0000","last_change":"2023-10-09T04:13:42.847477+0000","last_active":"2023-10-09T04:16:30.710361+0000","last_peered":"2023-10-09T04:16:30.710361+0000","last_clean":"2023-10-09T04:16:30.710361+0000","last_became_active":"2023-10-09T04:13:42.847180+0000","last_became_peered":"2023-10-09T04:13:42.847180+0000","last_unstale":"2023-10-09T04:16:30.710361+0000","last_undegraded":"2023-10-09T04:16:30.710361+0000","last_fullsized":"2023-10-09T04:16:30.710361+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8414,"log_dups_size":0,"ondisk_log_size":8414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:23.775147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":242,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465078+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:16:25.465078+0000","last_peered":"2023-10-09T04:16:25.465078+0000","last_clean":"2023-10-09T04:16:25.465078+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:16:25.465078+0000","last_undegraded":"2023-10-09T04:16:25.465078+0000","last_fullsized":"2023-10-09T04:16:25.465078+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472006+0000","last_change":"2023-10-09T04:15:25.435628+0000","last_active":"2023-10-09T04:16:25.472006+0000","last_peered":"2023-10-09T04:16:25.472006+0000","last_clean":"2023-10-09T04:16:25.472006+0000","last_became_active":"2023-10-09T04:15:25.435144+0000","last_became_peered":"2023-10-09T04:15:25.435144+0000","last_unstale":"2023-10-09T04:16:25.472006+0000","last_undegraded":"2023-10-09T04:16:25.472006+0000","last_fullsized":"2023-10-09T04:16:25.472006+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:50.156419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":266,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472188+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:16:25.472188+0000","last_peered":"2023-10-09T04:16:25.472188+0000","last_clean":"2023-10-09T04:16:25.472188+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:16:25.472188+0000","last_undegraded":"2023-10-09T04:16:25.472188+0000","last_fullsized":"2023-10-09T04:16:25.472188+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497679+0000","last_change":"2023-10-09T04:15:13.429104+0000","last_active":"2023-10-09T04:16:25.497679+0000","last_peered":"2023-10-09T04:16:25.497679+0000","last_clean":"2023-10-09T04:16:25.497679+0000","last_became_active":"2023-10-09T04:15:13.428584+0000","last_became_peered":"2023-10-09T04:15:13.428584+0000","last_unstale":"2023-10-09T04:16:25.497679+0000","last_undegraded":"2023-10-09T04:16:25.497679+0000","last_fullsized":"2023-10-09T04:16:25.497679+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.b","version":"279'8576","reported_seq":8979,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:16:33.849015+0000","last_change":"2023-10-09T04:16:24.651906+0000","last_active":"2023-10-09T04:16:33.849015+0000","last_peered":"2023-10-09T04:16:33.849015+0000","last_clean":"2023-10-09T04:14:14.593157+0000","last_became_active":"2023-10-09T04:16:24.620512+0000","last_became_peered":"2023-10-09T04:16:24.620512+0000","last_unstale":"2023-10-09T04:16:33.849015+0000","last_undegraded":"2023-10-09T04:16:24.612691+0000","last_fullsized":"2023-10-09T04:16:24.468624+0000","mapping_epoch":276,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8076,"log_dups_size":0,"ondisk_log_size":8076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:37.251260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2782,"num_objects_misplaced":2887,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8576,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":10682368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":2613},{"shards":"1,3","objects":-158}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":94,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.475585+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:16:25.475585+0000","last_peered":"2023-10-09T04:16:25.475585+0000","last_clean":"2023-10-09T04:16:25.475585+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:16:25.475585+0000","last_undegraded":"2023-10-09T04:16:25.475585+0000","last_fullsized":"2023-10-09T04:16:25.475585+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":251,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:28.873908+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:16:28.873908+0000","last_peered":"2023-10-09T04:16:28.873908+0000","last_clean":"2023-10-09T04:16:28.873908+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:16:28.873908+0000","last_undegraded":"2023-10-09T04:16:28.873908+0000","last_fullsized":"2023-10-09T04:16:28.873908+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":62,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465355+0000","last_change":"2023-10-09T04:16:24.470074+0000","last_active":"2023-10-09T04:16:25.465355+0000","last_peered":"2023-10-09T04:16:25.465355+0000","last_clean":"2023-10-09T04:16:25.465355+0000","last_became_active":"2023-10-09T04:16:24.469767+0000","last_became_peered":"2023-10-09T04:16:24.469767+0000","last_unstale":"2023-10-09T04:16:25.465355+0000","last_undegraded":"2023-10-09T04:16:25.465355+0000","last_fullsized":"2023-10-09T04:16:25.465355+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.823815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"279'8439","reported_seq":8485,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:30.685213+0000","last_change":"2023-10-09T04:13:42.638373+0000","last_active":"2023-10-09T04:16:30.685213+0000","last_peered":"2023-10-09T04:16:30.685213+0000","last_clean":"2023-10-09T04:16:30.685213+0000","last_became_active":"2023-10-09T04:13:42.637990+0000","last_became_peered":"2023-10-09T04:13:42.637990+0000","last_unstale":"2023-10-09T04:16:30.685213+0000","last_undegraded":"2023-10-09T04:16:30.685213+0000","last_fullsized":"2023-10-09T04:16:30.685213+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8439,"log_dups_size":0,"ondisk_log_size":8439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:37.637304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184352768,"num_objects":2813,"num_object_clones":0,"num_object_copies":8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8439,"num_write_kb":360064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":262,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.481594+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:16:25.481594+0000","last_peered":"2023-10-09T04:16:25.481594+0000","last_clean":"2023-10-09T04:16:25.481594+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:16:25.481594+0000","last_undegraded":"2023-10-09T04:16:25.481594+0000","last_fullsized":"2023-10-09T04:16:25.481594+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":633,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497452+0000","last_change":"2023-10-09T04:16:23.146986+0000","last_active":"2023-10-09T04:16:25.497452+0000","last_peered":"2023-10-09T04:16:25.497452+0000","last_clean":"2023-10-09T04:16:25.497452+0000","last_became_active":"2023-10-09T04:15:13.410161+0000","last_became_peered":"2023-10-09T04:15:13.410161+0000","last_unstale":"2023-10-09T04:16:25.497452+0000","last_undegraded":"2023-10-09T04:16:25.497452+0000","last_fullsized":"2023-10-09T04:16:25.497452+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.464999+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:16:25.464999+0000","last_peered":"2023-10-09T04:16:25.464999+0000","last_clean":"2023-10-09T04:16:25.464999+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:16:25.464999+0000","last_undegraded":"2023-10-09T04:16:25.464999+0000","last_fullsized":"2023-10-09T04:16:25.464999+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472149+0000","last_change":"2023-10-09T04:15:25.409271+0000","last_active":"2023-10-09T04:16:25.472149+0000","last_peered":"2023-10-09T04:16:25.472149+0000","last_clean":"2023-10-09T04:16:25.472149+0000","last_became_active":"2023-10-09T04:15:25.408986+0000","last_became_peered":"2023-10-09T04:15:25.408986+0000","last_unstale":"2023-10-09T04:16:25.472149+0000","last_undegraded":"2023-10-09T04:16:25.472149+0000","last_fullsized":"2023-10-09T04:16:25.472149+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:18.172890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497899+0000","last_change":"2023-10-09T04:15:13.423046+0000","last_active":"2023-10-09T04:16:25.497899+0000","last_peered":"2023-10-09T04:16:25.497899+0000","last_clean":"2023-10-09T04:16:25.497899+0000","last_became_active":"2023-10-09T04:15:13.422707+0000","last_became_peered":"2023-10-09T04:15:13.422707+0000","last_unstale":"2023-10-09T04:16:25.497899+0000","last_undegraded":"2023-10-09T04:16:25.497899+0000","last_fullsized":"2023-10-09T04:16:25.497899+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"279'8246","reported_seq":8294,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:33.841354+0000","last_change":"2023-10-09T04:13:42.635883+0000","last_active":"2023-10-09T04:16:33.841354+0000","last_peered":"2023-10-09T04:16:33.841354+0000","last_clean":"2023-10-09T04:16:33.841354+0000","last_became_active":"2023-10-09T04:13:42.635571+0000","last_became_peered":"2023-10-09T04:13:42.635571+0000","last_unstale":"2023-10-09T04:16:33.841354+0000","last_undegraded":"2023-10-09T04:16:33.841354+0000","last_fullsized":"2023-10-09T04:16:33.841354+0000","mapping_epoch":241,"log_start":"246'200","ondisk_log_start":"246'200","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8046,"log_dups_size":0,"ondisk_log_size":8046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:34.753582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172097536,"num_objects":2626,"num_object_clones":0,"num_object_copies":7878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8246,"num_write_kb":347904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":259,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497429+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:16:25.497429+0000","last_peered":"2023-10-09T04:16:25.497429+0000","last_clean":"2023-10-09T04:16:25.497429+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:16:25.497429+0000","last_undegraded":"2023-10-09T04:16:25.497429+0000","last_fullsized":"2023-10-09T04:16:25.497429+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":264,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:26.097395+0000","last_change":"2023-10-09T04:16:24.470732+0000","last_active":"2023-10-09T04:16:26.097395+0000","last_peered":"2023-10-09T04:16:26.097395+0000","last_clean":"2023-10-09T04:16:26.097395+0000","last_became_active":"2023-10-09T04:16:24.470433+0000","last_became_peered":"2023-10-09T04:16:24.470433+0000","last_unstale":"2023-10-09T04:16:26.097395+0000","last_undegraded":"2023-10-09T04:16:26.097395+0000","last_fullsized":"2023-10-09T04:16:26.097395+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:54.719857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":269,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467730+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:16:25.467730+0000","last_peered":"2023-10-09T04:16:25.467730+0000","last_clean":"2023-10-09T04:16:25.467730+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:16:25.467730+0000","last_undegraded":"2023-10-09T04:16:25.467730+0000","last_fullsized":"2023-10-09T04:16:25.467730+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497619+0000","last_change":"2023-10-09T04:15:13.428296+0000","last_active":"2023-10-09T04:16:25.497619+0000","last_peered":"2023-10-09T04:16:25.497619+0000","last_clean":"2023-10-09T04:16:25.497619+0000","last_became_active":"2023-10-09T04:15:13.428054+0000","last_became_peered":"2023-10-09T04:15:13.428054+0000","last_unstale":"2023-10-09T04:16:25.497619+0000","last_undegraded":"2023-10-09T04:16:25.497619+0000","last_fullsized":"2023-10-09T04:16:25.497619+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"279'8841","reported_seq":8889,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:35.160244+0000","last_change":"2023-10-09T04:13:42.847637+0000","last_active":"2023-10-09T04:16:35.160244+0000","last_peered":"2023-10-09T04:16:35.160244+0000","last_clean":"2023-10-09T04:16:35.160244+0000","last_became_active":"2023-10-09T04:13:42.847359+0000","last_became_peered":"2023-10-09T04:13:42.847359+0000","last_unstale":"2023-10-09T04:16:35.160244+0000","last_undegraded":"2023-10-09T04:16:35.160244+0000","last_fullsized":"2023-10-09T04:16:35.160244+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8841,"log_dups_size":0,"ondisk_log_size":8841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:39.956849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8841,"num_write_kb":371392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":262,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498281+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:16:25.498281+0000","last_peered":"2023-10-09T04:16:25.498281+0000","last_clean":"2023-10-09T04:16:25.498281+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:16:25.498281+0000","last_undegraded":"2023-10-09T04:16:25.498281+0000","last_fullsized":"2023-10-09T04:16:25.498281+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":250,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472320+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:16:25.472320+0000","last_peered":"2023-10-09T04:16:25.472320+0000","last_clean":"2023-10-09T04:16:25.472320+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:16:25.472320+0000","last_undegraded":"2023-10-09T04:16:25.472320+0000","last_fullsized":"2023-10-09T04:16:25.472320+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":221,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467313+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:16:25.467313+0000","last_peered":"2023-10-09T04:16:25.467313+0000","last_clean":"2023-10-09T04:16:25.467313+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:16:25.467313+0000","last_undegraded":"2023-10-09T04:16:25.467313+0000","last_fullsized":"2023-10-09T04:16:25.467313+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498067+0000","last_change":"2023-10-09T04:15:13.427126+0000","last_active":"2023-10-09T04:16:25.498067+0000","last_peered":"2023-10-09T04:16:25.498067+0000","last_clean":"2023-10-09T04:16:25.498067+0000","last_became_active":"2023-10-09T04:15:13.426823+0000","last_became_peered":"2023-10-09T04:15:13.426823+0000","last_unstale":"2023-10-09T04:16:25.498067+0000","last_undegraded":"2023-10-09T04:16:25.498067+0000","last_fullsized":"2023-10-09T04:16:25.498067+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"279'8365","reported_seq":8413,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:31.046356+0000","last_change":"2023-10-09T04:13:42.847596+0000","last_active":"2023-10-09T04:16:31.046356+0000","last_peered":"2023-10-09T04:16:31.046356+0000","last_clean":"2023-10-09T04:16:31.046356+0000","last_became_active":"2023-10-09T04:13:42.847298+0000","last_became_peered":"2023-10-09T04:13:42.847298+0000","last_unstale":"2023-10-09T04:16:31.046356+0000","last_undegraded":"2023-10-09T04:16:31.046356+0000","last_fullsized":"2023-10-09T04:16:31.046356+0000","mapping_epoch":241,"log_start":"247'1600","ondisk_log_start":"247'1600","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:42.275480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189464576,"num_objects":2891,"num_object_clones":0,"num_object_copies":8673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8365,"num_write_kb":360192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":183,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498613+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:16:25.498613+0000","last_peered":"2023-10-09T04:16:25.498613+0000","last_clean":"2023-10-09T04:16:25.498613+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:16:25.498613+0000","last_undegraded":"2023-10-09T04:16:25.498613+0000","last_fullsized":"2023-10-09T04:16:25.498613+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":235,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472641+0000","last_change":"2023-10-09T04:15:25.414014+0000","last_active":"2023-10-09T04:16:25.472641+0000","last_peered":"2023-10-09T04:16:25.472641+0000","last_clean":"2023-10-09T04:16:25.472641+0000","last_became_active":"2023-10-09T04:15:25.413619+0000","last_became_peered":"2023-10-09T04:15:25.413619+0000","last_unstale":"2023-10-09T04:16:25.472641+0000","last_undegraded":"2023-10-09T04:16:25.472641+0000","last_fullsized":"2023-10-09T04:16:25.472641+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:13.741581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":219,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.497572+0000","last_change":"2023-10-09T04:16:24.469186+0000","last_active":"2023-10-09T04:16:25.497572+0000","last_peered":"2023-10-09T04:16:25.497572+0000","last_clean":"2023-10-09T04:16:25.497572+0000","last_became_active":"2023-10-09T04:16:24.468836+0000","last_became_peered":"2023-10-09T04:16:24.468836+0000","last_unstale":"2023-10-09T04:16:25.497572+0000","last_undegraded":"2023-10-09T04:16:25.497572+0000","last_fullsized":"2023-10-09T04:16:25.497572+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:42.902638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":629,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.498797+0000","last_change":"2023-10-09T04:15:13.426933+0000","last_active":"2023-10-09T04:16:25.498797+0000","last_peered":"2023-10-09T04:16:25.498797+0000","last_clean":"2023-10-09T04:16:25.498797+0000","last_became_active":"2023-10-09T04:15:13.426437+0000","last_became_peered":"2023-10-09T04:15:13.426437+0000","last_unstale":"2023-10-09T04:16:25.498797+0000","last_undegraded":"2023-10-09T04:16:25.498797+0000","last_fullsized":"2023-10-09T04:16:25.498797+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"279'8248","reported_seq":8295,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:16:30.693393+0000","last_change":"2023-10-09T04:13:42.646443+0000","last_active":"2023-10-09T04:16:30.693393+0000","last_peered":"2023-10-09T04:16:30.693393+0000","last_clean":"2023-10-09T04:16:30.693393+0000","last_became_active":"2023-10-09T04:13:42.646138+0000","last_became_peered":"2023-10-09T04:13:42.646138+0000","last_unstale":"2023-10-09T04:16:30.693393+0000","last_undegraded":"2023-10-09T04:16:30.693393+0000","last_fullsized":"2023-10-09T04:16:30.693393+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8248,"log_dups_size":0,"ondisk_log_size":8248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:56.607273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187891736,"num_objects":2868,"num_object_clones":0,"num_object_copies":8604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2868,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8248,"num_write_kb":355649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":100,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.476178+0000","last_change":"2023-10-09T04:13:46.100979+0000","last_active":"2023-10-09T04:16:25.476178+0000","last_peered":"2023-10-09T04:16:25.476178+0000","last_clean":"2023-10-09T04:16:25.476178+0000","last_became_active":"2023-10-09T04:13:46.100727+0000","last_became_peered":"2023-10-09T04:13:46.100727+0000","last_unstale":"2023-10-09T04:16:25.476178+0000","last_undegraded":"2023-10-09T04:16:25.476178+0000","last_fullsized":"2023-10-09T04:16:25.476178+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:11.755013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":269,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467805+0000","last_change":"2023-10-09T04:16:20.109259+0000","last_active":"2023-10-09T04:16:25.467805+0000","last_peered":"2023-10-09T04:16:25.467805+0000","last_clean":"2023-10-09T04:16:25.467805+0000","last_became_active":"2023-10-09T04:15:25.496763+0000","last_became_peered":"2023-10-09T04:15:25.496763+0000","last_unstale":"2023-10-09T04:16:25.467805+0000","last_undegraded":"2023-10-09T04:16:25.467805+0000","last_fullsized":"2023-10-09T04:16:25.467805+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:15.807543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.471830+0000","last_change":"2023-10-09T04:15:25.406909+0000","last_active":"2023-10-09T04:16:25.471830+0000","last_peered":"2023-10-09T04:16:25.471830+0000","last_clean":"2023-10-09T04:16:25.471830+0000","last_became_active":"2023-10-09T04:15:25.406617+0000","last_became_peered":"2023-10-09T04:15:25.406617+0000","last_unstale":"2023-10-09T04:16:25.471830+0000","last_undegraded":"2023-10-09T04:16:25.471830+0000","last_fullsized":"2023-10-09T04:16:25.471830+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:34.910221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":242,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.467432+0000","last_change":"2023-10-09T04:15:25.414363+0000","last_active":"2023-10-09T04:16:25.467432+0000","last_peered":"2023-10-09T04:16:25.467432+0000","last_clean":"2023-10-09T04:16:25.467432+0000","last_became_active":"2023-10-09T04:15:25.413987+0000","last_became_peered":"2023-10-09T04:15:25.413987+0000","last_unstale":"2023-10-09T04:16:25.467432+0000","last_undegraded":"2023-10-09T04:16:25.467432+0000","last_fullsized":"2023-10-09T04:16:25.467432+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:45.394691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.476057+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:16:25.476057+0000","last_peered":"2023-10-09T04:16:25.476057+0000","last_clean":"2023-10-09T04:16:25.476057+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:16:25.476057+0000","last_undegraded":"2023-10-09T04:16:25.476057+0000","last_fullsized":"2023-10-09T04:16:25.476057+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":268,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.475362+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:16:25.475362+0000","last_peered":"2023-10-09T04:16:25.475362+0000","last_clean":"2023-10-09T04:16:25.475362+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:16:25.475362+0000","last_undegraded":"2023-10-09T04:16:25.475362+0000","last_fullsized":"2023-10-09T04:16:25.475362+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472152+0000","last_change":"2023-10-09T04:15:25.476221+0000","last_active":"2023-10-09T04:16:25.472152+0000","last_peered":"2023-10-09T04:16:25.472152+0000","last_clean":"2023-10-09T04:16:25.472152+0000","last_became_active":"2023-10-09T04:15:25.475459+0000","last_became_peered":"2023-10-09T04:15:25.475459+0000","last_unstale":"2023-10-09T04:16:25.472152+0000","last_undegraded":"2023-10-09T04:16:25.472152+0000","last_fullsized":"2023-10-09T04:16:25.472152+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:34.779580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":251,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.465471+0000","last_change":"2023-10-09T04:15:25.525420+0000","last_active":"2023-10-09T04:16:25.465471+0000","last_peered":"2023-10-09T04:16:25.465471+0000","last_clean":"2023-10-09T04:16:25.465471+0000","last_became_active":"2023-10-09T04:15:25.406419+0000","last_became_peered":"2023-10-09T04:15:25.406419+0000","last_unstale":"2023-10-09T04:16:25.465471+0000","last_undegraded":"2023-10-09T04:16:25.465471+0000","last_fullsized":"2023-10-09T04:16:25.465471+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:10.680233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.472090+0000","last_change":"2023-10-09T04:15:25.435313+0000","last_active":"2023-10-09T04:16:25.472090+0000","last_peered":"2023-10-09T04:16:25.472090+0000","last_clean":"2023-10-09T04:16:25.472090+0000","last_became_active":"2023-10-09T04:15:25.434996+0000","last_became_peered":"2023-10-09T04:15:25.434996+0000","last_unstale":"2023-10-09T04:16:25.472090+0000","last_undegraded":"2023-10-09T04:16:25.472090+0000","last_fullsized":"2023-10-09T04:16:25.472090+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:22.816036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":94,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.476062+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:16:25.476062+0000","last_peered":"2023-10-09T04:16:25.476062+0000","last_clean":"2023-10-09T04:16:25.476062+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:16:25.476062+0000","last_undegraded":"2023-10-09T04:16:25.476062+0000","last_fullsized":"2023-10-09T04:16:25.476062+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.475616+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:16:25.475616+0000","last_peered":"2023-10-09T04:16:25.475616+0000","last_clean":"2023-10-09T04:16:25.475616+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:16:25.475616+0000","last_undegraded":"2023-10-09T04:16:25.475616+0000","last_fullsized":"2023-10-09T04:16:25.475616+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":271,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:26.097610+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:16:26.097610+0000","last_peered":"2023-10-09T04:16:26.097610+0000","last_clean":"2023-10-09T04:16:26.097610+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:16:26.097610+0000","last_undegraded":"2023-10-09T04:16:26.097610+0000","last_fullsized":"2023-10-09T04:16:26.097610+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:16:25.471460+0000","last_change":"2023-10-09T04:15:25.423355+0000","last_active":"2023-10-09T04:16:25.471460+0000","last_peered":"2023-10-09T04:16:25.471460+0000","last_clean":"2023-10-09T04:16:25.471460+0000","last_became_active":"2023-10-09T04:15:25.423070+0000","last_became_peered":"2023-10-09T04:15:25.423070+0000","last_unstale":"2023-10-09T04:16:25.471460+0000","last_undegraded":"2023-10-09T04:16:25.471460+0000","last_fullsized":"2023-10-09T04:16:25.471460+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:56.629007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6134,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2970157080,"num_objects":45322,"num_object_clones":0,"num_object_copies":135966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2782,"num_objects_misplaced":13655,"num_objects_unfound":0,"num_objects_dirty":45322,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135842,"num_write_kb":5797185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11253,"num_bytes_recovered":725417984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9506205696,"data_stored":9506193480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124796,"ondisk_log_size":124796,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201568,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1499744,"kb_used_data":1202456,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92245408,"statfs":{"total":95995035648,"available":94459297792,"internally_reserved":0,"allocated":1231314944,"data_stored":1230485932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":4.8890000000000002,"15min":2.407},"min":{"1min":0.222,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":15.068,"5min":126.752,"15min":126.752},"last":1.175},{"interface":"front","average":{"1min":1.819,"5min":4.9050000000000002,"15min":2.4020000000000001},"min":{"1min":0.28399999999999997,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6910000000000007,"5min":128.88399999999999,"15min":128.88399999999999},"last":2.407}]},{"osd":1,"last update":"Mon Oct 9 04:15:58 2023","interfaces":[{"interface":"back","average":{"1min":10.446,"5min":6.2460000000000004,"15min":3.6080000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":103.324,"5min":103.324,"15min":245.28700000000001},"last":3.4329999999999998},{"interface":"front","average":{"1min":10.492000000000001,"5min":5.9619999999999997,"15min":3.5},"min":{"1min":0.25,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":104.248,"5min":104.248,"15min":244.626},"last":3.4489999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":2.1459999999999999,"15min":1.7410000000000001},"min":{"1min":0.30499999999999999,"5min":0.27800000000000002,"15min":0.27400000000000002},"max":{"1min":4.1420000000000003,"5min":76.774000000000001,"15min":76.774000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.641,"5min":2.3490000000000002,"15min":2.2130000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.6130000000000004,"5min":76.715999999999994,"15min":99.372},"last":0.53800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:15:58 2023","interfaces":[{"interface":"back","average":{"1min":6.2720000000000002,"5min":2.952,"15min":2.3250000000000002},"min":{"1min":0.309,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":85.575000000000003,"5min":85.575000000000003,"15min":99.134},"last":0.37},{"interface":"front","average":{"1min":6.907,"5min":3.1789999999999998,"15min":2.4649999999999999},"min":{"1min":0.376,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":85.528000000000006,"5min":85.528000000000006,"15min":99.117999999999995},"last":0.57199999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.49199999999999999,"15min":0.70199999999999996},"min":{"1min":0.28999999999999998,"5min":0.252,"15min":0.23400000000000001},"max":{"1min":1.893,"5min":1.9790000000000001,"15min":16.195},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.75,"5min":0.64600000000000002,"15min":0.76200000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.3919999999999999,"5min":3.3929999999999998,"15min":12.442},"last":0.61499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":6.3769999999999998,"5min":1.764,"15min":1.0269999999999999},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.27400000000000002},"max":{"1min":120.066,"5min":120.066,"15min":120.066},"last":0.49299999999999999},{"interface":"front","average":{"1min":6.173,"5min":1.7490000000000001,"15min":1.0860000000000001},"min":{"1min":0.34999999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":120.09699999999999,"5min":120.09699999999999,"15min":120.09699999999999},"last":0.47599999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.71499999999999997,"15min":0.74099999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":7.7370000000000001,"5min":7.7370000000000001,"15min":8.3559999999999999},"last":2.746},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.623,"15min":0.75800000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.318},"max":{"1min":4.375,"5min":4.375,"15min":5.9039999999999999},"last":1.5569999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463206,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1358220,"kb_used_data":1052740,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92386932,"statfs":{"total":95995035648,"available":94604218368,"internally_reserved":0,"allocated":1078005760,"data_stored":1077196735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":3.452,"15min":2.819},"min":{"1min":0.25,"5min":0.193,"15min":0.193},"max":{"1min":7.9820000000000002,"5min":120.55200000000001,"15min":120.55200000000001},"last":87.927999999999997},{"interface":"front","average":{"1min":2.5939999999999999,"5min":3.73,"15min":3.1480000000000001},"min":{"1min":0.26200000000000001,"5min":0.223,"15min":0.223},"max":{"1min":7.0179999999999998,"5min":120.252,"15min":120.252},"last":87.238}]},{"osd":1,"last update":"Mon Oct 9 04:16:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":4.0640000000000001,"15min":2.7999999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.7050000000000001,"5min":143.22499999999999,"15min":143.22499999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.266,"5min":4.0090000000000003,"15min":2.8679999999999999},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.8570000000000002,"5min":143.18700000000001,"15min":143.18700000000001},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":3.4910000000000001,"15min":2.819},"min":{"1min":0.34300000000000003,"5min":0.23899999999999999,"15min":0.218},"max":{"1min":8.3369999999999997,"5min":139.06899999999999,"15min":139.06899999999999},"last":1.7090000000000001},{"interface":"front","average":{"1min":2.012,"5min":3.5430000000000001,"15min":2.8650000000000002},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":8.3729999999999993,"5min":139.173,"15min":139.173},"last":1.738}]},{"osd":3,"last update":"Mon Oct 9 04:16:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":2.371,"15min":2.496},"min":{"1min":0.30599999999999999,"5min":0.29699999999999999,"15min":0.25900000000000001},"max":{"1min":10.375,"5min":51.082000000000001,"15min":57.509999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.992,"15min":2.4670000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.6639999999999997,"5min":51.031999999999996,"15min":57.453000000000003},"last":1.224}]},{"osd":4,"last update":"Mon Oct 9 04:16:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":0.77000000000000002,"15min":0.59199999999999997},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":4.6920000000000002,"5min":4.6920000000000002,"15min":4.6920000000000002},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.46,"5min":0.82399999999999995,"15min":0.73499999999999999},"min":{"1min":0.32100000000000001,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":4.3179999999999996,"5min":4.3179999999999996,"15min":7.7000000000000002},"last":0.443}]},{"osd":5,"last update":"Mon Oct 9 04:16:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":1.175,"15min":0.98399999999999999},"min":{"1min":0.34599999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.6310000000000002,"5min":22.879000000000001,"15min":54.773000000000003},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.8129999999999999,"5min":1.25,"15min":1.085},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.8220000000000001,"5min":23.006,"15min":54.853999999999999},"last":0.47799999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":0.99099999999999999,"15min":0.76400000000000001},"min":{"1min":0.38300000000000001,"5min":0.317,"15min":0.313},"max":{"1min":5.9560000000000004,"5min":5.9560000000000004,"15min":5.9560000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.131,"15min":0.86499999999999999},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28699999999999998},"max":{"1min":9.1829999999999998,"5min":9.1829999999999998,"15min":9.1829999999999998},"last":0.59199999999999997}]}]},{"osd":1,"up_from":89,"seq":382252089514,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1644352,"kb_used_data":1253752,"kb_used_omap":9,"kb_used_meta":390582,"kb_avail":92100800,"statfs":{"total":95995035648,"available":94311219200,"internally_reserved":0,"allocated":1283842048,"data_stored":1282690806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":399956969},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,2,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:15:56 2023","interfaces":[{"interface":"back","average":{"1min":2.7000000000000002,"5min":5.1959999999999997,"15min":3.0470000000000002},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":9.5229999999999997,"5min":68.048000000000002,"15min":85.977999999999994},"last":5.3090000000000002},{"interface":"front","average":{"1min":2.4550000000000001,"5min":4.8120000000000003,"15min":3.355},"min":{"1min":0.19800000000000001,"5min":0.182,"15min":0.182},"max":{"1min":10.205,"5min":61.670000000000002,"15min":99.831999999999994},"last":5.3860000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.3359999999999999,"15min":2.2010000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.191},"max":{"1min":7.6349999999999998,"5min":25.577000000000002,"15min":121.54600000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.8799999999999999,"15min":2.54},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":13.595000000000001,"5min":15.298999999999999,"15min":99.899000000000001},"last":1.2170000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:15:56 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.2999999999999998,"15min":3.2879999999999998},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":68.054000000000002,"5min":68.054000000000002,"15min":138.87299999999999},"last":5.2750000000000004},{"interface":"front","average":{"1min":6.101,"5min":4.9329999999999998,"15min":3.9329999999999998},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":68.563999999999993,"5min":68.563999999999993,"15min":138.49799999999999},"last":5.3499999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:15:56 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":2.4300000000000002,"15min":1.9219999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.254},"max":{"1min":10.130000000000001,"5min":15.215999999999999,"15min":99.989999999999995},"last":1.256},{"interface":"front","average":{"1min":2.681,"5min":2.3210000000000002,"15min":1.891},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":13.428000000000001,"5min":25.957999999999998,"15min":100.249},"last":0.32600000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:15:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.468,"15min":1.9179999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.263,"5min":15.266,"15min":99.933999999999997},"last":1.2889999999999999},{"interface":"front","average":{"1min":3.3170000000000002,"5min":2.52,"15min":2.048},"min":{"1min":0.34999999999999998,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":13.49,"5min":25.890999999999998,"15min":100.34099999999999},"last":0.377}]},{"osd":6,"last update":"Mon Oct 9 04:15:56 2023","interfaces":[{"interface":"back","average":{"1min":3.129,"5min":2.5110000000000001,"15min":2.1539999999999999},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.377000000000001,"5min":25.931000000000001,"15min":100.309},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.5190000000000001,"5min":2.0099999999999998,"15min":1.8779999999999999},"min":{"1min":0.436,"5min":0.36499999999999999,"15min":0.29799999999999999},"max":{"1min":6.5140000000000002,"5min":13.446,"15min":100.143},"last":0.441}]},{"osd":7,"last update":"Mon Oct 9 04:15:56 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":2.5019999999999998,"15min":2.1190000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.35799999999999998},"max":{"1min":13.459,"5min":25.984999999999999,"15min":100.214},"last":1.3600000000000001},{"interface":"front","average":{"1min":3.121,"5min":2.177,"15min":1.944},"min":{"1min":0.48699999999999999,"5min":0.375,"15min":0.317},"max":{"1min":7.9420000000000002,"5min":13.471,"15min":100.113},"last":0.47899999999999998}]}]},{"osd":0,"up_from":84,"seq":360777253039,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1728700,"kb_used_data":1314228,"kb_used_omap":28,"kb_used_meta":414435,"kb_avail":92016452,"statfs":{"total":95995035648,"available":94224846848,"internally_reserved":0,"allocated":1345769472,"data_stored":1344561119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":424382288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:15:55 2023","interfaces":[{"interface":"back","average":{"1min":8.7769999999999992,"5min":3.7839999999999998,"15min":2.956},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.17000000000000001},"max":{"1min":105.035,"5min":105.035,"15min":118.48},"last":2.302},{"interface":"front","average":{"1min":8.5519999999999996,"5min":4.6299999999999999,"15min":3.415},"min":{"1min":0.255,"5min":0.25,"15min":0.214},"max":{"1min":105.134,"5min":105.134,"15min":118.384},"last":2.7559999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":2.9079999999999999,"15min":2.6779999999999999},"min":{"1min":0.25700000000000001,"5min":0.22500000000000001,"15min":0.221},"max":{"1min":7.9649999999999999,"5min":13.108000000000001,"15min":118.54000000000001},"last":2.532},{"interface":"front","average":{"1min":3.2280000000000002,"5min":2.7629999999999999,"15min":2.782},"min":{"1min":0.23300000000000001,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":11.611000000000001,"5min":12.821,"15min":118.727},"last":14.304}]},{"osd":3,"last update":"Mon Oct 9 04:15:55 2023","interfaces":[{"interface":"back","average":{"1min":8.0649999999999995,"5min":4.923,"15min":4.8710000000000004},"min":{"1min":0.28699999999999998,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":99.709000000000003,"5min":99.709000000000003,"15min":156.488},"last":2.6320000000000001},{"interface":"front","average":{"1min":7.3150000000000004,"5min":4.8540000000000001,"15min":4.9100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.22500000000000001},"max":{"1min":99.560000000000002,"5min":99.560000000000002,"15min":156.50700000000001},"last":2.6419999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:15:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.9330000000000001,"15min":2.0030000000000001},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.7089999999999996,"5min":9.7270000000000003,"15min":118.633},"last":2.5569999999999999},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.7250000000000001,"15min":2.2330000000000001},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.0630000000000006,"5min":10.492000000000001,"15min":118.913},"last":12.994999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:15:40 2023","interfaces":[{"interface":"back","average":{"1min":11.44,"5min":3.7120000000000002,"15min":3.3999999999999999},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":156.96100000000001,"5min":156.96100000000001,"15min":182.68899999999999},"last":13.246},{"interface":"front","average":{"1min":11.721,"5min":3.6859999999999999,"15min":3.2090000000000001},"min":{"1min":0.38,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":156.97300000000001,"5min":156.97300000000001,"15min":182.66300000000001},"last":2.3380000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:15:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.8500000000000001,"15min":2.4500000000000002},"min":{"1min":0.39900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.797000000000001,"5min":137.36099999999999,"15min":137.36099999999999},"last":2.3900000000000001},{"interface":"front","average":{"1min":2.069,"5min":3.238,"15min":2.4820000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":7.3170000000000002,"5min":136.88999999999999,"15min":136.88999999999999},"last":2.585}]},{"osd":7,"last update":"Mon Oct 9 04:15:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.7879999999999998,"15min":3.1110000000000002},"min":{"1min":0.433,"5min":0.33800000000000002,"15min":0.32100000000000001},"max":{"1min":10.77,"5min":121.324,"15min":185.024},"last":2.3639999999999999},{"interface":"front","average":{"1min":2.4009999999999998,"5min":2.306,"15min":2.762},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":7.7859999999999996,"5min":12.417,"15min":184.98599999999999},"last":2.617}]}]},{"osd":2,"up_from":94,"seq":403726925988,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1651264,"kb_used_data":1219256,"kb_used_omap":81,"kb_used_meta":431918,"kb_avail":92093888,"statfs":{"total":95995035648,"available":94304141312,"internally_reserved":0,"allocated":1248518144,"data_stored":1247365588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":442284250},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":6.1120000000000001,"5min":5.0999999999999996,"15min":2.7290000000000001},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.17399999999999999},"max":{"1min":35.896000000000001,"5min":64.430999999999997,"15min":64.430999999999997},"last":2.573},{"interface":"front","average":{"1min":4.0679999999999996,"5min":4.21,"15min":2.7909999999999999},"min":{"1min":0.20799999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":34.981000000000002,"5min":60.125999999999998,"15min":60.125999999999998},"last":2.6299999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.8530000000000002,"15min":2.6419999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.315,"5min":50.680999999999997,"15min":68.009},"last":2.3479999999999999},{"interface":"front","average":{"1min":3.1429999999999998,"5min":2.8570000000000002,"15min":2.7490000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":15.419,"5min":53.353000000000002,"15min":66.975999999999999},"last":8.8970000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:15:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":2.6629999999999998,"15min":1.9330000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":7.2629999999999999,"5min":20.242999999999999,"15min":27.562000000000001},"last":1.0920000000000001},{"interface":"front","average":{"1min":2.1640000000000001,"5min":2.4769999999999999,"15min":2.105},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":6.4779999999999998,"5min":17.076000000000001,"15min":46.286999999999999},"last":2.2490000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.6000000000000001,"15min":1.6639999999999999},"min":{"1min":0.28599999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.265000000000001,"5min":10.265000000000001,"15min":46.075000000000003},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.784,"5min":2.0550000000000002,"15min":1.52},"min":{"1min":0.312,"5min":0.28899999999999998,"15min":0.27200000000000002},"max":{"1min":5.4960000000000004,"5min":20.134,"15min":20.134},"last":1.1200000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":3.085,"5min":1.8939999999999999,"15min":1.786},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":11.201000000000001,"5min":11.201000000000001,"15min":46.265000000000001},"last":2.2869999999999999},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.6639999999999999,"15min":1.8400000000000001},"min":{"1min":0.35799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.331,"5min":10.331,"15min":46.106000000000002},"last":0.69699999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":2.1859999999999999,"15min":1.9890000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":6.8239999999999998,"5min":20.184000000000001,"15min":114.14},"last":1.4550000000000001},{"interface":"front","average":{"1min":2.4780000000000002,"5min":1.7649999999999999,"15min":2.117},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33300000000000002},"max":{"1min":11.252000000000001,"5min":11.252000000000001,"15min":114.123},"last":2.3180000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.2759999999999998,"15min":2.1859999999999999},"min":{"1min":0.435,"5min":0.36299999999999999,"15min":0.34300000000000003},"max":{"1min":5.423,"5min":20.215,"15min":117.32599999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":2.9199999999999999,"5min":1.9350000000000001,"15min":2.262},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":11.284000000000001,"5min":11.284000000000001,"15min":117.28700000000001},"last":2.3740000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762461,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1946628,"kb_used_data":1306748,"kb_used_omap":7,"kb_used_meta":639800,"kb_avail":91798524,"statfs":{"total":95995035648,"available":94001688576,"internally_reserved":0,"allocated":1338109952,"data_stored":1336953799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":655155816},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":5.7510000000000003,"15min":4.7039999999999997},"min":{"1min":0.20899999999999999,"5min":0.20100000000000001,"15min":0.185},"max":{"1min":14.755000000000001,"5min":125.765,"15min":125.765},"last":4.5750000000000002},{"interface":"front","average":{"1min":3.4590000000000001,"5min":6.0309999999999997,"15min":4.899},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":25.57,"5min":125.46899999999999,"15min":125.46899999999999},"last":4.4379999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":5.4829999999999997,"5min":5.3700000000000001,"15min":3.7490000000000001},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":54.604999999999997,"5min":186.21299999999999,"15min":186.21299999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":5.6219999999999999,"5min":5.4720000000000004,"15min":3.573},"min":{"1min":0.25800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":54.819000000000003,"5min":191.78,"15min":191.78},"last":0.255}]},{"osd":2,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":5.6550000000000002,"5min":3.6080000000000001,"15min":2.9849999999999999},"min":{"1min":0.25800000000000001,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":72.343999999999994,"5min":72.343999999999994,"15min":106.105},"last":0.28499999999999998},{"interface":"front","average":{"1min":6.1280000000000001,"5min":3.9220000000000002,"15min":3.4950000000000001},"min":{"1min":0.315,"5min":0.214,"15min":0.214},"max":{"1min":72.480000000000004,"5min":72.480000000000004,"15min":106.70399999999999},"last":0.438}]},{"osd":4,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.4940000000000002,"5min":2.3039999999999998,"15min":2.3570000000000002},"min":{"1min":0.33100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":27.539999999999999,"5min":27.539999999999999,"15min":106.85899999999999},"last":0.504},{"interface":"front","average":{"1min":3.657,"5min":2.2429999999999999,"15min":1.8149999999999999},"min":{"1min":0.31900000000000001,"5min":0.29899999999999999,"15min":0.26900000000000002},"max":{"1min":26.663,"5min":26.663,"15min":37.442},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.4670000000000001,"5min":2.2690000000000001,"15min":1.714},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":17.901,"5min":17.901,"15min":17.901},"last":3.5219999999999998},{"interface":"front","average":{"1min":3.1000000000000001,"5min":2.2229999999999999,"15min":1.806},"min":{"1min":0.35899999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":18.068000000000001,"5min":18.068000000000001,"15min":18.068000000000001},"last":4.5140000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":2.1560000000000001,"15min":2.1739999999999999},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":14.567,"5min":14.976000000000001,"15min":106.036},"last":4.5519999999999996},{"interface":"front","average":{"1min":3.577,"5min":2.3090000000000002,"15min":2.2269999999999999},"min":{"1min":0.36499999999999999,"5min":0.318,"15min":0.318},"max":{"1min":14.529,"5min":14.529,"15min":106.741},"last":0.53400000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:15:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":2.2040000000000002,"15min":2.234},"min":{"1min":0.35699999999999998,"5min":0.34599999999999997,"15min":0.34399999999999997},"max":{"1min":14.291,"5min":14.291,"15min":106.82599999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.085,"5min":2.3239999999999998,"15min":2.1720000000000002},"min":{"1min":0.40000000000000002,"5min":0.38900000000000001,"15min":0.35999999999999999},"max":{"1min":20.757999999999999,"5min":20.757999999999999,"15min":106.084},"last":0.64100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986478,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":855312,"kb_used_data":545736,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92889840,"statfs":{"total":95995035648,"available":95119196160,"internally_reserved":0,"allocated":558833664,"data_stored":558020133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":13.718,"5min":7.4370000000000003,"15min":3.653},"min":{"1min":0.23000000000000001,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":154.62100000000001,"5min":154.62100000000001,"15min":154.62100000000001},"last":0.314},{"interface":"front","average":{"1min":13.945,"5min":7.5339999999999998,"15min":3.6349999999999998},"min":{"1min":0.28299999999999997,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":154.37299999999999,"5min":154.37299999999999,"15min":154.37299999999999},"last":2.0960000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:15:58 2023","interfaces":[{"interface":"back","average":{"1min":8.093,"5min":4.6100000000000003,"15min":2.637},"min":{"1min":0.35599999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":55.933,"5min":117.81,"15min":117.81},"last":2.8969999999999998},{"interface":"front","average":{"1min":7.3970000000000002,"5min":4.4930000000000003,"15min":2.5960000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26000000000000001},"max":{"1min":58.588000000000001,"5min":117.848,"15min":117.848},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.8159999999999998,"15min":1.835},"min":{"1min":0.308,"5min":0.255,"15min":0.245},"max":{"1min":7.5460000000000003,"5min":123.598,"15min":123.598},"last":2.0600000000000001},{"interface":"front","average":{"1min":1.5529999999999999,"5min":2.7029999999999998,"15min":1.756},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":5.0220000000000002,"5min":123.619,"15min":123.619},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:15:58 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":3.0920000000000001,"15min":1.867},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.9219999999999997,"5min":125.90300000000001,"15min":125.90300000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.53,"5min":3.2229999999999999,"15min":2.0129999999999999},"min":{"1min":0.32600000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.9770000000000003,"5min":127.90000000000001,"15min":127.90000000000001},"last":2.0800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:15:47 2023","interfaces":[{"interface":"back","average":{"1min":12.973000000000001,"5min":2.9990000000000001,"15min":1.419},"min":{"1min":0.29499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":111.809,"5min":111.809,"15min":111.809},"last":2.0139999999999998},{"interface":"front","average":{"1min":12.877000000000001,"5min":3.0059999999999998,"15min":1.552},"min":{"1min":0.255,"5min":0.255,"15min":0.25},"max":{"1min":112.002,"5min":112.002,"15min":112.002},"last":3.27}]},{"osd":6,"last update":"Mon Oct 9 04:15:58 2023","interfaces":[{"interface":"back","average":{"1min":7.0810000000000004,"5min":1.9399999999999999,"15min":1.196},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":76.307000000000002,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.94099999999999995},{"interface":"front","average":{"1min":6.9829999999999997,"5min":1.911,"15min":1.1100000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.30599999999999999},"max":{"1min":78.421999999999997,"5min":78.421999999999997,"15min":78.421999999999997},"last":0.371}]},{"osd":7,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.0740000000000001,"15min":0.878},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.718,"5min":34.039000000000001,"15min":34.039000000000001},"last":2.0369999999999999},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.1140000000000001,"15min":0.93700000000000006},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.6029999999999998,"5min":33.433999999999997,"15min":33.433999999999997},"last":1.012}]}]},{"osd":5,"up_from":39,"seq":167503724841,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1787396,"kb_used_data":1477820,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91957756,"statfs":{"total":95995035648,"available":94164742144,"internally_reserved":0,"allocated":1513287680,"data_stored":1512482147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":10.468,"5min":9.8209999999999997,"15min":4.2649999999999997},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":118.98399999999999,"5min":151.23599999999999,"15min":151.23599999999999},"last":6.202},{"interface":"front","average":{"1min":11.349,"5min":10.618,"15min":4.5730000000000004},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":118.846,"5min":151.566,"15min":151.566},"last":8.3810000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:15:55 2023","interfaces":[{"interface":"back","average":{"1min":8.6199999999999992,"5min":5.1929999999999996,"15min":2.996},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":109.32899999999999,"5min":109.32899999999999,"15min":109.32899999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":9.952,"5min":5.5279999999999996,"15min":3.2949999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":112.226,"5min":112.226,"15min":112.226},"last":5.5039999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:15:32 2023","interfaces":[{"interface":"back","average":{"1min":2.9039999999999999,"5min":4.0750000000000002,"15min":2.6629999999999998},"min":{"1min":0.30599999999999999,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":9.1639999999999997,"5min":106.508,"15min":106.508},"last":6.7519999999999998},{"interface":"front","average":{"1min":1.6180000000000001,"5min":3.968,"15min":2.5659999999999998},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.6699999999999999,"5min":106.462,"15min":106.462},"last":0.32100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":2.1629999999999998,"15min":2.1829999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.5049999999999999,"5min":27.925000000000001,"15min":91.834999999999994},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.7849999999999999,"5min":2.2290000000000001,"15min":2.2229999999999999},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":4.0700000000000003,"5min":27.939,"15min":91.802000000000007},"last":0.68400000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:15:33 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":0.77600000000000002,"15min":0.745},"min":{"1min":0.29199999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.4089999999999998,"5min":4.4089999999999998,"15min":5.7430000000000003},"last":5.633},{"interface":"front","average":{"1min":1.085,"5min":0.67800000000000005,"15min":0.76400000000000001},"min":{"1min":0.32700000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.5259999999999998,"5min":4.944,"15min":6.7709999999999999},"last":0.35499999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":2.1019999999999999,"15min":1.1890000000000001},"min":{"1min":0.307,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":3.6890000000000001,"5min":123.239,"15min":123.239},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1379999999999999,"5min":2.359,"15min":1.2809999999999999},"min":{"1min":0.371,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9239999999999995,"5min":123.014,"15min":123.014},"last":5.6509999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:15:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":0.749,"15min":0.84399999999999997},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":3.6680000000000001,"5min":3.6680000000000001,"15min":9.4930000000000003},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.573,"5min":0.84699999999999998,"15min":0.88400000000000001},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.307},"max":{"1min":4.4900000000000002,"5min":4.4900000000000002,"15min":8.5329999999999995},"last":5.5940000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1325793280,"data_stored":1325793280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1266876416,"data_stored":1266876416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1239093248,"data_stored":1239089176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1332609024,"data_stored":1332609024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":555548672,"data_stored":555548672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1498812416,"data_stored":1498808344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1066795008,"data_stored":1066795008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1220677632,"data_stored":1220673560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:16:37.190 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:16:37.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-10-09T04:16:37.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:36 smithi181 ceph-mon[196979]: 1.3 scrub starts 2023-10-09T04:16:37.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:36 smithi181 ceph-mon[196979]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:16:37.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:37.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:37.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:37.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:37.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:37.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:37.636 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:38.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:38.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:38.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:38.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:38.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:37 smithi163 ceph-mon[221067]: 1.9 scrub starts 2023-10-09T04:16:38.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2782/136038 objects degraded (2.045%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:38.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:37 smithi163 ceph-mon[221067]: pgmap v766: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 2782/130440 objects degraded (2.133%); 12946/130440 objects misplaced (9.925%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:16:38.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:37 smithi163 ceph-mon[213841]: 1.9 scrub starts 2023-10-09T04:16:38.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2782/136038 objects degraded (2.045%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:38.236 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:37 smithi163 ceph-mon[213841]: pgmap v766: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 2782/130440 objects degraded (2.133%); 12946/130440 objects misplaced (9.925%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:16:38.238 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:38.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:37 smithi181 ceph-mon[196979]: 1.9 scrub starts 2023-10-09T04:16:38.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2782/136038 objects degraded (2.045%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:38.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:37 smithi181 ceph-mon[196979]: pgmap v766: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 2782/130440 objects degraded (2.133%); 12946/130440 objects misplaced (9.925%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:16:38.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:38.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:38.416 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:38.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:38.493 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:38.557 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:38.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:38.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:38.662 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:38.771 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:38.884 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:38.901 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:38.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:39.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:39.042 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:16:39.043 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:16:39.126 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:39.268 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:38 smithi181 ceph-mon[196979]: 1.7 scrub starts 2023-10-09T04:16:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/601309829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:16:39.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:38 smithi163 ceph-mon[213841]: 1.7 scrub starts 2023-10-09T04:16:39.282 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/601309829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:16:39.282 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:38 smithi163 ceph-mon[221067]: 1.7 scrub starts 2023-10-09T04:16:39.282 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/601309829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:16:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:39.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:39.387 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:39.442 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:39.463 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:16:39.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:39.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:39.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:39.705 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: 1.6 scrub starts 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: failed to encode map e280 with expected crc 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/601309829' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: failed to encode map e280 with expected crc 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: failed to encode map e280 with expected crc 2023-10-09T04:16:39.970 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[213841]: pgmap v768: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 2782/128154 objects degraded (2.171%); 12946/128154 objects misplaced (10.102%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:16:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: 1.6 scrub starts 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: failed to encode map e280 with expected crc 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/601309829' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: failed to encode map e280 with expected crc 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: failed to encode map e280 with expected crc 2023-10-09T04:16:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:39 smithi181 ceph-mon[196979]: pgmap v768: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 2782/128154 objects degraded (2.171%); 12946/128154 objects misplaced (10.102%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:16:40.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: 1.6 scrub starts 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: failed to encode map e280 with expected crc 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/601309829' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: failed to encode map e280 with expected crc 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: failed to encode map e280 with expected crc 2023-10-09T04:16:40.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:39 smithi163 ceph-mon[221067]: pgmap v768: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 2782/128154 objects degraded (2.171%); 12946/128154 objects misplaced (10.102%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:16:40.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:40.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:40.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:40.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:40.760 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:40.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:16:40.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:40.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:40.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:40.974 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:16:40] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.43.0" 2023-10-09T04:16:40.977 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:41.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:41.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:41.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:41.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:41.881 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:42.510 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:16:42.558 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:16:42.594 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:16:42.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:42.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:42.637 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:16:42.672 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:16:42.696 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:42.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:42.723 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:16:42.763 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:42.764 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:16:42.799 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:16:42.811 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.812 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:16:42.827 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:42.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:42.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:42.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:42 smithi181 ceph-mon[196979]: pgmap v769: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 2782/121317 objects degraded (2.293%); 11982/121317 objects misplaced (9.877%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:42.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2782/121317 objects degraded (2.293%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:42.979 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4156329477' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:42.982 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:43.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:43.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:42 smithi163 ceph-mon[221067]: pgmap v769: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 2782/121317 objects degraded (2.293%); 11982/121317 objects misplaced (9.877%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:43.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2782/121317 objects degraded (2.293%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:43.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4156329477' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:43.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:42 smithi163 ceph-mon[213841]: pgmap v769: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 2782/121317 objects degraded (2.293%); 11982/121317 objects misplaced (9.877%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:43.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2782/121317 objects degraded (2.293%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:43.046 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4156329477' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:43.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:43.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:43.148 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:43.177 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:43.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:16:44.082 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:44.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:44.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:44.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:44.249 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:44.320 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:44.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:44.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:44.456 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:44.720 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 1 pg_num to 21 2023-10-09T04:16:44.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:44.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:44 smithi163 ceph-mon[221067]: pgmap v770: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 2782/121317 objects degraded (2.293%); 11982/121317 objects misplaced (9.877%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:44.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1613813485' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:16:44.730 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:16:44.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:44.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:44.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:44 smithi163 ceph-mon[213841]: pgmap v770: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 2782/121317 objects degraded (2.293%); 11982/121317 objects misplaced (9.877%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:44.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1613813485' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:16:44.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:16:44.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:44.926 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:45.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:44 smithi181 ceph-mon[196979]: pgmap v770: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 2782/121317 objects degraded (2.293%); 11982/121317 objects misplaced (9.877%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:16:45.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1613813485' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:16:45.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T04:16:45.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:45.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:45.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:45.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:45.267 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:45.582 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:45.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:45.634 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:45.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:45.771 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:45.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:16:45.849 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T04:16:45.849 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[213841]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T04:16:45.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[213841]: failed to encode map e281 with expected crc 2023-10-09T04:16:45.850 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T04:16:45.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T04:16:45.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[221067]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T04:16:45.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[221067]: failed to encode map e281 with expected crc 2023-10-09T04:16:45.850 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:45 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T04:16:46.000 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:46.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:45 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T04:16:46.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:45 smithi181 ceph-mon[196979]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T04:16:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:45 smithi181 ceph-mon[196979]: failed to encode map e281 with expected crc 2023-10-09T04:16:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:45 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T04:16:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:46.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:46.124 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:144331 8.d e282/241 8:bc87abcd:::benchmark_data_smithi163_404719_object53747:head v 282'8826, mlcod=282'8824)", 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:46.176289+0000", 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0070581419999999999, 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070933869999999996, 2023-10-09T04:16:46.194 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176289+0000", 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176289+0000", 2023-10-09T04:16:46.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176288+0000", 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176296+0000", 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6909999999999997e-06 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176313+0000", 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7227e-05 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176321+0000", 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2949999999999994e-06 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:16:46.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176396+0000", 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5364999999999994e-05 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.176420+0000", 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3391999999999999e-05 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:144330 8.b e282/276 8:db9606b1:::benchmark_data_smithi163_404719_object53746:head v 282'9037, mlcod=248'2592)", 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:46.177092+0000", 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0062544230000000003, 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00635872, 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:16:46.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177092+0000", 2023-10-09T04:16:46.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177092+0000", 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177092+0000", 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177099+0000", 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0190000000000001e-06 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177109+0000", 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0728e-05 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177116+0000", 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8179999999999998e-06 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177177+0000", 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0838999999999998e-05 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:16:46.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.177201+0000", 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4363e-05 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:144333 8.f 8:f1a171cd:::benchmark_data_smithi163_404719_object53749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:46.180703+0000", 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.002643538, 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002812625, 2023-10-09T04:16:46.204 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 144333 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180703+0000", 2023-10-09T04:16:46.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180703+0000", 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180702+0000", 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:16:46.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180711+0000", 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3999999999999998e-06 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180726+0000", 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4596000000000001e-05 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180735+0000", 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5829999999999996e-06 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180789+0000", 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4296999999999999e-05 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:16:46.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.180922+0000", 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013305500000000001 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.181038+0000", 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115389 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.209 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:144332 8.4 e282/265)", 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:46.183316+0000", 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 3.0834000000000003e-05, 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026883400000000002, 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.183316+0000", 2023-10-09T04:16:46.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.183316+0000", 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:16:46.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.183315+0000", 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.183355+0000", 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0046000000000002e-05 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.183368+0000", 2023-10-09T04:16:46.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3144e-05 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:46.183376+0000", 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6510000000000001e-06 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 4 2023-10-09T04:16:46.213 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:16:46.240 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:46.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:46.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:46.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:46.719 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:46.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:46.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:46.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[213841]: pgmap v772: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 2782/117903 objects degraded (2.360%); 11800/117903 objects misplaced (10.008%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:16:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[213841]: failed to encode map e282 with expected crc 2023-10-09T04:16:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[213841]: failed to encode map e282 with expected crc 2023-10-09T04:16:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T04:16:46.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[213841]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T04:16:46.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[221067]: pgmap v772: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 2782/117903 objects degraded (2.360%); 11800/117903 objects misplaced (10.008%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:16:46.838 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[221067]: failed to encode map e282 with expected crc 2023-10-09T04:16:46.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[221067]: failed to encode map e282 with expected crc 2023-10-09T04:16:46.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T04:16:46.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:46 smithi163 ceph-mon[221067]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T04:16:46.919 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:46.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:46 smithi181 ceph-mon[196979]: pgmap v772: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 2782/117903 objects degraded (2.360%); 11800/117903 objects misplaced (10.008%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:16:46.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:46 smithi181 ceph-mon[196979]: failed to encode map e282 with expected crc 2023-10-09T04:16:46.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:46 smithi181 ceph-mon[196979]: failed to encode map e282 with expected crc 2023-10-09T04:16:46.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T04:16:46.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:46 smithi181 ceph-mon[196979]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T04:16:46.973 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:47.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:47.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:47.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:47.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:47.266 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:47.662 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:16:47.777 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:47.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:47.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:47.923 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[213841]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T04:16:47.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[213841]: failed to encode map e283 with expected crc 2023-10-09T04:16:47.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[213841]: failed to encode map e283 with expected crc 2023-10-09T04:16:47.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2782/117903 objects degraded (2.360%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:47.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[221067]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T04:16:47.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[221067]: failed to encode map e283 with expected crc 2023-10-09T04:16:47.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[221067]: failed to encode map e283 with expected crc 2023-10-09T04:16:47.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2782/117903 objects degraded (2.360%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:47.967 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:48.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:47 smithi181 ceph-mon[196979]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T04:16:48.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:47 smithi181 ceph-mon[196979]: failed to encode map e283 with expected crc 2023-10-09T04:16:48.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:47 smithi181 ceph-mon[196979]: failed to encode map e283 with expected crc 2023-10-09T04:16:48.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2782/117903 objects degraded (2.360%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:48.136 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:48.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:48.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:48.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:48.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:48.325 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:48.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:48.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:48.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:48.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:48.599 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:48.689 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:48.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:48.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:48.846 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:48.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:48.919 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:48.919 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:48 smithi181 ceph-mon[196979]: pgmap v775: 109 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 2782/112203 objects degraded (2.479%); 11005/112203 objects misplaced (9.808%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:16:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:48.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:49.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:49.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:48 smithi163 ceph-mon[213841]: pgmap v775: 109 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 2782/112203 objects degraded (2.479%); 11005/112203 objects misplaced (9.808%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:16:49.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:48 smithi163 ceph-mon[221067]: pgmap v775: 109 pgs: 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 2782/112203 objects degraded (2.479%); 11005/112203 objects misplaced (9.808%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:16:49.106 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:49.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:49.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:49.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:49.829 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:49.842 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:16:49.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:49.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:49.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2455251767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:49.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:49.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:49.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2455251767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:49.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:49.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:50.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:50.117 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:50.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2455251767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:16:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:16:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:50.623 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:50.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:50.651 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:16:50.651 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:16:50.651 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T04:16:50.651 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:16:50.651 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:16:50.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T04:16:50.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:50.802 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:50.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:50.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:50 smithi181 ceph-mon[196979]: pgmap v776: 109 pgs: 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 2782/109863 objects degraded (2.532%); 11005/109863 objects misplaced (10.017%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:16:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:50 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:50 smithi181 ceph-mon[196979]: failed to encode map e284 with expected crc 2023-10-09T04:16:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2455251767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:50 smithi181 ceph-mon[196979]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T04:16:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:50 smithi181 ceph-mon[196979]: failed to encode map e284 with expected crc 2023-10-09T04:16:51.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:16:51.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[221067]: pgmap v776: 109 pgs: 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 2782/109863 objects degraded (2.532%); 11005/109863 objects misplaced (10.017%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:16:51.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:51.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[221067]: failed to encode map e284 with expected crc 2023-10-09T04:16:51.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2455251767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:51.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[221067]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T04:16:51.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[221067]: failed to encode map e284 with expected crc 2023-10-09T04:16:51.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[213841]: pgmap v776: 109 pgs: 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 94 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 2782/109863 objects degraded (2.532%); 11005/109863 objects misplaced (10.017%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:16:51.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:51.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[213841]: failed to encode map e284 with expected crc 2023-10-09T04:16:51.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2455251767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:16:51.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[213841]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T04:16:51.066 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:50 smithi163 ceph-mon[213841]: failed to encode map e284 with expected crc 2023-10-09T04:16:51.067 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:16:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:16:50] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.43.0" 2023-10-09T04:16:51.075 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:51.238 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:16:51.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:16:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:51.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:16:51.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:51.767 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:51.821 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:16:51.825 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/38865714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:51.862 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:16:51.903 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:16:51.937 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:16:51.976 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:16:52.010 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:16:52.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/38865714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:52.047 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:16:52.081 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:16:52.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/38865714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.103 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:16:52.290 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:52.292 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:16:52.292 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:16:52.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:52.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:52.394 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:52.445 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:53.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:52 smithi163 ceph-mon[221067]: pgmap v778: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 2782/102993 objects degraded (2.701%); 9937/102993 objects misplaced (9.648%); 9.6 MiB/s, 152 objects/s recovering 2023-10-09T04:16:53.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2782/102993 objects degraded (2.701%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:53.132 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:52 smithi163 ceph-mon[213841]: pgmap v778: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 2782/102993 objects degraded (2.701%); 9937/102993 objects misplaced (9.648%); 9.6 MiB/s, 152 objects/s recovering 2023-10-09T04:16:53.132 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2782/102993 objects degraded (2.701%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:53.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:52 smithi181 ceph-mon[196979]: pgmap v778: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 2782/102993 objects degraded (2.701%); 9937/102993 objects misplaced (9.648%); 9.6 MiB/s, 152 objects/s recovering 2023-10-09T04:16:53.230 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2782/102993 objects degraded (2.701%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:53.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:16:53.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:53.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:53.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:16:53.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:53.835 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:53.897 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:53.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:54.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:54.013 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:54.071 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:54.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:54.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:54.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:16:54.480 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:54.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:16:54.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:54.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:16:54.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:54.844 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:54.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:54.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:55.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:55.050 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:54 smithi163 ceph-mon[221067]: pgmap v779: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 2782/102993 objects degraded (2.701%); 9937/102993 objects misplaced (9.648%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:16:55.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:54 smithi163 ceph-mon[213841]: pgmap v779: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 2782/102993 objects degraded (2.701%); 9937/102993 objects misplaced (9.648%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:16:55.053 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:55.095 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:55.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:55.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:55.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:55.241 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:54 smithi181 ceph-mon[196979]: pgmap v779: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 2782/102993 objects degraded (2.701%); 9937/102993 objects misplaced (9.648%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:16:55.244 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:55.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:55.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:55.424 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:55.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:16:55.474 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:55.548 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:55.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:55.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:55.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:55.715 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:55.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:55.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:55.866 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:55 smithi181 ceph-mon[196979]: pgmap v780: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 2782/99624 objects degraded (2.792%); 9726/99624 objects misplaced (9.763%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T04:16:55.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:55.941 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:56.004 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:56.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:56.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:56.119 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:55 smithi163 ceph-mon[213841]: pgmap v780: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 2782/99624 objects degraded (2.792%); 9726/99624 objects misplaced (9.763%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T04:16:56.119 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:55 smithi163 ceph-mon[221067]: pgmap v780: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 2782/99624 objects degraded (2.792%); 9726/99624 objects misplaced (9.763%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T04:16:56.179 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:56.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:56.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:56.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:16:56.460 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:56.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:16:56.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:56.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:16:56.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:56.713 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:56.763 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:56.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:56.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:56.897 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:16:56.915 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:57.016 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/730527241' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:57.016 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:56 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:57.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:57.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:57.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:16:57.223 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:57.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/730527241' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:57.261 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:56 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:57.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/730527241' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:57.261 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:56 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:16:57.297 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:16:57.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:57.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:16:57.334 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:16:57.334 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:16:57.335 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:16:57.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:57.448 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:57.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:57.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:57.891 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:16:57.932 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:57.952 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:58.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: failed to encode map e285 with expected crc 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: failed to encode map e285 with expected crc 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2782/99624 objects degraded (2.792%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[213841]: pgmap v782: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 2782/93945 objects degraded (2.961%); 8808/93945 objects misplaced (9.376%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:16:58.045 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T04:16:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: failed to encode map e285 with expected crc 2023-10-09T04:16:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: failed to encode map e285 with expected crc 2023-10-09T04:16:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2782/99624 objects degraded (2.792%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:58.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:57 smithi163 ceph-mon[221067]: pgmap v782: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 2782/93945 objects degraded (2.961%); 8808/93945 objects misplaced (9.376%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:16:58.208 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:16:58.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:16:58.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:16:58.211 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T04:16:58.212 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: failed to encode map e285 with expected crc 2023-10-09T04:16:58.212 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: failed to encode map e285 with expected crc 2023-10-09T04:16:58.212 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2782/99624 objects degraded (2.792%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:16:58.212 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:57 smithi181 ceph-mon[196979]: pgmap v782: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 104 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 2782/93945 objects degraded (2.961%); 8808/93945 objects misplaced (9.376%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:16:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:58.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:16:58.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:16:58.371 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:58.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:16:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:58.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:16:58.452 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:16:58.452 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:16:58.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:16:58.570 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:58.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:16:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:58.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:16:58.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:58.779 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:59.586 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:16:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:16:59.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:16:59.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:16:59.727 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:59 smithi163 ceph-mon[221067]: failed to encode map e286 with expected crc 2023-10-09T04:16:59.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:59 smithi163 ceph-mon[221067]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T04:16:59.728 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:16:59 smithi163 ceph-mon[221067]: failed to encode map e286 with expected crc 2023-10-09T04:16:59.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:59 smithi163 ceph-mon[213841]: failed to encode map e286 with expected crc 2023-10-09T04:16:59.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:59 smithi163 ceph-mon[213841]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T04:16:59.728 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:16:59 smithi163 ceph-mon[213841]: failed to encode map e286 with expected crc 2023-10-09T04:16:59.748 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:16:59.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:59 smithi181 ceph-mon[196979]: failed to encode map e286 with expected crc 2023-10-09T04:16:59.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:59 smithi181 ceph-mon[196979]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T04:16:59.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:16:59 smithi181 ceph-mon[196979]: failed to encode map e286 with expected crc 2023-10-09T04:16:59.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:17:00.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:00.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:00.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:00.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:00.412 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:00.473 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:00.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:00.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:00.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:00.634 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:00.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:00.713 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:17:00 smithi181 conmon[175350]: logger=cleanup t=2023-10-09T04:17:00.596721642Z level=info msg="Completed cleanup jobs" duration=66.216406ms 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: pgmap v785: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 104 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1046/91794 objects degraded (1.140%); 8779/91794 objects misplaced (9.564%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: 1.8 scrub starts 2023-10-09T04:17:00.714 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:00 smithi181 ceph-mon[196979]: 1.8 scrub ok 2023-10-09T04:17:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:00.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:00.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:00.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:17:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:17:00] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.43.0" 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: pgmap v785: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 104 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1046/91794 objects degraded (1.140%); 8779/91794 objects misplaced (9.564%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: 1.8 scrub starts 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[221067]: 1.8 scrub ok 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T04:17:00.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: failed to encode map e287 with expected crc 2023-10-09T04:17:00.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: pgmap v785: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 104 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1046/91794 objects degraded (1.140%); 8779/91794 objects misplaced (9.564%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T04:17:00.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: 1.8 scrub starts 2023-10-09T04:17:00.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:00 smithi163 ceph-mon[213841]: 1.8 scrub ok 2023-10-09T04:17:00.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:17:00.903 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:01.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:01.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:01.087 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:01.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:01.188 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:01.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:01.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:01.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:01.395 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:01.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:01.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:01.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:01.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:01.660 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:01 smithi163 ceph-mon[213841]: 1.9 scrub starts 2023-10-09T04:17:01.661 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:01 smithi163 ceph-mon[213841]: 1.9 scrub ok 2023-10-09T04:17:01.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:01 smithi163 ceph-mon[221067]: 1.9 scrub starts 2023-10-09T04:17:01.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:01 smithi163 ceph-mon[221067]: 1.9 scrub ok 2023-10-09T04:17:01.698 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:01.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:01 smithi181 ceph-mon[196979]: 1.9 scrub starts 2023-10-09T04:17:01.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:01 smithi181 ceph-mon[196979]: 1.9 scrub ok 2023-10-09T04:17:01.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:01.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:01.865 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:17:01.918 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:17:01.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:01.971 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:17:02.008 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:17:02.028 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:02.049 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:17:02.093 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:17:02.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:02.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:02.146 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:17:02.181 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:17:02.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:02.290 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:02.374 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:02.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:02.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:02.525 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[221067]: pgmap v786: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1046/86289 objects degraded (1.212%); 8225/86289 objects misplaced (9.532%); 6.8 MiB/s, 140 objects/s recovering 2023-10-09T04:17:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3604306669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[221067]: 1.a scrub starts 2023-10-09T04:17:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[221067]: 1.a scrub ok 2023-10-09T04:17:02.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/91794 objects degraded (1.140%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:02.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[213841]: pgmap v786: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1046/86289 objects degraded (1.212%); 8225/86289 objects misplaced (9.532%); 6.8 MiB/s, 140 objects/s recovering 2023-10-09T04:17:02.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3604306669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:02.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[213841]: 1.a scrub starts 2023-10-09T04:17:02.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[213841]: 1.a scrub ok 2023-10-09T04:17:02.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/91794 objects degraded (1.140%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:02.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:02.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:02.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:02.750 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:02 smithi181 ceph-mon[196979]: pgmap v786: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1046/86289 objects degraded (1.212%); 8225/86289 objects misplaced (9.532%); 6.8 MiB/s, 140 objects/s recovering 2023-10-09T04:17:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3604306669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:02 smithi181 ceph-mon[196979]: 1.a scrub starts 2023-10-09T04:17:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:02 smithi181 ceph-mon[196979]: 1.a scrub ok 2023-10-09T04:17:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/91794 objects degraded (1.140%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:03.273 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:17:03.273 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:17:03.273 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T04:17:03.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:17:03.490 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:03.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:03.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:03.627 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:03.685 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:03.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:03.736 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:03.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:03.825 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:03.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:03.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:03.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:03.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:04.010 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:04.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:04.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:04.177 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:04.215 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:04.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:04.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:04.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:04.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:04.425 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:04.431 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:04.495 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:04.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:04.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:04.633 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:04.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:04.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:04 smithi181 ceph-mon[196979]: pgmap v787: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 1046/86289 objects degraded (1.212%); 8225/86289 objects misplaced (9.532%); 1.2 MiB/s, 47 objects/s recovering 2023-10-09T04:17:04.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:04.805 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:04.838 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:04.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:04 smithi163 ceph-mon[221067]: pgmap v787: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 1046/86289 objects degraded (1.212%); 8225/86289 objects misplaced (9.532%); 1.2 MiB/s, 47 objects/s recovering 2023-10-09T04:17:04.897 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:04.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:04 smithi163 ceph-mon[213841]: pgmap v787: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 1046/86289 objects degraded (1.212%); 8225/86289 objects misplaced (9.532%); 1.2 MiB/s, 47 objects/s recovering 2023-10-09T04:17:04.897 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:04.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:04.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:04.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:05.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:05.090 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:05.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:05.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:05.300 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:05.329 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:05.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:05.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:05.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:05.532 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:05.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:05.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:05.713 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:05.755 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:17:05.755 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:05.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:05.827 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/172722535' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:05.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:05.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:05 smithi181 ceph-mon[196979]: 1.3 scrub starts 2023-10-09T04:17:05.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:05 smithi181 ceph-mon[196979]: 1.3 scrub ok 2023-10-09T04:17:05.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:05.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:05.863 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/172722535' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[213841]: 1.3 scrub starts 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[213841]: 1.3 scrub ok 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/172722535' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[221067]: 1.3 scrub starts 2023-10-09T04:17:05.864 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:05 smithi163 ceph-mon[221067]: 1.3 scrub ok 2023-10-09T04:17:05.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:05.970 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:06.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:17:06.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:06.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:06.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:06.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:06.228 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:06.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:06.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:06.983 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[213841]: pgmap v788: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 1046/83370 objects degraded (1.255%); 7832/83370 objects misplaced (9.394%); 2.7 MiB/s, 66 objects/s recovering 2023-10-09T04:17:06.983 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:17:06.983 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:06.983 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[213841]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[213841]: failed to encode map e288 with expected crc 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[213841]: failed to encode map e288 with expected crc 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[221067]: pgmap v788: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 1046/83370 objects degraded (1.255%); 7832/83370 objects misplaced (9.394%); 2.7 MiB/s, 66 objects/s recovering 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[221067]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[221067]: failed to encode map e288 with expected crc 2023-10-09T04:17:06.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:06 smithi163 ceph-mon[221067]: failed to encode map e288 with expected crc 2023-10-09T04:17:07.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:06 smithi181 ceph-mon[196979]: pgmap v788: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 437 op/s; 1046/83370 objects degraded (1.255%); 7832/83370 objects misplaced (9.394%); 2.7 MiB/s, 66 objects/s recovering 2023-10-09T04:17:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:06 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:17:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:06 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:06 smithi181 ceph-mon[196979]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T04:17:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:06 smithi181 ceph-mon[196979]: failed to encode map e288 with expected crc 2023-10-09T04:17:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:06 smithi181 ceph-mon[196979]: failed to encode map e288 with expected crc 2023-10-09T04:17:07.047 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:07.121 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:07.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:07.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:07.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:07.783 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:07.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:07.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:07.947 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:07 smithi163 ceph-mon[213841]: 1.6 scrub starts 2023-10-09T04:17:07.947 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:07 smithi163 ceph-mon[213841]: 1.6 scrub ok 2023-10-09T04:17:07.947 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/83370 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:07.961 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:07 smithi163 ceph-mon[221067]: 1.6 scrub starts 2023-10-09T04:17:07.962 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:07 smithi163 ceph-mon[221067]: 1.6 scrub ok 2023-10-09T04:17:07.962 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/83370 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:07.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:07 smithi181 ceph-mon[196979]: 1.6 scrub starts 2023-10-09T04:17:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:07 smithi181 ceph-mon[196979]: 1.6 scrub ok 2023-10-09T04:17:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/83370 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:08.044 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:08.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:08.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:08.324 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:17:08.324 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:17:08.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.324 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:156904 8.2 e288/266 8:4668adad:::benchmark_data_smithi163_404719_object66320:head v 288'10027, mlcod=288'10025)", 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:08.315922+0000", 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 9.9491e-05, 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013485299999999999, 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315922+0000", 2023-10-09T04:17:08.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315922+0000", 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315920+0000", 2023-10-09T04:17:08.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315930+0000", 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0197000000000001e-05 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:08.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315945+0000", 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4222999999999999e-05 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315954+0000", 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0380000000000005e-06 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.316000+0000", 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5927000000000003e-05 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.316027+0000", 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7895e-05 2023-10-09T04:17:08.329 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:156906 8.2 e288/266)", 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:08.315969+0000", 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.2237999999999997e-05, 2023-10-09T04:17:08.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018804600000000001, 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315969+0000", 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315969+0000", 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315969+0000", 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315973+0000", 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9369999999999997e-06 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315978+0000", 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5790000000000003e-06 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:08.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:08.315983+0000", 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2200000000000003e-06 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 2 2023-10-09T04:17:08.334 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:17:08.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:08.424 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:08.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:08.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:08.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:08.678 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:08.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:08.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:08.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:08.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:08.963 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:09.016 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:17:09.018 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:08 smithi181 ceph-mon[196979]: pgmap v790: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 1046/78069 objects degraded (1.340%); 7500/78069 objects misplaced (9.607%); 2.2 MiB/s, 58 objects/s recovering 2023-10-09T04:17:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:09.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:09.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:08 smithi163 ceph-mon[213841]: pgmap v790: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 1046/78069 objects degraded (1.340%); 7500/78069 objects misplaced (9.607%); 2.2 MiB/s, 58 objects/s recovering 2023-10-09T04:17:09.065 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:08 smithi163 ceph-mon[221067]: pgmap v790: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 1046/78069 objects degraded (1.340%); 7500/78069 objects misplaced (9.607%); 2.2 MiB/s, 58 objects/s recovering 2023-10-09T04:17:09.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:09.158 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:09.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:09.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:09.242 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:17:09.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:09.391 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:09.699 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:09.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:09.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:09.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:09.843 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:09.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:17:09.904 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:09.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:10.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:10.095 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:10.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:17:10.461 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:10.479 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":790,"stamp":"2023-10-09T04:17:07.635119+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1704419077,"num_objects":26023,"num_object_clones":0,"num_object_copies":78069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":7500,"num_objects_unfound":0,"num_objects_dirty":26023,"num_whiteouts":0,"num_read":351,"num_read_kb":645,"num_write":155395,"num_write_kb":5798900,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13988,"num_bytes_recovered":891284957,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130967,"ondisk_log_size":130967,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":333,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8883012,"kb_used_data":5731332,"kb_used_omap":126,"kb_used_meta":3150913,"kb_avail":741078204,"statfs":{"total":767960285184,"available":758864080896,"internally_reserved":0,"allocated":5868883968,"data_stored":5860643541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3226535547},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,1,7,4,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-299808947,"num_objects":-4575,"num_object_clones":0,"num_object_copies":-13725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1279,"num_objects_unfound":0,"num_objects_dirty":-4575,"num_whiteouts":0,"num_read":-150,"num_read_kb":-280,"num_write":4505,"num_write_kb":-730,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":480,"num_bytes_recovered":18796423,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.177890"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480801+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:16:58.480801+0000","last_peered":"2023-10-09T04:16:58.480801+0000","last_clean":"2023-10-09T04:16:58.480801+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:16:58.480801+0000","last_undegraded":"2023-10-09T04:16:58.480801+0000","last_fullsized":"2023-10-09T04:16:58.480801+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":243,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480042+0000","last_change":"2023-10-09T04:15:25.414174+0000","last_active":"2023-10-09T04:16:58.480042+0000","last_peered":"2023-10-09T04:16:58.480042+0000","last_clean":"2023-10-09T04:16:58.480042+0000","last_became_active":"2023-10-09T04:15:25.413807+0000","last_became_peered":"2023-10-09T04:15:25.413807+0000","last_unstale":"2023-10-09T04:16:58.480042+0000","last_undegraded":"2023-10-09T04:16:58.480042+0000","last_fullsized":"2023-10-09T04:16:58.480042+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:01.968150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":258,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.476947+0000","last_change":"2023-10-09T04:15:25.414811+0000","last_active":"2023-10-09T04:16:59.476947+0000","last_peered":"2023-10-09T04:16:59.476947+0000","last_clean":"2023-10-09T04:16:59.476947+0000","last_became_active":"2023-10-09T04:15:25.414562+0000","last_became_peered":"2023-10-09T04:15:25.414562+0000","last_unstale":"2023-10-09T04:16:59.476947+0000","last_undegraded":"2023-10-09T04:16:59.476947+0000","last_fullsized":"2023-10-09T04:16:59.476947+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:10.270201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.657465+0000","last_change":"2023-10-09T04:16:24.469036+0000","last_active":"2023-10-09T04:16:59.657465+0000","last_peered":"2023-10-09T04:16:59.657465+0000","last_clean":"2023-10-09T04:16:59.657465+0000","last_became_active":"2023-10-09T04:16:24.468649+0000","last_became_peered":"2023-10-09T04:16:24.468649+0000","last_unstale":"2023-10-09T04:16:59.657465+0000","last_undegraded":"2023-10-09T04:16:59.657465+0000","last_fullsized":"2023-10-09T04:16:59.657465+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:34:53.505550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":244,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499655+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:16:59.499655+0000","last_peered":"2023-10-09T04:16:59.499655+0000","last_clean":"2023-10-09T04:16:59.499655+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:16:59.499655+0000","last_undegraded":"2023-10-09T04:16:59.499655+0000","last_fullsized":"2023-10-09T04:16:59.499655+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":221,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463598+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:16:59.463598+0000","last_peered":"2023-10-09T04:16:59.463598+0000","last_clean":"2023-10-09T04:16:59.463598+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:16:59.463598+0000","last_undegraded":"2023-10-09T04:16:59.463598+0000","last_fullsized":"2023-10-09T04:16:59.463598+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":245,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.657276+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:16:59.657276+0000","last_peered":"2023-10-09T04:16:59.657276+0000","last_clean":"2023-10-09T04:16:59.657276+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:16:59.657276+0000","last_undegraded":"2023-10-09T04:16:59.657276+0000","last_fullsized":"2023-10-09T04:16:59.657276+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":268,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.488727+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:16:59.488727+0000","last_peered":"2023-10-09T04:16:59.488727+0000","last_clean":"2023-10-09T04:16:59.488727+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:16:59.488727+0000","last_undegraded":"2023-10-09T04:16:59.488727+0000","last_fullsized":"2023-10-09T04:16:59.488727+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":649,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479833+0000","last_change":"2023-10-09T04:16:46.787881+0000","last_active":"2023-10-09T04:16:58.479833+0000","last_peered":"2023-10-09T04:16:58.479833+0000","last_clean":"2023-10-09T04:16:58.479833+0000","last_became_active":"2023-10-09T04:16:46.787580+0000","last_became_peered":"2023-10-09T04:16:46.787580+0000","last_unstale":"2023-10-09T04:16:58.479833+0000","last_undegraded":"2023-10-09T04:16:58.479833+0000","last_fullsized":"2023-10-09T04:16:58.479833+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463459+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:16:59.463459+0000","last_peered":"2023-10-09T04:16:59.463459+0000","last_clean":"2023-10-09T04:16:59.463459+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:16:59.463459+0000","last_undegraded":"2023-10-09T04:16:59.463459+0000","last_fullsized":"2023-10-09T04:16:59.463459+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":106,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.488700+0000","last_change":"2023-10-09T04:13:46.101209+0000","last_active":"2023-10-09T04:16:59.488700+0000","last_peered":"2023-10-09T04:16:59.488700+0000","last_clean":"2023-10-09T04:16:59.488700+0000","last_became_active":"2023-10-09T04:13:46.100828+0000","last_became_peered":"2023-10-09T04:13:46.100828+0000","last_unstale":"2023-10-09T04:16:59.488700+0000","last_undegraded":"2023-10-09T04:16:59.488700+0000","last_fullsized":"2023-10-09T04:16:59.488700+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:59.155026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":281,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481781+0000","last_change":"2023-10-09T04:15:25.545630+0000","last_active":"2023-10-09T04:16:58.481781+0000","last_peered":"2023-10-09T04:16:58.481781+0000","last_clean":"2023-10-09T04:16:58.481781+0000","last_became_active":"2023-10-09T04:15:25.410551+0000","last_became_peered":"2023-10-09T04:15:25.410551+0000","last_unstale":"2023-10-09T04:16:58.481781+0000","last_undegraded":"2023-10-09T04:16:58.481781+0000","last_fullsized":"2023-10-09T04:16:58.481781+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:15.300784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.476060+0000","last_change":"2023-10-09T04:16:24.482108+0000","last_active":"2023-10-09T04:16:58.476060+0000","last_peered":"2023-10-09T04:16:58.476060+0000","last_clean":"2023-10-09T04:16:58.476060+0000","last_became_active":"2023-10-09T04:16:24.481813+0000","last_became_peered":"2023-10-09T04:16:24.481813+0000","last_unstale":"2023-10-09T04:16:58.476060+0000","last_undegraded":"2023-10-09T04:16:58.476060+0000","last_fullsized":"2023-10-09T04:16:58.476060+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:51.848459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":41,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481009+0000","last_change":"2023-10-09T04:15:25.523555+0000","last_active":"2023-10-09T04:16:58.481009+0000","last_peered":"2023-10-09T04:16:58.481009+0000","last_clean":"2023-10-09T04:16:58.481009+0000","last_became_active":"2023-10-09T04:15:25.408994+0000","last_became_peered":"2023-10-09T04:15:25.408994+0000","last_unstale":"2023-10-09T04:16:58.481009+0000","last_undegraded":"2023-10-09T04:16:58.481009+0000","last_fullsized":"2023-10-09T04:16:58.481009+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:39.516242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481730+0000","last_change":"2023-10-09T04:15:25.476381+0000","last_active":"2023-10-09T04:16:58.481730+0000","last_peered":"2023-10-09T04:16:58.481730+0000","last_clean":"2023-10-09T04:16:58.481730+0000","last_became_active":"2023-10-09T04:15:25.475863+0000","last_became_peered":"2023-10-09T04:15:25.475863+0000","last_unstale":"2023-10-09T04:16:58.481730+0000","last_undegraded":"2023-10-09T04:16:58.481730+0000","last_fullsized":"2023-10-09T04:16:58.481730+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:46.820781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":648,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479916+0000","last_change":"2023-10-09T04:16:46.781567+0000","last_active":"2023-10-09T04:16:58.479916+0000","last_peered":"2023-10-09T04:16:58.479916+0000","last_clean":"2023-10-09T04:16:58.479916+0000","last_became_active":"2023-10-09T04:16:46.781202+0000","last_became_peered":"2023-10-09T04:16:46.781202+0000","last_unstale":"2023-10-09T04:16:58.479916+0000","last_undegraded":"2023-10-09T04:16:58.479916+0000","last_fullsized":"2023-10-09T04:16:58.479916+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":280,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479215+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:16:58.479215+0000","last_peered":"2023-10-09T04:16:58.479215+0000","last_clean":"2023-10-09T04:16:58.479215+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:16:58.479215+0000","last_undegraded":"2023-10-09T04:16:58.479215+0000","last_fullsized":"2023-10-09T04:16:58.479215+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479574+0000","last_change":"2023-10-09T04:16:24.616676+0000","last_active":"2023-10-09T04:16:58.479574+0000","last_peered":"2023-10-09T04:16:58.479574+0000","last_clean":"2023-10-09T04:16:58.479574+0000","last_became_active":"2023-10-09T04:16:24.616320+0000","last_became_peered":"2023-10-09T04:16:24.616320+0000","last_unstale":"2023-10-09T04:16:58.479574+0000","last_undegraded":"2023-10-09T04:16:58.479574+0000","last_fullsized":"2023-10-09T04:16:58.479574+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:32.174105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":650,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.478390+0000","last_change":"2023-10-09T04:16:46.782428+0000","last_active":"2023-10-09T04:16:58.478390+0000","last_peered":"2023-10-09T04:16:58.478390+0000","last_clean":"2023-10-09T04:16:58.478390+0000","last_became_active":"2023-10-09T04:16:46.779655+0000","last_became_peered":"2023-10-09T04:16:46.779655+0000","last_unstale":"2023-10-09T04:16:58.478390+0000","last_undegraded":"2023-10-09T04:16:58.478390+0000","last_fullsized":"2023-10-09T04:16:58.478390+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499758+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:16:59.499758+0000","last_peered":"2023-10-09T04:16:59.499758+0000","last_clean":"2023-10-09T04:16:59.499758+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:16:59.499758+0000","last_undegraded":"2023-10-09T04:16:59.499758+0000","last_fullsized":"2023-10-09T04:16:59.499758+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":246,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463583+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:16:59.463583+0000","last_peered":"2023-10-09T04:16:59.463583+0000","last_clean":"2023-10-09T04:16:59.463583+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:16:59.463583+0000","last_undegraded":"2023-10-09T04:16:59.463583+0000","last_fullsized":"2023-10-09T04:16:59.463583+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":649,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479297+0000","last_change":"2023-10-09T04:16:46.781804+0000","last_active":"2023-10-09T04:16:58.479297+0000","last_peered":"2023-10-09T04:16:58.479297+0000","last_clean":"2023-10-09T04:16:58.479297+0000","last_became_active":"2023-10-09T04:16:46.778588+0000","last_became_peered":"2023-10-09T04:16:46.778588+0000","last_unstale":"2023-10-09T04:16:58.479297+0000","last_undegraded":"2023-10-09T04:16:58.479297+0000","last_fullsized":"2023-10-09T04:16:58.479297+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480923+0000","last_change":"2023-10-09T04:15:25.409247+0000","last_active":"2023-10-09T04:16:58.480923+0000","last_peered":"2023-10-09T04:16:58.480923+0000","last_clean":"2023-10-09T04:16:58.480923+0000","last_became_active":"2023-10-09T04:15:25.408950+0000","last_became_peered":"2023-10-09T04:15:25.408950+0000","last_unstale":"2023-10-09T04:16:58.480923+0000","last_undegraded":"2023-10-09T04:16:58.480923+0000","last_fullsized":"2023-10-09T04:16:58.480923+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:46.455335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.657694+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:16:59.657694+0000","last_peered":"2023-10-09T04:16:59.657694+0000","last_clean":"2023-10-09T04:16:59.657694+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:16:59.657694+0000","last_undegraded":"2023-10-09T04:16:59.657694+0000","last_fullsized":"2023-10-09T04:16:59.657694+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":646,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.478794+0000","last_change":"2023-10-09T04:16:46.778289+0000","last_active":"2023-10-09T04:16:58.478794+0000","last_peered":"2023-10-09T04:16:58.478794+0000","last_clean":"2023-10-09T04:16:58.478794+0000","last_became_active":"2023-10-09T04:16:46.777951+0000","last_became_peered":"2023-10-09T04:16:46.777951+0000","last_unstale":"2023-10-09T04:16:58.478794+0000","last_undegraded":"2023-10-09T04:16:58.478794+0000","last_fullsized":"2023-10-09T04:16:58.478794+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":200,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499652+0000","last_change":"2023-10-09T04:16:24.625253+0000","last_active":"2023-10-09T04:16:59.499652+0000","last_peered":"2023-10-09T04:16:59.499652+0000","last_clean":"2023-10-09T04:16:59.499652+0000","last_became_active":"2023-10-09T04:16:24.617700+0000","last_became_peered":"2023-10-09T04:16:24.617700+0000","last_unstale":"2023-10-09T04:16:59.499652+0000","last_undegraded":"2023-10-09T04:16:59.499652+0000","last_fullsized":"2023-10-09T04:16:59.499652+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:29.757285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":74,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.485772+0000","last_change":"2023-10-09T04:16:24.474334+0000","last_active":"2023-10-09T04:16:58.485772+0000","last_peered":"2023-10-09T04:16:58.485772+0000","last_clean":"2023-10-09T04:16:58.485772+0000","last_became_active":"2023-10-09T04:16:24.474003+0000","last_became_peered":"2023-10-09T04:16:24.474003+0000","last_unstale":"2023-10-09T04:16:58.485772+0000","last_undegraded":"2023-10-09T04:16:58.485772+0000","last_fullsized":"2023-10-09T04:16:58.485772+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:38.410877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":646,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481313+0000","last_change":"2023-10-09T04:16:46.780829+0000","last_active":"2023-10-09T04:16:58.481313+0000","last_peered":"2023-10-09T04:16:58.481313+0000","last_clean":"2023-10-09T04:16:58.481313+0000","last_became_active":"2023-10-09T04:16:46.780531+0000","last_became_peered":"2023-10-09T04:16:46.780531+0000","last_unstale":"2023-10-09T04:16:58.481313+0000","last_undegraded":"2023-10-09T04:16:58.481313+0000","last_fullsized":"2023-10-09T04:16:58.481313+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"288'9722","reported_seq":15682,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.740874+0000","last_change":"2023-10-09T04:16:59.486729+0000","last_active":"2023-10-09T04:17:05.740874+0000","last_peered":"2023-10-09T04:17:05.740874+0000","last_clean":"2023-10-09T04:17:05.740874+0000","last_became_active":"2023-10-09T04:16:59.466744+0000","last_became_peered":"2023-10-09T04:16:59.466744+0000","last_unstale":"2023-10-09T04:17:05.740874+0000","last_undegraded":"2023-10-09T04:17:05.740874+0000","last_fullsized":"2023-10-09T04:17:05.740874+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9722,"log_dups_size":0,"ondisk_log_size":9722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:35.727516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102498304,"num_objects":1564,"num_object_clones":0,"num_object_copies":4692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9722,"num_write_kb":361152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":180289536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":275,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479702+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:16:58.479702+0000","last_peered":"2023-10-09T04:16:58.479702+0000","last_clean":"2023-10-09T04:16:58.479702+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:16:58.479702+0000","last_undegraded":"2023-10-09T04:16:58.479702+0000","last_fullsized":"2023-10-09T04:16:58.479702+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":229,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463144+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:16:59.463144+0000","last_peered":"2023-10-09T04:16:59.463144+0000","last_clean":"2023-10-09T04:16:59.463144+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:16:59.463144+0000","last_undegraded":"2023-10-09T04:16:59.463144+0000","last_fullsized":"2023-10-09T04:16:59.463144+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":650,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479760+0000","last_change":"2023-10-09T04:16:46.798945+0000","last_active":"2023-10-09T04:16:58.479760+0000","last_peered":"2023-10-09T04:16:58.479760+0000","last_clean":"2023-10-09T04:16:58.479760+0000","last_became_active":"2023-10-09T04:16:46.798622+0000","last_became_peered":"2023-10-09T04:16:46.798622+0000","last_unstale":"2023-10-09T04:16:58.479760+0000","last_undegraded":"2023-10-09T04:16:58.479760+0000","last_fullsized":"2023-10-09T04:16:58.479760+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":16,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"287'9711","reported_seq":15576,"reported_epoch":287,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:17:05.222958+0000","last_change":"2023-10-09T04:16:57.541338+0000","last_active":"2023-10-09T04:17:05.222958+0000","last_peered":"2023-10-09T04:17:05.222958+0000","last_clean":"2023-10-09T04:15:24.138637+0000","last_became_active":"2023-10-09T04:15:26.489098+0000","last_became_peered":"2023-10-09T04:15:26.489098+0000","last_unstale":"2023-10-09T04:17:05.222958+0000","last_undegraded":"2023-10-09T04:17:05.222958+0000","last_fullsized":"2023-10-09T04:17:05.222958+0000","mapping_epoch":266,"log_start":"254'4118","ondisk_log_start":"254'4118","created":241,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5593,"log_dups_size":0,"ondisk_log_size":5593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:37.061888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105840640,"num_objects":1615,"num_object_clones":0,"num_object_copies":4845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1599,"num_objects_unfound":0,"num_objects_dirty":1615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9711,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2887,"num_bytes_recovered":189202432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,7,0],"avail_no_missing":["6","0","7"],"object_location_counts":[{"shards":"0,6,7","objects":1615}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":279,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479355+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:16:58.479355+0000","last_peered":"2023-10-09T04:16:58.479355+0000","last_clean":"2023-10-09T04:16:58.479355+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:16:58.479355+0000","last_undegraded":"2023-10-09T04:16:58.479355+0000","last_fullsized":"2023-10-09T04:16:58.479355+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":41,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481074+0000","last_change":"2023-10-09T04:16:24.476123+0000","last_active":"2023-10-09T04:16:58.481074+0000","last_peered":"2023-10-09T04:16:58.481074+0000","last_clean":"2023-10-09T04:16:58.481074+0000","last_became_active":"2023-10-09T04:16:24.475818+0000","last_became_peered":"2023-10-09T04:16:24.475818+0000","last_unstale":"2023-10-09T04:16:58.481074+0000","last_undegraded":"2023-10-09T04:16:58.481074+0000","last_fullsized":"2023-10-09T04:16:58.481074+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:57.051534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":649,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479613+0000","last_change":"2023-10-09T04:16:46.779365+0000","last_active":"2023-10-09T04:16:58.479613+0000","last_peered":"2023-10-09T04:16:58.479613+0000","last_clean":"2023-10-09T04:16:58.479613+0000","last_became_active":"2023-10-09T04:16:46.779035+0000","last_became_peered":"2023-10-09T04:16:46.779035+0000","last_unstale":"2023-10-09T04:16:58.479613+0000","last_undegraded":"2023-10-09T04:16:58.479613+0000","last_fullsized":"2023-10-09T04:16:58.479613+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.6","version":"288'9743","reported_seq":9800,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:06.520798+0000","last_change":"2023-10-09T04:13:42.646173+0000","last_active":"2023-10-09T04:17:06.520798+0000","last_peered":"2023-10-09T04:17:06.520798+0000","last_clean":"2023-10-09T04:17:06.520798+0000","last_became_active":"2023-10-09T04:13:42.645934+0000","last_became_peered":"2023-10-09T04:13:42.645934+0000","last_unstale":"2023-10-09T04:17:06.520798+0000","last_undegraded":"2023-10-09T04:17:06.520798+0000","last_fullsized":"2023-10-09T04:17:06.520798+0000","mapping_epoch":241,"log_start":"254'4245","ondisk_log_start":"254'4245","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.323104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105316352,"num_objects":1607,"num_object_clones":0,"num_object_copies":4821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9743,"num_write_kb":363200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":230,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481561+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:16:58.481561+0000","last_peered":"2023-10-09T04:16:58.481561+0000","last_clean":"2023-10-09T04:16:58.481561+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:16:58.481561+0000","last_undegraded":"2023-10-09T04:16:58.481561+0000","last_fullsized":"2023-10-09T04:16:58.481561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":208,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479805+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:16:58.479805+0000","last_peered":"2023-10-09T04:16:58.479805+0000","last_clean":"2023-10-09T04:16:58.479805+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:16:58.479805+0000","last_undegraded":"2023-10-09T04:16:58.479805+0000","last_fullsized":"2023-10-09T04:16:58.479805+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":649,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479042+0000","last_change":"2023-10-09T04:16:46.787228+0000","last_active":"2023-10-09T04:16:58.479042+0000","last_peered":"2023-10-09T04:16:58.479042+0000","last_clean":"2023-10-09T04:16:58.479042+0000","last_became_active":"2023-10-09T04:16:46.786694+0000","last_became_peered":"2023-10-09T04:16:46.786694+0000","last_unstale":"2023-10-09T04:16:58.479042+0000","last_undegraded":"2023-10-09T04:16:58.479042+0000","last_fullsized":"2023-10-09T04:16:58.479042+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"287'9720","reported_seq":9774,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:03.911363+0000","last_change":"2023-10-09T04:13:42.847572+0000","last_active":"2023-10-09T04:17:03.911363+0000","last_peered":"2023-10-09T04:17:03.911363+0000","last_clean":"2023-10-09T04:17:03.911363+0000","last_became_active":"2023-10-09T04:13:42.847257+0000","last_became_peered":"2023-10-09T04:13:42.847257+0000","last_unstale":"2023-10-09T04:17:03.911363+0000","last_undegraded":"2023-10-09T04:17:03.911363+0000","last_fullsized":"2023-10-09T04:17:03.911363+0000","mapping_epoch":241,"log_start":"247'1600","ondisk_log_start":"247'1600","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8120,"log_dups_size":0,"ondisk_log_size":8120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:19.215045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109314048,"num_objects":1668,"num_object_clones":0,"num_object_copies":5004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9720,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":247,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463626+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:16:59.463626+0000","last_peered":"2023-10-09T04:16:59.463626+0000","last_clean":"2023-10-09T04:16:59.463626+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:16:59.463626+0000","last_undegraded":"2023-10-09T04:16:59.463626+0000","last_fullsized":"2023-10-09T04:16:59.463626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499773+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:16:59.499773+0000","last_peered":"2023-10-09T04:16:59.499773+0000","last_clean":"2023-10-09T04:16:59.499773+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:16:59.499773+0000","last_undegraded":"2023-10-09T04:16:59.499773+0000","last_fullsized":"2023-10-09T04:16:59.499773+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":650,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:01.100037+0000","last_change":"2023-10-09T04:17:01.099989+0000","last_active":"2023-10-09T04:17:01.100037+0000","last_peered":"2023-10-09T04:17:01.100037+0000","last_clean":"2023-10-09T04:17:01.100037+0000","last_became_active":"2023-10-09T04:15:13.426360+0000","last_became_peered":"2023-10-09T04:15:13.426360+0000","last_unstale":"2023-10-09T04:17:01.100037+0000","last_undegraded":"2023-10-09T04:17:01.100037+0000","last_fullsized":"2023-10-09T04:17:01.100037+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:01.099879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:01.099879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:03.454967+0000","scrub_duration":0.0081520749999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.0","version":"287'9703","reported_seq":9758,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.742554+0000","last_change":"2023-10-09T04:13:42.847429+0000","last_active":"2023-10-09T04:17:05.742554+0000","last_peered":"2023-10-09T04:17:05.742554+0000","last_clean":"2023-10-09T04:17:05.742554+0000","last_became_active":"2023-10-09T04:13:42.847130+0000","last_became_peered":"2023-10-09T04:13:42.847130+0000","last_unstale":"2023-10-09T04:17:05.742554+0000","last_undegraded":"2023-10-09T04:17:05.742554+0000","last_fullsized":"2023-10-09T04:17:05.742554+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9703,"log_dups_size":0,"ondisk_log_size":9703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.587257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105185280,"num_objects":1605,"num_object_clones":0,"num_object_copies":4815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9703,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":220,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499838+0000","last_change":"2023-10-09T04:15:25.538809+0000","last_active":"2023-10-09T04:16:59.499838+0000","last_peered":"2023-10-09T04:16:59.499838+0000","last_clean":"2023-10-09T04:16:59.499838+0000","last_became_active":"2023-10-09T04:15:25.419652+0000","last_became_peered":"2023-10-09T04:15:25.419652+0000","last_unstale":"2023-10-09T04:16:59.499838+0000","last_undegraded":"2023-10-09T04:16:59.499838+0000","last_fullsized":"2023-10-09T04:16:59.499838+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:02.875159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480887+0000","last_change":"2023-10-09T04:15:25.427175+0000","last_active":"2023-10-09T04:16:58.480887+0000","last_peered":"2023-10-09T04:16:58.480887+0000","last_clean":"2023-10-09T04:16:58.480887+0000","last_became_active":"2023-10-09T04:15:25.426875+0000","last_became_peered":"2023-10-09T04:15:25.426875+0000","last_unstale":"2023-10-09T04:16:58.480887+0000","last_undegraded":"2023-10-09T04:16:58.480887+0000","last_fullsized":"2023-10-09T04:16:58.480887+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:46.697878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":650,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:00.124532+0000","last_change":"2023-10-09T04:17:00.124487+0000","last_active":"2023-10-09T04:17:00.124532+0000","last_peered":"2023-10-09T04:17:00.124532+0000","last_clean":"2023-10-09T04:17:00.124532+0000","last_became_active":"2023-10-09T04:15:13.421001+0000","last_became_peered":"2023-10-09T04:15:13.421001+0000","last_unstale":"2023-10-09T04:17:00.124532+0000","last_undegraded":"2023-10-09T04:17:00.124532+0000","last_fullsized":"2023-10-09T04:17:00.124532+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:00.124399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:00.124399+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:25.787390+0000","scrub_duration":0.011147822999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1","version":"287'9805","reported_seq":9861,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:03.890295+0000","last_change":"2023-10-09T04:13:42.633796+0000","last_active":"2023-10-09T04:17:03.890295+0000","last_peered":"2023-10-09T04:17:03.890295+0000","last_clean":"2023-10-09T04:17:03.890295+0000","last_became_active":"2023-10-09T04:13:42.633490+0000","last_became_peered":"2023-10-09T04:13:42.633490+0000","last_unstale":"2023-10-09T04:17:03.890295+0000","last_undegraded":"2023-10-09T04:17:03.890295+0000","last_fullsized":"2023-10-09T04:17:03.890295+0000","mapping_epoch":241,"log_start":"247'1600","ondisk_log_start":"247'1600","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8205,"log_dups_size":0,"ondisk_log_size":8205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:46.137500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112263168,"num_objects":1713,"num_object_clones":0,"num_object_copies":5139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9805,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480895+0000","last_change":"2023-10-09T04:15:25.432206+0000","last_active":"2023-10-09T04:16:58.480895+0000","last_peered":"2023-10-09T04:16:58.480895+0000","last_clean":"2023-10-09T04:16:58.480895+0000","last_became_active":"2023-10-09T04:15:25.430422+0000","last_became_peered":"2023-10-09T04:15:25.430422+0000","last_unstale":"2023-10-09T04:16:58.480895+0000","last_undegraded":"2023-10-09T04:16:58.480895+0000","last_fullsized":"2023-10-09T04:16:58.480895+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.755818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481195+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:16:58.481195+0000","last_peered":"2023-10-09T04:16:58.481195+0000","last_clean":"2023-10-09T04:16:58.481195+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:16:58.481195+0000","last_undegraded":"2023-10-09T04:16:58.481195+0000","last_fullsized":"2023-10-09T04:16:58.481195+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":649,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479045+0000","last_change":"2023-10-09T04:16:46.778683+0000","last_active":"2023-10-09T04:16:58.479045+0000","last_peered":"2023-10-09T04:16:58.479045+0000","last_clean":"2023-10-09T04:16:58.479045+0000","last_became_active":"2023-10-09T04:16:46.778327+0000","last_became_peered":"2023-10-09T04:16:46.778327+0000","last_unstale":"2023-10-09T04:16:58.479045+0000","last_undegraded":"2023-10-09T04:16:58.479045+0000","last_fullsized":"2023-10-09T04:16:58.479045+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":16,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"288'9969","reported_seq":15869,"reported_epoch":288,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:17:06.551442+0000","last_change":"2023-10-09T04:15:26.506798+0000","last_active":"2023-10-09T04:17:06.551442+0000","last_peered":"2023-10-09T04:17:06.551442+0000","last_clean":"2023-10-09T04:15:24.385046+0000","last_became_active":"2023-10-09T04:15:26.497446+0000","last_became_peered":"2023-10-09T04:15:26.497446+0000","last_unstale":"2023-10-09T04:17:06.551442+0000","last_undegraded":"2023-10-09T04:17:06.551442+0000","last_fullsized":"2023-10-09T04:17:06.551442+0000","mapping_epoch":266,"log_start":"254'4481","ondisk_log_start":"254'4481","created":241,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:01.043194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105185280,"num_objects":1605,"num_object_clones":0,"num_object_copies":4815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1605,"num_objects_unfound":0,"num_objects_dirty":1605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9969,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2907,"num_bytes_recovered":190513152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[0,5,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1605}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480801+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:16:58.480801+0000","last_peered":"2023-10-09T04:16:58.480801+0000","last_clean":"2023-10-09T04:16:58.480801+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:16:58.480801+0000","last_undegraded":"2023-10-09T04:16:58.480801+0000","last_fullsized":"2023-10-09T04:16:58.480801+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":41,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481267+0000","last_change":"2023-10-09T04:16:24.617626+0000","last_active":"2023-10-09T04:16:58.481267+0000","last_peered":"2023-10-09T04:16:58.481267+0000","last_clean":"2023-10-09T04:16:58.481267+0000","last_became_active":"2023-10-09T04:16:24.617298+0000","last_became_peered":"2023-10-09T04:16:24.617298+0000","last_unstale":"2023-10-09T04:16:58.481267+0000","last_undegraded":"2023-10-09T04:16:58.481267+0000","last_fullsized":"2023-10-09T04:16:58.481267+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:28.718470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":647,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:02.097355+0000","last_change":"2023-10-09T04:17:02.097310+0000","last_active":"2023-10-09T04:17:02.097355+0000","last_peered":"2023-10-09T04:17:02.097355+0000","last_clean":"2023-10-09T04:17:02.097355+0000","last_became_active":"2023-10-09T04:15:13.428029+0000","last_became_peered":"2023-10-09T04:15:13.428029+0000","last_unstale":"2023-10-09T04:17:02.097355+0000","last_undegraded":"2023-10-09T04:17:02.097355+0000","last_fullsized":"2023-10-09T04:17:02.097355+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:02.097241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:02.097241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:47.238973+0000","scrub_duration":0.0088630900000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.3","version":"287'9622","reported_seq":9688,"reported_epoch":287,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:17:05.703634+0000","last_change":"2023-10-09T04:16:25.591418+0000","last_active":"2023-10-09T04:17:05.703634+0000","last_peered":"2023-10-09T04:17:05.703634+0000","last_clean":"2023-10-09T04:16:23.412068+0000","last_became_active":"2023-10-09T04:16:25.564970+0000","last_became_peered":"2023-10-09T04:16:25.564970+0000","last_unstale":"2023-10-09T04:17:05.703634+0000","last_undegraded":"2023-10-09T04:17:05.703634+0000","last_fullsized":"2023-10-09T04:17:05.703634+0000","mapping_epoch":277,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9122,"log_dups_size":0,"ondisk_log_size":9122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:44.646528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108134400,"num_objects":1650,"num_object_clones":0,"num_object_copies":4950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1650,"num_objects_unfound":0,"num_objects_dirty":1650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9622,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1650}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":75,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479386+0000","last_change":"2023-10-09T04:16:24.471755+0000","last_active":"2023-10-09T04:16:58.479386+0000","last_peered":"2023-10-09T04:16:58.479386+0000","last_clean":"2023-10-09T04:16:58.479386+0000","last_became_active":"2023-10-09T04:16:24.471509+0000","last_became_peered":"2023-10-09T04:16:24.471509+0000","last_unstale":"2023-10-09T04:16:58.479386+0000","last_undegraded":"2023-10-09T04:16:58.479386+0000","last_fullsized":"2023-10-09T04:16:58.479386+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:57.228210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":259,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.485844+0000","last_change":"2023-10-09T04:15:25.418196+0000","last_active":"2023-10-09T04:16:58.485844+0000","last_peered":"2023-10-09T04:16:58.485844+0000","last_clean":"2023-10-09T04:16:58.485844+0000","last_became_active":"2023-10-09T04:15:25.417900+0000","last_became_peered":"2023-10-09T04:15:25.417900+0000","last_unstale":"2023-10-09T04:16:58.485844+0000","last_undegraded":"2023-10-09T04:16:58.485844+0000","last_fullsized":"2023-10-09T04:16:58.485844+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:19.256359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":650,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480037+0000","last_change":"2023-10-09T04:16:46.771877+0000","last_active":"2023-10-09T04:16:58.480037+0000","last_peered":"2023-10-09T04:16:58.480037+0000","last_clean":"2023-10-09T04:16:58.480037+0000","last_became_active":"2023-10-09T04:16:46.771545+0000","last_became_peered":"2023-10-09T04:16:46.771545+0000","last_unstale":"2023-10-09T04:16:58.480037+0000","last_undegraded":"2023-10-09T04:16:58.480037+0000","last_fullsized":"2023-10-09T04:16:58.480037+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.e","version":"288'9805","reported_seq":19475,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:06.441169+0000","last_change":"2023-10-09T04:16:21.813363+0000","last_active":"2023-10-09T04:17:06.441169+0000","last_peered":"2023-10-09T04:17:06.441169+0000","last_clean":"2023-10-09T04:17:06.441169+0000","last_became_active":"2023-10-09T04:16:21.793534+0000","last_became_peered":"2023-10-09T04:16:21.793534+0000","last_unstale":"2023-10-09T04:17:06.441169+0000","last_undegraded":"2023-10-09T04:17:06.441169+0000","last_fullsized":"2023-10-09T04:17:06.441169+0000","mapping_epoch":274,"log_start":"254'4336","ondisk_log_start":"254'4336","created":241,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:38.188955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105185280,"num_objects":1605,"num_object_clones":0,"num_object_copies":4815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9805,"num_write_kb":365120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4794,"num_bytes_recovered":302186496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":264,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479775+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:16:58.479775+0000","last_peered":"2023-10-09T04:16:58.479775+0000","last_clean":"2023-10-09T04:16:58.479775+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:16:58.479775+0000","last_undegraded":"2023-10-09T04:16:58.479775+0000","last_fullsized":"2023-10-09T04:16:58.479775+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":264,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463666+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:16:59.463666+0000","last_peered":"2023-10-09T04:16:59.463666+0000","last_clean":"2023-10-09T04:16:59.463666+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:16:59.463666+0000","last_undegraded":"2023-10-09T04:16:59.463666+0000","last_fullsized":"2023-10-09T04:16:59.463666+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":238,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463312+0000","last_change":"2023-10-09T04:16:24.473165+0000","last_active":"2023-10-09T04:16:59.463312+0000","last_peered":"2023-10-09T04:16:59.463312+0000","last_clean":"2023-10-09T04:16:59.463312+0000","last_became_active":"2023-10-09T04:16:24.472923+0000","last_became_peered":"2023-10-09T04:16:24.472923+0000","last_unstale":"2023-10-09T04:16:59.463312+0000","last_undegraded":"2023-10-09T04:16:59.463312+0000","last_fullsized":"2023-10-09T04:16:59.463312+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:01.846352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":649,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480050+0000","last_change":"2023-10-09T04:16:46.777750+0000","last_active":"2023-10-09T04:16:58.480050+0000","last_peered":"2023-10-09T04:16:58.480050+0000","last_clean":"2023-10-09T04:16:58.480050+0000","last_became_active":"2023-10-09T04:16:46.777315+0000","last_became_peered":"2023-10-09T04:16:46.777315+0000","last_unstale":"2023-10-09T04:16:58.480050+0000","last_undegraded":"2023-10-09T04:16:58.480050+0000","last_fullsized":"2023-10-09T04:16:58.480050+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.8","version":"288'9651","reported_seq":9706,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.742557+0000","last_change":"2023-10-09T04:13:42.847477+0000","last_active":"2023-10-09T04:17:05.742557+0000","last_peered":"2023-10-09T04:17:05.742557+0000","last_clean":"2023-10-09T04:17:05.742557+0000","last_became_active":"2023-10-09T04:13:42.847180+0000","last_became_peered":"2023-10-09T04:13:42.847180+0000","last_unstale":"2023-10-09T04:17:05.742557+0000","last_undegraded":"2023-10-09T04:17:05.742557+0000","last_fullsized":"2023-10-09T04:17:05.742557+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9651,"log_dups_size":0,"ondisk_log_size":9651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:23.775147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108331008,"num_objects":1653,"num_object_clones":0,"num_object_copies":4959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9651,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":251,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463818+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:16:59.463818+0000","last_peered":"2023-10-09T04:16:59.463818+0000","last_clean":"2023-10-09T04:16:59.463818+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:16:59.463818+0000","last_undegraded":"2023-10-09T04:16:59.463818+0000","last_fullsized":"2023-10-09T04:16:59.463818+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481655+0000","last_change":"2023-10-09T04:15:25.435628+0000","last_active":"2023-10-09T04:16:58.481655+0000","last_peered":"2023-10-09T04:16:58.481655+0000","last_clean":"2023-10-09T04:16:58.481655+0000","last_became_active":"2023-10-09T04:15:25.435144+0000","last_became_peered":"2023-10-09T04:15:25.435144+0000","last_unstale":"2023-10-09T04:16:58.481655+0000","last_undegraded":"2023-10-09T04:16:58.481655+0000","last_fullsized":"2023-10-09T04:16:58.481655+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:50.156419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479699+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:16:58.479699+0000","last_peered":"2023-10-09T04:16:58.479699+0000","last_clean":"2023-10-09T04:16:58.479699+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:16:58.479699+0000","last_undegraded":"2023-10-09T04:16:58.479699+0000","last_fullsized":"2023-10-09T04:16:58.479699+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":647,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.478903+0000","last_change":"2023-10-09T04:16:46.777572+0000","last_active":"2023-10-09T04:16:58.478903+0000","last_peered":"2023-10-09T04:16:58.478903+0000","last_clean":"2023-10-09T04:16:58.478903+0000","last_became_active":"2023-10-09T04:16:46.777232+0000","last_became_peered":"2023-10-09T04:16:46.777232+0000","last_unstale":"2023-10-09T04:16:58.478903+0000","last_undegraded":"2023-10-09T04:16:58.478903+0000","last_fullsized":"2023-10-09T04:16:58.478903+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.b","version":"287'9634","reported_seq":10568,"reported_epoch":287,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:17:03.903581+0000","last_change":"2023-10-09T04:16:57.541026+0000","last_active":"2023-10-09T04:17:03.903581+0000","last_peered":"2023-10-09T04:17:03.903581+0000","last_clean":"2023-10-09T04:14:14.593157+0000","last_became_active":"2023-10-09T04:16:24.620512+0000","last_became_peered":"2023-10-09T04:16:24.620512+0000","last_unstale":"2023-10-09T04:17:03.903581+0000","last_undegraded":"2023-10-09T04:16:24.612691+0000","last_fullsized":"2023-10-09T04:16:24.468624+0000","mapping_epoch":276,"log_start":"247'1500","ondisk_log_start":"247'1500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8134,"log_dups_size":0,"ondisk_log_size":8134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:37.251260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110886912,"num_objects":1692,"num_object_clones":0,"num_object_copies":5076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":2646,"num_objects_unfound":0,"num_objects_dirty":1692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9634,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27852800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":1600},{"shards":"1,3","objects":-954}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":103,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.473507+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:16:58.473507+0000","last_peered":"2023-10-09T04:16:58.473507+0000","last_clean":"2023-10-09T04:16:58.473507+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:16:58.473507+0000","last_undegraded":"2023-10-09T04:16:58.473507+0000","last_fullsized":"2023-10-09T04:16:58.473507+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":259,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479238+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:16:58.479238+0000","last_peered":"2023-10-09T04:16:58.479238+0000","last_clean":"2023-10-09T04:16:58.479238+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:16:58.479238+0000","last_undegraded":"2023-10-09T04:16:58.479238+0000","last_fullsized":"2023-10-09T04:16:58.479238+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463524+0000","last_change":"2023-10-09T04:16:24.470074+0000","last_active":"2023-10-09T04:16:59.463524+0000","last_peered":"2023-10-09T04:16:59.463524+0000","last_clean":"2023-10-09T04:16:59.463524+0000","last_became_active":"2023-10-09T04:16:24.469767+0000","last_became_peered":"2023-10-09T04:16:24.469767+0000","last_unstale":"2023-10-09T04:16:59.463524+0000","last_undegraded":"2023-10-09T04:16:59.463524+0000","last_fullsized":"2023-10-09T04:16:59.463524+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.823815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"287'9696","reported_seq":9751,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.734053+0000","last_change":"2023-10-09T04:13:42.638373+0000","last_active":"2023-10-09T04:17:05.734053+0000","last_peered":"2023-10-09T04:17:05.734053+0000","last_clean":"2023-10-09T04:17:05.734053+0000","last_became_active":"2023-10-09T04:13:42.637990+0000","last_became_peered":"2023-10-09T04:13:42.637990+0000","last_unstale":"2023-10-09T04:17:05.734053+0000","last_undegraded":"2023-10-09T04:17:05.734053+0000","last_fullsized":"2023-10-09T04:17:05.734053+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9696,"log_dups_size":0,"ondisk_log_size":9696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:37.637304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101974016,"num_objects":1556,"num_object_clones":0,"num_object_copies":4668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9696,"num_write_kb":360064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":271,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463376+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:16:59.463376+0000","last_peered":"2023-10-09T04:16:59.463376+0000","last_clean":"2023-10-09T04:16:59.463376+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:16:59.463376+0000","last_undegraded":"2023-10-09T04:16:59.463376+0000","last_fullsized":"2023-10-09T04:16:59.463376+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":651,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.478624+0000","last_change":"2023-10-09T04:16:46.771675+0000","last_active":"2023-10-09T04:16:58.478624+0000","last_peered":"2023-10-09T04:16:58.478624+0000","last_clean":"2023-10-09T04:16:58.478624+0000","last_became_active":"2023-10-09T04:16:46.771308+0000","last_became_peered":"2023-10-09T04:16:46.771308+0000","last_unstale":"2023-10-09T04:16:58.478624+0000","last_undegraded":"2023-10-09T04:16:58.478624+0000","last_fullsized":"2023-10-09T04:16:58.478624+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":184,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463831+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:16:59.463831+0000","last_peered":"2023-10-09T04:16:59.463831+0000","last_clean":"2023-10-09T04:16:59.463831+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:16:59.463831+0000","last_undegraded":"2023-10-09T04:16:59.463831+0000","last_fullsized":"2023-10-09T04:16:59.463831+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481278+0000","last_change":"2023-10-09T04:15:25.409271+0000","last_active":"2023-10-09T04:16:58.481278+0000","last_peered":"2023-10-09T04:16:58.481278+0000","last_clean":"2023-10-09T04:16:58.481278+0000","last_became_active":"2023-10-09T04:15:25.408986+0000","last_became_peered":"2023-10-09T04:15:25.408986+0000","last_unstale":"2023-10-09T04:16:58.481278+0000","last_undegraded":"2023-10-09T04:16:58.481278+0000","last_fullsized":"2023-10-09T04:16:58.481278+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:18.172890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":660,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.198108+0000","last_change":"2023-10-09T04:17:05.198063+0000","last_active":"2023-10-09T04:17:05.198108+0000","last_peered":"2023-10-09T04:17:05.198108+0000","last_clean":"2023-10-09T04:17:05.198108+0000","last_became_active":"2023-10-09T04:16:46.761340+0000","last_became_peered":"2023-10-09T04:16:46.761340+0000","last_unstale":"2023-10-09T04:17:05.198108+0000","last_undegraded":"2023-10-09T04:17:05.198108+0000","last_fullsized":"2023-10-09T04:17:05.198108+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:05.197962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:05.197962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:26.889072+0000","scrub_duration":0.0029764380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"287'9269","reported_seq":9325,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:03.892095+0000","last_change":"2023-10-09T04:13:42.635883+0000","last_active":"2023-10-09T04:17:03.892095+0000","last_peered":"2023-10-09T04:17:03.892095+0000","last_clean":"2023-10-09T04:17:03.892095+0000","last_became_active":"2023-10-09T04:13:42.635571+0000","last_became_peered":"2023-10-09T04:13:42.635571+0000","last_unstale":"2023-10-09T04:17:03.892095+0000","last_undegraded":"2023-10-09T04:17:03.892095+0000","last_fullsized":"2023-10-09T04:17:03.892095+0000","mapping_epoch":241,"log_start":"247'1100","ondisk_log_start":"247'1100","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:34.753582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9269,"num_write_kb":347904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":267,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.478545+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:16:58.478545+0000","last_peered":"2023-10-09T04:16:58.478545+0000","last_clean":"2023-10-09T04:16:58.478545+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:16:58.478545+0000","last_undegraded":"2023-10-09T04:16:58.478545+0000","last_fullsized":"2023-10-09T04:16:58.478545+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":273,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499667+0000","last_change":"2023-10-09T04:16:24.470732+0000","last_active":"2023-10-09T04:16:59.499667+0000","last_peered":"2023-10-09T04:16:59.499667+0000","last_clean":"2023-10-09T04:16:59.499667+0000","last_became_active":"2023-10-09T04:16:24.470433+0000","last_became_peered":"2023-10-09T04:16:24.470433+0000","last_unstale":"2023-10-09T04:16:59.499667+0000","last_undegraded":"2023-10-09T04:16:59.499667+0000","last_fullsized":"2023-10-09T04:16:59.499667+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:54.719857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":278,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.657648+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:16:59.657648+0000","last_peered":"2023-10-09T04:16:59.657648+0000","last_clean":"2023-10-09T04:16:59.657648+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:16:59.657648+0000","last_undegraded":"2023-10-09T04:16:59.657648+0000","last_fullsized":"2023-10-09T04:16:59.657648+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":650,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479126+0000","last_change":"2023-10-09T04:16:46.787052+0000","last_active":"2023-10-09T04:16:58.479126+0000","last_peered":"2023-10-09T04:16:58.479126+0000","last_clean":"2023-10-09T04:16:58.479126+0000","last_became_active":"2023-10-09T04:16:46.786290+0000","last_became_peered":"2023-10-09T04:16:46.786290+0000","last_unstale":"2023-10-09T04:16:58.479126+0000","last_undegraded":"2023-10-09T04:16:58.479126+0000","last_fullsized":"2023-10-09T04:16:58.479126+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"287'9962","reported_seq":10018,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.250971+0000","last_change":"2023-10-09T04:13:42.847637+0000","last_active":"2023-10-09T04:17:05.250971+0000","last_peered":"2023-10-09T04:17:05.250971+0000","last_clean":"2023-10-09T04:17:05.250971+0000","last_became_active":"2023-10-09T04:13:42.847359+0000","last_became_peered":"2023-10-09T04:13:42.847359+0000","last_unstale":"2023-10-09T04:17:05.250971+0000","last_undegraded":"2023-10-09T04:17:05.250971+0000","last_fullsized":"2023-10-09T04:17:05.250971+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9962,"log_dups_size":0,"ondisk_log_size":9962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:39.956849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107741184,"num_objects":1644,"num_object_clones":0,"num_object_copies":4932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9962,"num_write_kb":371392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":270,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481636+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:16:58.481636+0000","last_peered":"2023-10-09T04:16:58.481636+0000","last_clean":"2023-10-09T04:16:58.481636+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:16:58.481636+0000","last_undegraded":"2023-10-09T04:16:58.481636+0000","last_fullsized":"2023-10-09T04:16:58.481636+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479853+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:16:58.479853+0000","last_peered":"2023-10-09T04:16:58.479853+0000","last_clean":"2023-10-09T04:16:58.479853+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:16:58.479853+0000","last_undegraded":"2023-10-09T04:16:58.479853+0000","last_fullsized":"2023-10-09T04:16:58.479853+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":231,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.499896+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:16:59.499896+0000","last_peered":"2023-10-09T04:16:59.499896+0000","last_clean":"2023-10-09T04:16:59.499896+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:16:59.499896+0000","last_undegraded":"2023-10-09T04:16:59.499896+0000","last_fullsized":"2023-10-09T04:16:59.499896+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":647,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481398+0000","last_change":"2023-10-09T04:16:46.771645+0000","last_active":"2023-10-09T04:16:58.481398+0000","last_peered":"2023-10-09T04:16:58.481398+0000","last_clean":"2023-10-09T04:16:58.481398+0000","last_became_active":"2023-10-09T04:16:46.771347+0000","last_became_peered":"2023-10-09T04:16:46.771347+0000","last_unstale":"2023-10-09T04:16:58.481398+0000","last_undegraded":"2023-10-09T04:16:58.481398+0000","last_fullsized":"2023-10-09T04:16:58.481398+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"288'9642","reported_seq":9699,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:06.539606+0000","last_change":"2023-10-09T04:13:42.847596+0000","last_active":"2023-10-09T04:17:06.539606+0000","last_peered":"2023-10-09T04:17:06.539606+0000","last_clean":"2023-10-09T04:17:06.539606+0000","last_became_active":"2023-10-09T04:13:42.847298+0000","last_became_peered":"2023-10-09T04:13:42.847298+0000","last_unstale":"2023-10-09T04:17:06.539606+0000","last_undegraded":"2023-10-09T04:17:06.539606+0000","last_fullsized":"2023-10-09T04:17:06.539606+0000","mapping_epoch":241,"log_start":"254'4155","ondisk_log_start":"254'4155","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:42.275480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105775104,"num_objects":1614,"num_object_clones":0,"num_object_copies":4842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9642,"num_write_kb":360192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":191,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479929+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:16:58.479929+0000","last_peered":"2023-10-09T04:16:58.479929+0000","last_clean":"2023-10-09T04:16:58.479929+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:16:58.479929+0000","last_undegraded":"2023-10-09T04:16:58.479929+0000","last_fullsized":"2023-10-09T04:16:58.479929+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":243,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480259+0000","last_change":"2023-10-09T04:15:25.414014+0000","last_active":"2023-10-09T04:16:58.480259+0000","last_peered":"2023-10-09T04:16:58.480259+0000","last_clean":"2023-10-09T04:16:58.480259+0000","last_became_active":"2023-10-09T04:15:25.413619+0000","last_became_peered":"2023-10-09T04:15:25.413619+0000","last_unstale":"2023-10-09T04:16:58.480259+0000","last_undegraded":"2023-10-09T04:16:58.480259+0000","last_fullsized":"2023-10-09T04:16:58.480259+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:13.741581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":227,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.478809+0000","last_change":"2023-10-09T04:16:24.469186+0000","last_active":"2023-10-09T04:16:58.478809+0000","last_peered":"2023-10-09T04:16:58.478809+0000","last_clean":"2023-10-09T04:16:58.478809+0000","last_became_active":"2023-10-09T04:16:24.468836+0000","last_became_peered":"2023-10-09T04:16:24.468836+0000","last_unstale":"2023-10-09T04:16:58.478809+0000","last_undegraded":"2023-10-09T04:16:58.478809+0000","last_fullsized":"2023-10-09T04:16:58.478809+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:42.902638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":650,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.479993+0000","last_change":"2023-10-09T04:16:46.782032+0000","last_active":"2023-10-09T04:16:58.479993+0000","last_peered":"2023-10-09T04:16:58.479993+0000","last_clean":"2023-10-09T04:16:58.479993+0000","last_became_active":"2023-10-09T04:16:46.778146+0000","last_became_peered":"2023-10-09T04:16:46.778146+0000","last_unstale":"2023-10-09T04:16:58.479993+0000","last_undegraded":"2023-10-09T04:16:58.479993+0000","last_fullsized":"2023-10-09T04:16:58.479993+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"287'9512","reported_seq":9568,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.740986+0000","last_change":"2023-10-09T04:13:42.646443+0000","last_active":"2023-10-09T04:17:05.740986+0000","last_peered":"2023-10-09T04:17:05.740986+0000","last_clean":"2023-10-09T04:17:05.740986+0000","last_became_active":"2023-10-09T04:13:42.646138+0000","last_became_peered":"2023-10-09T04:13:42.646138+0000","last_unstale":"2023-10-09T04:17:05.740986+0000","last_undegraded":"2023-10-09T04:17:05.740986+0000","last_fullsized":"2023-10-09T04:17:05.740986+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":9512,"log_dups_size":0,"ondisk_log_size":9512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:56.607273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054232,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9512,"num_write_kb":355649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":108,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.473199+0000","last_change":"2023-10-09T04:13:46.100979+0000","last_active":"2023-10-09T04:16:58.473199+0000","last_peered":"2023-10-09T04:16:58.473199+0000","last_clean":"2023-10-09T04:16:58.473199+0000","last_became_active":"2023-10-09T04:13:46.100727+0000","last_became_peered":"2023-10-09T04:13:46.100727+0000","last_unstale":"2023-10-09T04:16:58.473199+0000","last_undegraded":"2023-10-09T04:16:58.473199+0000","last_fullsized":"2023-10-09T04:16:58.473199+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:11.755013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":278,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.657381+0000","last_change":"2023-10-09T04:16:20.109259+0000","last_active":"2023-10-09T04:16:59.657381+0000","last_peered":"2023-10-09T04:16:59.657381+0000","last_clean":"2023-10-09T04:16:59.657381+0000","last_became_active":"2023-10-09T04:15:25.496763+0000","last_became_peered":"2023-10-09T04:15:25.496763+0000","last_unstale":"2023-10-09T04:16:59.657381+0000","last_undegraded":"2023-10-09T04:16:59.657381+0000","last_fullsized":"2023-10-09T04:16:59.657381+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:15.807543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480989+0000","last_change":"2023-10-09T04:15:25.406909+0000","last_active":"2023-10-09T04:16:58.480989+0000","last_peered":"2023-10-09T04:16:58.480989+0000","last_clean":"2023-10-09T04:16:58.480989+0000","last_became_active":"2023-10-09T04:15:25.406617+0000","last_became_peered":"2023-10-09T04:15:25.406617+0000","last_unstale":"2023-10-09T04:16:58.480989+0000","last_undegraded":"2023-10-09T04:16:58.480989+0000","last_fullsized":"2023-10-09T04:16:58.480989+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:34.910221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":252,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:17:01.149917+0000","last_change":"2023-10-09T04:15:25.414363+0000","last_active":"2023-10-09T04:17:01.149917+0000","last_peered":"2023-10-09T04:17:01.149917+0000","last_clean":"2023-10-09T04:17:01.149917+0000","last_became_active":"2023-10-09T04:15:25.413987+0000","last_became_peered":"2023-10-09T04:15:25.413987+0000","last_unstale":"2023-10-09T04:17:01.149917+0000","last_undegraded":"2023-10-09T04:17:01.149917+0000","last_fullsized":"2023-10-09T04:17:01.149917+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:45.394691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.473179+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:16:58.473179+0000","last_peered":"2023-10-09T04:16:58.473179+0000","last_clean":"2023-10-09T04:16:58.473179+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:16:58.473179+0000","last_undegraded":"2023-10-09T04:16:58.473179+0000","last_fullsized":"2023-10-09T04:16:58.473179+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":277,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463676+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:16:59.463676+0000","last_peered":"2023-10-09T04:16:59.463676+0000","last_clean":"2023-10-09T04:16:59.463676+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:16:59.463676+0000","last_undegraded":"2023-10-09T04:16:59.463676+0000","last_fullsized":"2023-10-09T04:16:59.463676+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.481847+0000","last_change":"2023-10-09T04:15:25.476221+0000","last_active":"2023-10-09T04:16:58.481847+0000","last_peered":"2023-10-09T04:16:58.481847+0000","last_clean":"2023-10-09T04:16:58.481847+0000","last_became_active":"2023-10-09T04:15:25.475459+0000","last_became_peered":"2023-10-09T04:15:25.475459+0000","last_unstale":"2023-10-09T04:16:58.481847+0000","last_undegraded":"2023-10-09T04:16:58.481847+0000","last_fullsized":"2023-10-09T04:16:58.481847+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:34.779580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":260,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.463749+0000","last_change":"2023-10-09T04:15:25.525420+0000","last_active":"2023-10-09T04:16:59.463749+0000","last_peered":"2023-10-09T04:16:59.463749+0000","last_clean":"2023-10-09T04:16:59.463749+0000","last_became_active":"2023-10-09T04:15:25.406419+0000","last_became_peered":"2023-10-09T04:15:25.406419+0000","last_unstale":"2023-10-09T04:16:59.463749+0000","last_undegraded":"2023-10-09T04:16:59.463749+0000","last_fullsized":"2023-10-09T04:16:59.463749+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:10.680233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480832+0000","last_change":"2023-10-09T04:15:25.435313+0000","last_active":"2023-10-09T04:16:58.480832+0000","last_peered":"2023-10-09T04:16:58.480832+0000","last_clean":"2023-10-09T04:16:58.480832+0000","last_became_active":"2023-10-09T04:15:25.434996+0000","last_became_peered":"2023-10-09T04:15:25.434996+0000","last_unstale":"2023-10-09T04:16:58.480832+0000","last_undegraded":"2023-10-09T04:16:58.480832+0000","last_fullsized":"2023-10-09T04:16:58.480832+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:22.816036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":103,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.472935+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:16:58.472935+0000","last_peered":"2023-10-09T04:16:58.472935+0000","last_clean":"2023-10-09T04:16:58.472935+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:16:58.472935+0000","last_undegraded":"2023-10-09T04:16:58.472935+0000","last_fullsized":"2023-10-09T04:16:58.472935+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":102,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.473571+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:16:58.473571+0000","last_peered":"2023-10-09T04:16:58.473571+0000","last_clean":"2023-10-09T04:16:58.473571+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:16:58.473571+0000","last_undegraded":"2023-10-09T04:16:58.473571+0000","last_fullsized":"2023-10-09T04:16:58.473571+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":280,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:16:59.657361+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:16:59.657361+0000","last_peered":"2023-10-09T04:16:59.657361+0000","last_clean":"2023-10-09T04:16:59.657361+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:16:59.657361+0000","last_undegraded":"2023-10-09T04:16:59.657361+0000","last_fullsized":"2023-10-09T04:16:59.657361+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-10-09T04:16:58.480683+0000","last_change":"2023-10-09T04:15:25.423355+0000","last_active":"2023-10-09T04:16:58.480683+0000","last_peered":"2023-10-09T04:16:58.480683+0000","last_clean":"2023-10-09T04:16:58.480683+0000","last_became_active":"2023-10-09T04:15:25.423070+0000","last_became_peered":"2023-10-09T04:15:25.423070+0000","last_unstale":"2023-10-09T04:16:58.480683+0000","last_undegraded":"2023-10-09T04:16:58.480683+0000","last_fullsized":"2023-10-09T04:16:58.480683+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:56.629007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6134,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1703739416,"num_objects":25998,"num_object_clones":0,"num_object_copies":77994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":7500,"num_objects_unfound":0,"num_objects_dirty":25998,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155166,"num_write_kb":5797185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13950,"num_bytes_recovered":890044416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5790052352,"data_stored":5790040136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127531,"ondisk_log_size":127531,"up":48,"acting":47,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":674157,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":350,"num_read_kb":644,"num_write":168,"num_write_kb":1686,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1234407,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201575,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1035584,"kb_used_data":730104,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92709568,"statfs":{"total":95995035648,"available":94934597632,"internally_reserved":0,"allocated":747626496,"data_stored":746765194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":3.5139999999999998,"15min":2.5830000000000002},"min":{"1min":0.33900000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":9.4220000000000006,"5min":88.144999999999996,"15min":126.752},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.8479999999999999,"5min":3.371,"15min":2.5569999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.332000000000001,"5min":89.284999999999997,"15min":128.88399999999999},"last":0.315}]},{"osd":1,"last update":"Mon Oct 9 04:17:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1890000000000001,"5min":4.7720000000000002,"15min":3.7789999999999999},"min":{"1min":0.38,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":6.774,"5min":103.324,"15min":245.28700000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.9079999999999999,"5min":4.5060000000000002,"15min":3.6469999999999998},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":10.913,"5min":104.248,"15min":244.626},"last":0.26000000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":2.1139999999999999,"15min":1.79},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27400000000000002},"max":{"1min":4.6779999999999999,"5min":76.774000000000001,"15min":76.774000000000001},"last":3.9980000000000002},{"interface":"front","average":{"1min":2.774,"5min":2.6240000000000001,"15min":2.3580000000000001},"min":{"1min":0.39700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":10.291,"5min":76.715999999999994,"15min":99.372},"last":4.9889999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:17:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":3.1480000000000001,"15min":2.415},"min":{"1min":0.37,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":6.9420000000000002,"5min":85.575000000000003,"15min":99.134},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.4750000000000001,"5min":3.4620000000000002,"15min":2.589},"min":{"1min":0.40000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.9269999999999996,"5min":85.528000000000006,"15min":99.117999999999995},"last":0.52300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.59899999999999998,"15min":0.73899999999999999},"min":{"1min":0.25800000000000001,"5min":0.252,"15min":0.23400000000000001},"max":{"1min":4.7110000000000003,"5min":4.7110000000000003,"15min":16.195},"last":1.8979999999999999},{"interface":"front","average":{"1min":2.3399999999999999,"5min":1.0249999999999999,"15min":0.88400000000000001},"min":{"1min":0.34699999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.282,"5min":9.282,"15min":12.442},"last":4.9500000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.089,"15min":1.133},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":8.4130000000000003,"5min":120.066,"15min":120.066},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.9079999999999999,"15min":1.1399999999999999},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.4580000000000002,"5min":120.09699999999999,"15min":120.09699999999999},"last":3.0790000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5390000000000001,"5min":1.121,"15min":0.876},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.452,"5min":8.452,"15min":8.452},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.357,"5min":0.76700000000000002,"15min":0.81000000000000005},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.0990000000000002,"5min":5.0990000000000002,"15min":5.9039999999999999},"last":2.0499999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463212,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":999468,"kb_used_data":689892,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92745684,"statfs":{"total":95995035648,"available":94971580416,"internally_reserved":0,"allocated":706449408,"data_stored":705608093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":3.452,"15min":2.819},"min":{"1min":0.25,"5min":0.193,"15min":0.193},"max":{"1min":7.9820000000000002,"5min":120.55200000000001,"15min":120.55200000000001},"last":0.89200000000000002},{"interface":"front","average":{"1min":2.5939999999999999,"5min":3.73,"15min":3.1480000000000001},"min":{"1min":0.26200000000000001,"5min":0.223,"15min":0.223},"max":{"1min":7.0179999999999998,"5min":120.252,"15min":120.252},"last":1.2609999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":4.5110000000000001,"15min":3.0249999999999999},"min":{"1min":0.318,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.526999999999999,"5min":143.22499999999999,"15min":143.22499999999999},"last":3.4609999999999999},{"interface":"front","average":{"1min":2.1909999999999998,"5min":4.0650000000000004,"15min":2.98},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.8129999999999997,"5min":143.18700000000001,"15min":143.18700000000001},"last":1.3600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":3.4910000000000001,"15min":2.819},"min":{"1min":0.34300000000000003,"5min":0.23899999999999999,"15min":0.218},"max":{"1min":8.3369999999999997,"5min":139.06899999999999,"15min":139.06899999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.012,"5min":3.5430000000000001,"15min":2.8650000000000002},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":8.3729999999999993,"5min":139.173,"15min":139.173},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":2.464,"15min":2.6509999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":7.0099999999999998,"5min":51.082000000000001,"15min":57.509999999999998},"last":3.7130000000000001},{"interface":"front","average":{"1min":2.9020000000000001,"5min":2.3500000000000001,"15min":2.617},"min":{"1min":0.39800000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2669999999999995,"5min":51.031999999999996,"15min":57.453000000000003},"last":4.1630000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.93300000000000005,"15min":0.64800000000000002},"min":{"1min":0.28000000000000003,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":5.9829999999999997,"5min":5.9829999999999997,"15min":5.9829999999999997},"last":0.376},{"interface":"front","average":{"1min":1.579,"5min":0.97699999999999998,"15min":0.80900000000000005},"min":{"1min":0.307,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":7.7439999999999998,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.33000000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.0629999999999999,"15min":1.0389999999999999},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.8040000000000003,"5min":7.6310000000000002,"15min":54.773000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.1299999999999999,"15min":1.147},"min":{"1min":0.35099999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.0289999999999999,"5min":7.0289999999999999,"15min":54.853999999999999},"last":0.36499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":0.99099999999999999,"15min":0.76400000000000001},"min":{"1min":0.38300000000000001,"5min":0.317,"15min":0.313},"max":{"1min":5.9560000000000004,"5min":5.9560000000000004,"15min":5.9560000000000004},"last":2.9009999999999998},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.131,"15min":0.86499999999999999},"min":{"1min":0.36899999999999999,"5min":0.28899999999999998,"15min":0.28699999999999998},"max":{"1min":9.1829999999999998,"5min":9.1829999999999998,"15min":9.1829999999999998},"last":2.952}]}]},{"osd":1,"up_from":89,"seq":382252089520,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1160108,"kb_used_data":761316,"kb_used_omap":9,"kb_used_meta":398774,"kb_avail":92585044,"statfs":{"total":95995035648,"available":94807085056,"internally_reserved":0,"allocated":779587584,"data_stored":778392699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":408345577},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":4.5979999999999999,"15min":3.2519999999999998},"min":{"1min":0.217,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":24.111999999999998,"5min":68.048000000000002,"15min":85.977999999999994},"last":7.9610000000000003},{"interface":"front","average":{"1min":4.4989999999999997,"5min":4.2240000000000002,"15min":3.629},"min":{"1min":0.26600000000000001,"5min":0.182,"15min":0.182},"max":{"1min":21.323,"5min":58.027000000000001,"15min":99.831999999999994},"last":7.7000000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.3359999999999999,"15min":2.2010000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.191},"max":{"1min":7.6349999999999998,"5min":25.577000000000002,"15min":121.54600000000001},"last":7.9299999999999997},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.8799999999999999,"15min":2.54},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":13.595000000000001,"5min":15.298999999999999,"15min":99.899000000000001},"last":7.7329999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":4.0289999999999999,"15min":3.3479999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.6600000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":7.9850000000000003},{"interface":"front","average":{"1min":3.7669999999999999,"5min":4.9669999999999996,"15min":4.1589999999999998},"min":{"1min":0.28799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":21.859000000000002,"5min":68.563999999999993,"15min":138.49799999999999},"last":7.7640000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":2.3359999999999999,"15min":2.0459999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.254},"max":{"1min":10.09,"5min":11.973000000000001,"15min":99.989999999999995},"last":0.443},{"interface":"front","average":{"1min":2.0019999999999998,"5min":2.4990000000000001,"15min":1.9770000000000001},"min":{"1min":0.32600000000000001,"5min":0.318,"15min":0.25900000000000001},"max":{"1min":9.2929999999999993,"5min":25.957999999999998,"15min":100.249},"last":0.39300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":2.4350000000000001,"15min":2.0510000000000002},"min":{"1min":0.35799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":14.314,"5min":14.314,"15min":99.933999999999997},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.427,"5min":2.7269999999999999,"15min":2.1619999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.307},"max":{"1min":9.3350000000000009,"5min":25.890999999999998,"15min":100.34099999999999},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":2.7229999999999999,"15min":2.258},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30299999999999999},"max":{"1min":9.3640000000000008,"5min":25.931000000000001,"15min":100.309},"last":2.7919999999999998},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.8959999999999999,"15min":1.9019999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":3.9079999999999999,"5min":13.446,"15min":100.143},"last":0.63600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.585,"5min":2.7650000000000001,"15min":2.242},"min":{"1min":0.434,"5min":0.374,"15min":0.35799999999999998},"max":{"1min":9.391,"5min":25.984999999999999,"15min":100.214},"last":2.8340000000000001},{"interface":"front","average":{"1min":2.0470000000000002,"5min":2.2269999999999999,"15min":2.0289999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":11.023,"5min":13.471,"15min":100.113},"last":7.8479999999999999}]}]},{"osd":0,"up_from":84,"seq":360777253047,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146600,"kb_used_data":731936,"kb_used_omap":28,"kb_used_meta":414435,"kb_avail":92598552,"statfs":{"total":95995035648,"available":94820917248,"internally_reserved":0,"allocated":749502464,"data_stored":748250838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":424382288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:16:56 2023","interfaces":[{"interface":"back","average":{"1min":10.387,"5min":5.298,"15min":3.6240000000000001},"min":{"1min":0.217,"5min":0.187,"15min":0.17000000000000001},"max":{"1min":120.511,"5min":120.511,"15min":120.511},"last":3.6150000000000002},{"interface":"front","average":{"1min":9.8219999999999992,"5min":5.8760000000000003,"15min":4.0460000000000003},"min":{"1min":0.30299999999999999,"5min":0.25,"15min":0.214},"max":{"1min":120.934,"5min":120.934,"15min":120.934},"last":4.3620000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":2.786,"15min":2.7320000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":4.944,"5min":13.108000000000001,"15min":118.54000000000001},"last":4.7270000000000003},{"interface":"front","average":{"1min":4.7089999999999996,"5min":3.3570000000000002,"15min":3.0680000000000001},"min":{"1min":0.23100000000000001,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":14.304,"5min":14.304,"15min":118.727},"last":2.4950000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:16:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":4.4909999999999997,"15min":4.9669999999999996},"min":{"1min":0.219,"5min":0.219,"15min":0.20999999999999999},"max":{"1min":9.9529999999999994,"5min":99.709000000000003,"15min":156.488},"last":3.585},{"interface":"front","average":{"1min":2.5009999999999999,"5min":4.516,"15min":5.0469999999999997},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.22500000000000001},"max":{"1min":18.823,"5min":99.560000000000002,"15min":156.50700000000001},"last":4.5019999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.8700000000000001,"15min":2.0529999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":5.6550000000000002,"5min":9.7270000000000003,"15min":118.633},"last":4.4009999999999998},{"interface":"front","average":{"1min":4.4039999999999999,"5min":2.375,"15min":2.4860000000000002},"min":{"1min":0.378,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.201000000000001,"5min":13.201000000000001,"15min":118.913},"last":2.5950000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":5.4029999999999996,"5min":4.5289999999999999,"15min":3.718},"min":{"1min":0.42599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":13.246,"5min":156.96100000000001,"15min":182.68899999999999},"last":2.5600000000000001},{"interface":"front","average":{"1min":1.377,"5min":3.5009999999999999,"15min":3.258},"min":{"1min":0.31,"5min":0.28399999999999997,"15min":0.28100000000000003},"max":{"1min":5.8029999999999999,"5min":156.97300000000001,"15min":182.66300000000001},"last":3.641}]},{"osd":6,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.5289999999999999,"15min":2.5009999999999999},"min":{"1min":0.34300000000000003,"5min":0.33400000000000002,"15min":0.30599999999999999},"max":{"1min":4.7880000000000003,"5min":10.797000000000001,"15min":137.36099999999999},"last":3.6720000000000002},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.0859999999999999,"15min":2.5470000000000002},"min":{"1min":0.39200000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":5.6849999999999996,"5min":12.496,"15min":136.88999999999999},"last":4.4299999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.611,"15min":3.157},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.32100000000000001},"max":{"1min":3.8010000000000002,"5min":10.77,"15min":185.024},"last":3.6949999999999998},{"interface":"front","average":{"1min":1.514,"5min":2.1680000000000001,"15min":2.8180000000000001},"min":{"1min":0.41599999999999998,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":5.7089999999999996,"5min":12.417,"15min":184.98599999999999},"last":4.4580000000000002}]}]},{"osd":2,"up_from":94,"seq":403726925994,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196656,"kb_used_data":732840,"kb_used_omap":81,"kb_used_meta":463726,"kb_avail":92548496,"statfs":{"total":95995035648,"available":94769659904,"internally_reserved":0,"allocated":750428160,"data_stored":749228175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":474855642},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":6.1120000000000001,"5min":5.0999999999999996,"15min":2.7290000000000001},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.17399999999999999},"max":{"1min":35.896000000000001,"5min":64.430999999999997,"15min":64.430999999999997},"last":0.215},{"interface":"front","average":{"1min":4.0679999999999996,"5min":4.21,"15min":2.7909999999999999},"min":{"1min":0.20799999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":34.981000000000002,"5min":60.125999999999998,"15min":60.125999999999998},"last":0.97099999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8439999999999999,"5min":2.8530000000000002,"15min":2.6419999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":11.315,"5min":50.680999999999997,"15min":68.009},"last":0.52700000000000002},{"interface":"front","average":{"1min":3.1429999999999998,"5min":2.8570000000000002,"15min":2.7490000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":15.419,"5min":53.353000000000002,"15min":66.975999999999999},"last":0.61699999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:16:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":2.6789999999999998,"15min":2.1139999999999999},"min":{"1min":0.22700000000000001,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":8.6859999999999999,"5min":20.242999999999999,"15min":27.562000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.6869999999999998,"15min":2.3210000000000002},"min":{"1min":0.23899999999999999,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":15.333,"5min":17.076000000000001,"15min":46.286999999999999},"last":4.2089999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.6000000000000001,"15min":1.6639999999999999},"min":{"1min":0.28599999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.265000000000001,"5min":10.265000000000001,"15min":46.075000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.784,"5min":2.0550000000000002,"15min":1.52},"min":{"1min":0.312,"5min":0.28899999999999998,"15min":0.27200000000000002},"max":{"1min":5.4960000000000004,"5min":20.134,"15min":20.134},"last":2.3370000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":3.085,"5min":1.8939999999999999,"15min":1.786},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":11.201000000000001,"5min":11.201000000000001,"15min":46.265000000000001},"last":1.6020000000000001},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.6639999999999999,"15min":1.8400000000000001},"min":{"1min":0.35799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.331,"5min":10.331,"15min":46.106000000000002},"last":1.5680000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":2.1859999999999999,"15min":1.9890000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":6.8239999999999998,"5min":20.184000000000001,"15min":114.14},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.4780000000000002,"5min":1.7649999999999999,"15min":2.117},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33300000000000002},"max":{"1min":11.252000000000001,"5min":11.252000000000001,"15min":114.123},"last":0.93600000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.2759999999999998,"15min":2.1859999999999999},"min":{"1min":0.435,"5min":0.36299999999999999,"15min":0.34300000000000003},"max":{"1min":5.423,"5min":20.215,"15min":117.32599999999999},"last":1.2549999999999999},{"interface":"front","average":{"1min":2.9199999999999999,"5min":1.9350000000000001,"15min":2.262},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":11.284000000000001,"5min":11.284000000000001,"15min":117.28700000000001},"last":1.335}]}]},{"osd":3,"up_from":99,"seq":425201762468,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1367616,"kb_used_data":731768,"kb_used_omap":7,"kb_used_meta":635704,"kb_avail":92377536,"statfs":{"total":95995035648,"available":94594596864,"internally_reserved":0,"allocated":749330432,"data_stored":748122762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":650961512},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":6.2850000000000001,"5min":6.0449999999999999,"15min":5.0620000000000003},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":78.771000000000001,"5min":125.765,"15min":125.765},"last":5.1130000000000004},{"interface":"front","average":{"1min":6.5449999999999999,"5min":6.5250000000000004,"15min":5.274},"min":{"1min":0.35999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":78.853999999999999,"5min":125.46899999999999,"15min":125.46899999999999},"last":6.5709999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":7.9779999999999998,"5min":6.359,"15min":4.2590000000000003},"min":{"1min":0.28000000000000003,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":77.599999999999994,"5min":186.21299999999999,"15min":186.21299999999999},"last":8.6140000000000008},{"interface":"front","average":{"1min":7.0780000000000003,"5min":6.1079999999999997,"15min":4.0209999999999999},"min":{"1min":0.23000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":78.810000000000002,"5min":191.78,"15min":191.78},"last":3.1200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":3.1579999999999999,"15min":3.097},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":12.593,"5min":72.343999999999994,"15min":106.105},"last":2.4449999999999998},{"interface":"front","average":{"1min":6.9119999999999999,"5min":4.5990000000000002,"15min":3.931},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":80.676000000000002,"5min":80.676000000000002,"15min":106.70399999999999},"last":6.4470000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.0899999999999999,"15min":2.4260000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1529999999999996,"5min":27.539999999999999,"15min":106.85899999999999},"last":6.5179999999999998},{"interface":"front","average":{"1min":2.8239999999999998,"5min":2.4329999999999998,"15min":1.9630000000000001},"min":{"1min":0.34999999999999998,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":8.8550000000000004,"5min":26.663,"15min":37.442},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5009999999999999,"5min":2.5880000000000001,"15min":1.9079999999999999},"min":{"1min":0.315,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":11.785,"5min":17.901,"15min":17.901},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.3849999999999998,"5min":2.2509999999999999,"15min":1.8899999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":8.2309999999999999,"5min":18.068000000000001,"15min":18.068000000000001},"last":2.4820000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":2.133,"15min":2.2440000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.758,"5min":14.976000000000001,"15min":106.036},"last":2.5259999999999998},{"interface":"front","average":{"1min":2.1309999999999998,"5min":2.2629999999999999,"15min":2.3279999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.1069999999999993,"5min":14.529,"15min":106.741},"last":6.5439999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9430000000000001,"5min":2.1160000000000001,"15min":2.3220000000000001},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.34399999999999997},"max":{"1min":7.1280000000000001,"5min":14.291,"15min":106.82599999999999},"last":6.4900000000000002},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.6400000000000001,"15min":2.3809999999999998},"min":{"1min":0.51300000000000001,"5min":0.38900000000000001,"15min":0.35999999999999999},"max":{"1min":11.539999999999999,"5min":20.757999999999999,"15min":106.084},"last":4.6500000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986484,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":831408,"kb_used_data":521832,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92913744,"statfs":{"total":95995035648,"available":95143673856,"internally_reserved":0,"allocated":534355968,"data_stored":533510147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":13.718,"5min":7.4370000000000003,"15min":3.653},"min":{"1min":0.23000000000000001,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":154.62100000000001,"5min":154.62100000000001,"15min":154.62100000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":13.945,"5min":7.5339999999999998,"15min":3.6349999999999998},"min":{"1min":0.28299999999999997,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":154.37299999999999,"5min":154.37299999999999,"15min":154.37299999999999},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":12.978,"5min":6.6230000000000002,"15min":3.4649999999999999},"min":{"1min":0.29799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":96.861999999999995,"5min":117.81,"15min":117.81},"last":0.308},{"interface":"front","average":{"1min":12.298999999999999,"5min":6.3849999999999998,"15min":3.3799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":96.787000000000006,"5min":117.848,"15min":117.848},"last":5.6619999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.8159999999999998,"15min":1.835},"min":{"1min":0.308,"5min":0.255,"15min":0.245},"max":{"1min":7.5460000000000003,"5min":123.598,"15min":123.598},"last":1.8779999999999999},{"interface":"front","average":{"1min":1.5529999999999999,"5min":2.7029999999999998,"15min":1.756},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":5.0220000000000002,"5min":123.619,"15min":123.619},"last":1.9039999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":3.298,"15min":2.02},"min":{"1min":0.36699999999999999,"5min":0.308,"15min":0.27100000000000002},"max":{"1min":9.5310000000000006,"5min":125.90300000000001,"15min":125.90300000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.855,"5min":3.3900000000000001,"15min":2.1669999999999998},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.29099999999999998},"max":{"1min":8.1389999999999993,"5min":127.90000000000001,"15min":127.90000000000001},"last":1.7230000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:16:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":3.1869999999999998,"15min":1.4810000000000001},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":7.5049999999999999,"5min":111.809,"15min":111.809},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.792,"5min":3.2669999999999999,"15min":1.637},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.25},"max":{"1min":6.6520000000000001,"5min":112.002,"15min":112.002},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.085,"15min":1.254},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":5.5519999999999996,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.8799999999999999,"15min":1.1100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.5430000000000001,"5min":78.421999999999997,"15min":78.421999999999997},"last":0.80600000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:16:24 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.0740000000000001,"15min":0.878},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.718,"5min":34.039000000000001,"15min":34.039000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.1140000000000001,"15min":0.93700000000000006},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.6029999999999998,"5min":33.433999999999997,"15min":33.433999999999997},"last":0.72199999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724848,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145572,"kb_used_data":831644,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92599580,"statfs":{"total":95995035648,"available":94821969920,"internally_reserved":0,"allocated":851603456,"data_stored":850765633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":8.5860000000000003,"5min":10.26,"15min":4.7969999999999997},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":111.312,"5min":151.23599999999999,"15min":151.23599999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":9.0990000000000002,"5min":10.897,"15min":5.1420000000000003},"min":{"1min":0.23000000000000001,"5min":0.22,"15min":0.22},"max":{"1min":111.096,"5min":151.566,"15min":151.566},"last":0.33600000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":7.6699999999999999,"5min":5.8490000000000002,"15min":3.472},"min":{"1min":0.34000000000000002,"5min":0.217,"15min":0.217},"max":{"1min":111.194,"5min":111.194,"15min":111.194},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.3539999999999992,"5min":6.2759999999999998,"15min":3.8140000000000001},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":111.247,"5min":112.226,"15min":112.226},"last":1.458}]},{"osd":2,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7549999999999999,"5min":4.4320000000000004,"15min":2.8079999999999998},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":9.8640000000000008,"5min":106.508,"15min":106.508},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.6219999999999999,"5min":4.0990000000000002,"15min":2.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":8.5280000000000005,"5min":106.462,"15min":106.462},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.5099999999999998,"15min":2.343},"min":{"1min":0.38,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.544,"5min":27.925000000000001,"15min":91.834999999999994},"last":0.32500000000000001},{"interface":"front","average":{"1min":3.4460000000000002,"5min":2.609,"15min":2.4129999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.122999999999999,"5min":27.939,"15min":91.802000000000007},"last":0.40400000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":1.1539999999999999,"15min":0.88500000000000001},"min":{"1min":0.251,"5min":0.246,"15min":0.246},"max":{"1min":10.162000000000001,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.9770000000000001,"5min":0.94799999999999995,"15min":0.86499999999999999},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.297000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":0.96099999999999997,"15min":1.2609999999999999},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":12.068,"5min":12.068,"15min":123.239},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.913,"5min":1.2709999999999999,"15min":1.373},"min":{"1min":0.33800000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.7720000000000002,"5min":9.7720000000000002,"15min":123.014},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":0.98199999999999998,"15min":0.92000000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.096,"5min":12.096,"15min":12.096},"last":0.432},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.2370000000000001,"15min":1.016},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.307},"max":{"1min":9.9760000000000009,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.48499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":728956928,"data_stored":728956928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":769064960,"data_stored":769064960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":742199296,"data_stored":742195224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":743571456,"data_stored":743571456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":528547840,"data_stored":528547840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":839061504,"data_stored":839057432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":700973056,"data_stored":700973056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":737677312,"data_stored":737673240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:17:10.488 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-10-09T04:17:10.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:10.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:10.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:10.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:10.651 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:10.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:10.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:10.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:10.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:10 smithi163 ceph-mon[213841]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:17:10.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:10 smithi163 ceph-mon[213841]: pgmap v791: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 1046/75771 objects degraded (1.380%); 7337/75771 objects misplaced (9.683%); 2.8 MiB/s, 64 objects/s recovering 2023-10-09T04:17:10.838 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:17:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:17:10] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-10-09T04:17:10.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:10 smithi163 ceph-mon[221067]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:17:10.839 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:10 smithi163 ceph-mon[221067]: pgmap v791: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 1046/75771 objects degraded (1.380%); 7337/75771 objects misplaced (9.683%); 2.8 MiB/s, 64 objects/s recovering 2023-10-09T04:17:10.850 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:10.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:10.925 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:10 smithi181 ceph-mon[196979]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:17:10.925 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:10 smithi181 ceph-mon[196979]: pgmap v791: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 1046/75771 objects degraded (1.380%); 7337/75771 objects misplaced (9.683%); 2.8 MiB/s, 64 objects/s recovering 2023-10-09T04:17:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:10.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:11.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:11.072 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:11.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:11.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:11.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:11.178 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:11.242 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:11.270 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:11.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:11.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:11.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:11.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:11.541 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:11.778 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:11 smithi163 ceph-mon[221067]: 1.0 scrub starts 2023-10-09T04:17:11.778 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:11 smithi163 ceph-mon[221067]: 1.0 scrub ok 2023-10-09T04:17:12.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:11 smithi163 ceph-mon[213841]: 1.0 scrub starts 2023-10-09T04:17:12.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:11 smithi163 ceph-mon[213841]: 1.0 scrub ok 2023-10-09T04:17:12.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:11 smithi181 ceph-mon[196979]: 1.0 scrub starts 2023-10-09T04:17:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:11 smithi181 ceph-mon[196979]: 1.0 scrub ok 2023-10-09T04:17:12.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:12.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:12.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:12.509 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:12.512 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:12.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:12.552 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:12.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:12.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:12.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:12.825 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:12 smithi181 ceph-mon[196979]: pgmap v792: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/69282 objects degraded (1.510%); 6814/69282 objects misplaced (9.835%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:17:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/69282 objects degraded (1.510%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:13.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:12 smithi163 ceph-mon[221067]: pgmap v792: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/69282 objects degraded (1.510%); 6814/69282 objects misplaced (9.835%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:17:13.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/69282 objects degraded (1.510%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:13.090 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:12 smithi163 ceph-mon[213841]: pgmap v792: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/69282 objects degraded (1.510%); 6814/69282 objects misplaced (9.835%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:17:13.090 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/69282 objects degraded (1.510%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:13.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:13.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:13.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:13.321 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:13.373 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:13.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:13.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:13.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:13.771 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:13.809 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:13.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:13.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:14.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:14.029 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:14.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:14.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:14.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:14.292 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:14.414 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:14.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:14.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:14.622 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:14.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:14 smithi163 ceph-mon[213841]: pgmap v793: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/69282 objects degraded (1.510%); 6814/69282 objects misplaced (9.835%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:17:15.074 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:15.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:15.079 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:14 smithi163 ceph-mon[221067]: pgmap v793: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/69282 objects degraded (1.510%); 6814/69282 objects misplaced (9.835%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:17:15.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:15.210 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:14 smithi181 ceph-mon[196979]: pgmap v793: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/69282 objects degraded (1.510%); 6814/69282 objects misplaced (9.835%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:17:15.213 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:15.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:15.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:15.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:15.405 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:15.493 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:15.631 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:17:15.698 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:17:15.749 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:17:15.783 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:17:15.814 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:17:15.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:17:15.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:17:15.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:17:15.935 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:17:16.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:15 smithi163 ceph-mon[221067]: 1.7 scrub starts 2023-10-09T04:17:16.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:15 smithi163 ceph-mon[221067]: 1.7 scrub ok 2023-10-09T04:17:16.126 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2760469768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:16.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:15 smithi163 ceph-mon[213841]: 1.7 scrub starts 2023-10-09T04:17:16.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:15 smithi163 ceph-mon[213841]: 1.7 scrub ok 2023-10-09T04:17:16.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2760469768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:15 smithi181 ceph-mon[196979]: 1.7 scrub starts 2023-10-09T04:17:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:15 smithi181 ceph-mon[196979]: 1.7 scrub ok 2023-10-09T04:17:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2760469768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:17.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:16 smithi163 ceph-mon[221067]: pgmap v794: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 1046/65793 objects degraded (1.590%); 6449/65793 objects misplaced (9.802%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:17:17.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:16 smithi163 ceph-mon[213841]: pgmap v794: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 1046/65793 objects degraded (1.590%); 6449/65793 objects misplaced (9.802%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:17:17.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:17.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:16 smithi181 ceph-mon[196979]: pgmap v794: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 1046/65793 objects degraded (1.590%); 6449/65793 objects misplaced (9.802%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:17:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:17.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:17.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:17.479 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:17.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:17.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:17.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:17.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:18.008 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:18.095 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:17:18.095 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:17:18.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/65793 objects degraded (1.590%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:18.138 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3202180684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:18.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/65793 objects degraded (1.590%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:18.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3202180684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:18.152 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:18.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:18.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:18.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/65793 objects degraded (1.590%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:18.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3202180684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:18.310 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:18.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:18.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:18.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:18.497 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 1 pgp_num to 21 2023-10-09T04:17:18.508 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:17:18.607 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:18.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:18.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:18.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:18.835 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: pgmap v795: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1046/60237 objects degraded (1.736%); 6086/60237 objects misplaced (10.103%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1904930127' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: failed to encode map e289 with expected crc 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3202180684' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-10-09T04:17:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:18 smithi181 ceph-mon[196979]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T04:17:19.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:17:19.295 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: pgmap v795: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1046/60237 objects degraded (1.736%); 6086/60237 objects misplaced (10.103%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1904930127' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: failed to encode map e289 with expected crc 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3202180684' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-10-09T04:17:19.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[213841]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: pgmap v795: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1046/60237 objects degraded (1.736%); 6086/60237 objects misplaced (10.103%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1904930127' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: failed to encode map e289 with expected crc 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3202180684' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-10-09T04:17:19.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:18 smithi163 ceph-mon[221067]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T04:17:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:19.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:19.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:19.439 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:19.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:19.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:19.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:19.618 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:19.679 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:19.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:19.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:19.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:19.897 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:19 smithi181 ceph-mon[196979]: failed to encode map e289 with expected crc 2023-10-09T04:17:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:19 smithi181 ceph-mon[196979]: failed to encode map e289 with expected crc 2023-10-09T04:17:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:20.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:19 smithi163 ceph-mon[213841]: failed to encode map e289 with expected crc 2023-10-09T04:17:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:19 smithi163 ceph-mon[213841]: failed to encode map e289 with expected crc 2023-10-09T04:17:20.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:19 smithi163 ceph-mon[221067]: failed to encode map e289 with expected crc 2023-10-09T04:17:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:19 smithi163 ceph-mon[221067]: failed to encode map e289 with expected crc 2023-10-09T04:17:20.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:20.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:20.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:20.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:20.710 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:17:20.733 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:20.810 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:17:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:17:20] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-10-09T04:17:20.850 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:20.941 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:21.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:21.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:21.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:20 smithi163 ceph-mon[213841]: pgmap v797: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 1046/58227 objects degraded (1.796%); 5906/58227 objects misplaced (10.143%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:21.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:20 smithi163 ceph-mon[221067]: pgmap v797: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 1046/58227 objects degraded (1.796%); 5906/58227 objects misplaced (10.143%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:21.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:21.235 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:21.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:20 smithi181 ceph-mon[196979]: pgmap v797: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 1046/58227 objects degraded (1.796%); 5906/58227 objects misplaced (10.143%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:21.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:21.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:21.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:21.541 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:21.607 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:21.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:21.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:21.822 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:21.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:21.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:22.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:22.106 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:22.164 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:22.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:22.271 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:22.302 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:22.343 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:22.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:22.447 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:22.473 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:22.519 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:22.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:22.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:22.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:17:22.626 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:22.746 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:22.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:22 smithi163 ceph-mon[213841]: pgmap v798: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1046/51936 objects degraded (2.014%); 5398/51936 objects misplaced (10.394%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:22.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/51936 objects degraded (2.014%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:22.931 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:22 smithi163 ceph-mon[221067]: pgmap v798: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1046/51936 objects degraded (2.014%); 5398/51936 objects misplaced (10.394%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:22.932 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/51936 objects degraded (2.014%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:22.944 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:22.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:22.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:22.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:23.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:23.120 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:23.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:23.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:23.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:23.207 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:22 smithi181 ceph-mon[196979]: pgmap v798: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1046/51936 objects degraded (2.014%); 5398/51936 objects misplaced (10.394%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:23.207 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/51936 objects degraded (2.014%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:17:23.308 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:23.398 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:23.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:23.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:23.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:23.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:23.878 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:17:23.911 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:17:23.959 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:17:23.960 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:23.998 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:17:24.037 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:17:24.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1207243868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:24.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1207243868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:24.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:24.091 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:17:24.131 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:17:24.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:24.221 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1207243868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:24.223 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:24.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:24.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:24.335 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.336 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:17:24.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:24.429 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:24.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:24.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:24.788 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:24.836 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:24.907 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:24.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:25.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:25.048 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:24 smithi163 ceph-mon[213841]: pgmap v799: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1046/51936 objects degraded (2.014%); 5398/51936 objects misplaced (10.394%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:24 smithi163 ceph-mon[221067]: pgmap v799: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1046/51936 objects degraded (2.014%); 5398/51936 objects misplaced (10.394%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:25.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:25.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:25.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:25.133 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:24 smithi181 ceph-mon[196979]: pgmap v799: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1046/51936 objects degraded (2.014%); 5398/51936 objects misplaced (10.394%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:17:25.233 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:25.273 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:25.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:25.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:25.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:25.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:25.486 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:25.564 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:25.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:25.611 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:17:25.612 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:17:25.612 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:17:25.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:17:25.618 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:25.676 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:25.855 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:25.945 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:25 smithi163 ceph-mon[213841]: pgmap v800: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1015 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/48762 objects degraded (2.145%); 5083/48762 objects misplaced (10.424%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:17:26.123 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:17:26.123 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:17:26.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:17:26.123 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:17:26.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.123 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:644 7.c 7:3fcf2acf:::benchmark_data_smithi163_346082_object643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 477.52684792700001, 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21846595499999999, 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 644 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591666+0000", 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591664+0000", 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591688+0000", 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3495000000000001e-05 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591699+0000", 2023-10-09T04:17:26.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1834000000000001e-05 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591707+0000", 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6029999999999999e-06 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591761+0000", 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4044e-05 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591884+0000", 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012263899999999999 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592028+0000", 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014450300000000001 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809992+0000", 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21796359500000001 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810008+0000", 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6293000000000001e-05 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810044+0000", 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5468e-05 2023-10-09T04:17:26.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810090+0000", 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6409999999999998e-05 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810132+0000", 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1647999999999998e-05 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:645 7.c 7:3c7dda99:::benchmark_data_smithi163_346082_object644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 477.526792935, 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21961103900000001, 2023-10-09T04:17:26.131 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 645 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591721+0000", 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591720+0000", 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591740+0000", 2023-10-09T04:17:26.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9584000000000001e-05 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591748+0000", 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591755+0000", 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7190000000000001e-06 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809375+0000", 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.217619379 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809503+0000", 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012796899999999999 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:17:26.135 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809689+0000", 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000186577 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810749+0000", 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010602400000000001 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810776+0000", 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6995000000000002e-05 2023-10-09T04:17:26.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811241+0000", 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046449799999999998 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811284+0000", 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3325999999999998e-05 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.811332+0000", 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7512999999999997e-05 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:647 7.2 e171/167 7:4b12a746:::benchmark_data_smithi163_346082_object646:head v 171'43, mlcod=171'39)", 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 477.52654002600002, 2023-10-09T04:17:26.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21782781300000001, 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591974+0000", 2023-10-09T04:17:26.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591973+0000", 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.591996+0000", 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2821000000000001e-05 2023-10-09T04:17:26.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592006+0000", 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0288999999999999e-05 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592016+0000", 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0186e-05 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592076+0000", 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9373000000000002e-05 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808728+0000", 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21665226300000001 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809757+0000", 2023-10-09T04:17:26.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001029409 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809801+0000", 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4079000000000003e-05 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.143 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:644 7.c e171/167 ondisk, result = 0)", 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 477.52577398300002, 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.217269874, 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592740+0000", 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592739+0000", 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592743+0000", 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.442e-06 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808711+0000", 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21596727399999999 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.808728+0000", 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7541999999999999e-05 2023-10-09T04:17:26.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809978+0000", 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001249687 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809985+0000", 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.498e-06 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810010+0000", 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4077999999999999e-05 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:644 7.c e171/167 ondisk, result = 0)", 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 477.52567553599999, 2023-10-09T04:17:26.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21730428800000001, 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592838+0000", 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.592842+0000", 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1080000000000003e-06 2023-10-09T04:17:26.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809055+0000", 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.216213023 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.809069+0000", 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3861e-05 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810034+0000", 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000965032 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810039+0000", 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5489999999999997e-06 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:28.810142+0000", 2023-10-09T04:17:26.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010331999999999999 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:38946 7.c 7:3abe6352:::benchmark_data_smithi163_346082_object38945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 455.14914052, 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23044379800000001, 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.153 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38946 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969373+0000", 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969372+0000", 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969391+0000", 2023-10-09T04:17:26.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9732999999999999e-05 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969404+0000", 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2989999999999999e-05 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969412+0000", 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9889999999999992e-06 2023-10-09T04:17:26.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969455+0000", 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3009000000000001e-05 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969584+0000", 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012886500000000001 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.969717+0000", 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013274599999999999 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972017+0000", 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023001100000000002 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199698+0000", 2023-10-09T04:17:26.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22768131599999999 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199742+0000", 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.375e-05 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199779+0000", 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6940999999999997e-05 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199817+0000", 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7861000000000002e-05 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38943 7.16 e175/174 7:6e66cd39:::benchmark_data_smithi163_346082_object38942:head v 175'2310, mlcod=175'2307)", 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 455.147153188, 2023-10-09T04:17:26.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22686308999999999, 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:17:26.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971360+0000", 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971382+0000", 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1302e-05 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971388+0000", 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0599999999999996e-06 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.971393+0000", 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3909999999999997e-06 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972116+0000", 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072317400000000004 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972130+0000", 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3424999999999999e-05 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972810+0000", 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068082899999999998 2023-10-09T04:17:26.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198224+0000", 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.225413118 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:38951 7.c 7:39c83fcb:::benchmark_data_smithi163_346082_object38950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-10-09T04:17:26.165 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 455.14623098800001, 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22769973700000001, 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38951 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972283+0000", 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972281+0000", 2023-10-09T04:17:26.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972307+0000", 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5286000000000002e-05 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972319+0000", 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2325e-05 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972341+0000", 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1837000000000001e-05 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972386+0000", 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5516000000000003e-05 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972524+0000", 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013795999999999999 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972659+0000", 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013506 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199867+0000", 2023-10-09T04:17:26.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22720789899999999 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199914+0000", 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6495999999999997e-05 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199923+0000", 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7970000000000006e-06 2023-10-09T04:17:26.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199950+0000", 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7152999999999999e-05 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199982+0000", 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2765000000000002e-05 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.172 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38945 7.a e175/167 7:5f8231b3:::benchmark_data_smithi163_346082_object38944:head v 175'2413, mlcod=175'2410)", 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 455.14571842800001, 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.228137689, 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.174 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972795+0000", 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972816+0000", 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1577999999999999e-05 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198218+0000", 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22540215399999999 2023-10-09T04:17:26.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198227+0000", 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9449999999999999e-06 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198279+0000", 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1304999999999998e-05 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198298+0000", 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9219999999999999e-05 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200904+0000", 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002605797 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200933+0000", 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9247e-05 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35115.0:38946 7.c e175/167 ondisk, result = 0)", 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 455.14569101400002, 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22688220200000001, 2023-10-09T04:17:26.178 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972823+0000", 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972822+0000", 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.972826+0000", 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1230000000000002e-06 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198187+0000", 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22536124799999999 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198204+0000", 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7017999999999999e-05 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199684+0000", 2023-10-09T04:17:26.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001479209 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199692+0000", 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1899999999999995e-06 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.199705+0000", 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2989999999999999e-05 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.182 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35115.0:38948 7.11 e175/174 7:88c099c1:::benchmark_data_smithi163_346082_object38947:head v 175'2351, mlcod=175'2349)", 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 455.14513586100003, 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22730851799999999, 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973378+0000", 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973377+0000", 2023-10-09T04:17:26.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:50.973404+0000", 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7041000000000001e-05 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198189+0000", 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22478490800000001 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198205+0000", 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6181e-05 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198251+0000", 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6136e-05 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.198278+0000", 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7257e-05 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200653+0000", 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023747289999999999 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:09:51.200686+0000", 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3191999999999997e-05 2023-10-09T04:17:26.187 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80498 7.13 7:ce654277:::benchmark_data_smithi163_346082_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.80067292899997, 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1130738259999999, 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.188 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80498 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317841+0000", 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317838+0000", 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317881+0000", 2023-10-09T04:17:26.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3013000000000003e-05 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317895+0000", 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3732e-05 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317903+0000", 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5209999999999997e-06 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318091+0000", 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188829 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318101+0000", 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.55e-06 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323224+0000", 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051229439999999999 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323236+0000", 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1992e-05 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.423747+0000", 2023-10-09T04:17:26.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.100511362 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.423904+0000", 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015625099999999999 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.424075+0000", 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000170977 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430599+0000", 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065249189999999997 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430661+0000", 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1789000000000001e-05 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430676+0000", 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4297999999999999e-05 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430753+0000", 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7937999999999998e-05 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430805+0000", 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1829000000000001e-05 2023-10-09T04:17:26.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430915+0000", 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109257 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80508 7.13 7:cf9b83a2:::benchmark_data_smithi163_346082_object80507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:17:26.197 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.800596816, 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1132838570000001, 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80508 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317917+0000", 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317916+0000", 2023-10-09T04:17:26.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317944+0000", 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7642000000000001e-05 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317952+0000", 2023-10-09T04:17:26.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3070000000000003e-06 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317958+0000", 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1290000000000001e-06 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318152+0000", 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019322999999999999 2023-10-09T04:17:26.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318158+0000", 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7700000000000004e-06 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323259+0000", 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051002950000000003 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323278+0000", 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814999999999999e-05 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425771+0000", 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.102493095 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425915+0000", 2023-10-09T04:17:26.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000144451 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.426078+0000", 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000162595 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430682+0000", 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046044320000000003 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430721+0000", 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9020000000000002e-05 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430993+0000", 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027199400000000001 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431048+0000", 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4808000000000001e-05 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431097+0000", 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9586000000000003e-05 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431201+0000", 2023-10-09T04:17:26.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103231 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80516 7.13 7:c9a4c09d:::benchmark_data_smithi163_346082_object80515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.80055005999998, 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.116727963, 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.207 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80516 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317964+0000", 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317963+0000", 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317983+0000", 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9938000000000001e-05 2023-10-09T04:17:26.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317990+0000", 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7850000000000003e-06 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.317993+0000", 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3680000000000002e-06 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318180+0000", 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018708599999999999 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.318187+0000", 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2589999999999997e-06 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323302+0000", 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051155979999999998 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.323310+0000", 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0660000000000004e-06 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427478+0000", 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1041673599999999 2023-10-09T04:17:26.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427613+0000", 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000135888 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.427760+0000", 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014688799999999999 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430688+0000", 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029275270000000001 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431261+0000", 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00057298199999999998 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434445+0000", 2023-10-09T04:17:26.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031837290000000002 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434514+0000", 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8955999999999995e-05 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434567+0000", 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3599e-05 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434692+0000", 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012444600000000001 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80551 7.13 7:c9852ceb:::benchmark_data_smithi163_346082_object80550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:17:26.216 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.64803963200001, 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96918742400000002, 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80551 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470474+0000", 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470472+0000", 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:26.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470498+0000", 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5672000000000001e-05 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470514+0000", 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5846999999999999e-05 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470522+0000", 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9629999999999998e-06 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470576+0000", 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4360999999999997e-05 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470594+0000", 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7754999999999999e-05 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.428996+0000", 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95840238700000002 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429120+0000", 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012304300000000001 2023-10-09T04:17:26.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429313+0000", 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019363099999999999 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431216+0000", 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019029240000000001 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434407+0000", 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031906230000000001 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434794+0000", 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00038685899999999998 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439487+0000", 2023-10-09T04:17:26.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046938939999999997 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439541+0000", 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3309000000000001e-05 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439661+0000", 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120699 2023-10-09T04:17:26.224 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80553 7.3 7:c0606c4f:::benchmark_data_smithi163_346082_object80552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.64484950899998, 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95719071, 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.225 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80553 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473664+0000", 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473663+0000", 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473684+0000", 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1585000000000001e-05 2023-10-09T04:17:26.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473699+0000", 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4456e-05 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473705+0000", 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6630000000000004e-06 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473767+0000", 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1607999999999997e-05 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.473779+0000", 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2330999999999999e-05 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.419942+0000", 2023-10-09T04:17:26.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94616239800000002 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.420106+0000", 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016377699999999999 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.420277+0000", 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017136500000000001 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425713+0000", 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005435601 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.425748+0000", 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5747999999999997e-05 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430649+0000", 2023-10-09T04:17:26.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049003709999999997 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430704+0000", 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5140999999999999e-05 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430751+0000", 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7691999999999997e-05 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.430855+0000", 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103417 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80558 7.13 7:c83af463:::benchmark_data_smithi163_346082_object80557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.638953636, 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96028118200000001, 2023-10-09T04:17:26.233 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80558 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479560+0000", 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479559+0000", 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479582+0000", 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2878999999999999e-05 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479594+0000", 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1328000000000001e-05 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479601+0000", 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3690000000000001e-06 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479661+0000", 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0053000000000003e-05 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.479673+0000", 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1877e-05 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431306+0000", 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95163314200000004 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431474+0000", 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016787400000000001 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4,6", 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.431652+0000", 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017823200000000001 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434479+0000", 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028264900000000001 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434839+0000", 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00036005299999999999 2023-10-09T04:17:26.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.434892+0000", 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3001000000000003e-05 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439712+0000", 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048203079999999997 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439752+0000", 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9496999999999998e-05 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.439841+0000", 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9649999999999997e-05 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:122577 8.f e271/241 8:f3163eda:::benchmark_data_smithi163_404719_object31993:head v 271'7855, mlcod=271'7853)", 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:08.504457+0000", 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 77.614057008000003, 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10856619100000001, 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.504457+0000", 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.504457+0000", 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.504456+0000", 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.504463+0000", 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1980000000000004e-06 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.504473+0000", 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4579999999999999e-06 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.504479+0000", 2023-10-09T04:17:26.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4359999999999999e-06 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.586389+0000", 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.081910208999999998 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.586420+0000", 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0355999999999999e-05 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.612973+0000", 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.026553335000000001 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:08.613023+0000", 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9929999999999998e-05 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:131768 8.b e277/276 8:d94053f1:::benchmark_data_smithi163_404719_object41184:head v 277'8257, mlcod=0'0)", 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:24.654905+0000", 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.463608589000003, 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.104257012, 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654905+0000", 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654905+0000", 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654904+0000", 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654914+0000", 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8530000000000004e-06 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654937+0000", 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3285000000000002e-05 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654947+0000", 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0392e-05 2023-10-09T04:17:26.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.655242+0000", 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029410699999999999 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.655271+0000", 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9618e-05 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.759109+0000", 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.103838073 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.759162+0000", 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2828999999999999e-05 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:147253 8.f e284/241 8:f332af63:::benchmark_data_smithi163_404719_object56669:head v 284'9448, mlcod=284'9444)", 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:50.975084+0000", 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 35.143429212000001, 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.092493423000000005, 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.975084+0000", 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.975084+0000", 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.975084+0000", 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.975088+0000", 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1019999999999999e-06 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.975094+0000", 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8819999999999998e-06 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:17:26.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.975099+0000", 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6380000000000003e-06 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.979787+0000", 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046877910000000002 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:50.979807+0000", 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0003000000000001e-05 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:51.067535+0000", 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.087728532999999997 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:51.067578+0000", 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2824000000000002e-05 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.256 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:17:26.257 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:17:26.257 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:17:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:26.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:26.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:25 smithi181 ceph-mon[196979]: pgmap v800: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1015 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/48762 objects degraded (2.145%); 5083/48762 objects misplaced (10.424%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:17:26.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:25 smithi163 ceph-mon[221067]: pgmap v800: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1015 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1046/48762 objects degraded (2.145%); 5083/48762 objects misplaced (10.424%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:17:26.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:26.477 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:27.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:17:27.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/48762 objects degraded (2.145%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:27.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/48762 objects degraded (2.145%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:27.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/48762 objects degraded (2.145%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:27.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:27.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:27.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:17:28.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:28.132 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:28.161 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:28.372 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:28.379 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:28.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:28.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:28.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:28.563 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:28.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:28.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:28.671 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:28 smithi163 ceph-mon[213841]: pgmap v801: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 904 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 1046/43449 objects degraded (2.407%); 4697/43449 objects misplaced (10.810%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:17:28.671 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:28 smithi163 ceph-mon[221067]: pgmap v801: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 904 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 1046/43449 objects degraded (2.407%); 4697/43449 objects misplaced (10.810%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:17:28.742 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:28.772 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:28.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:28 smithi181 ceph-mon[196979]: pgmap v801: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 904 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 1046/43449 objects degraded (2.407%); 4697/43449 objects misplaced (10.810%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:17:28.856 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:29.302 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:29.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:29.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:29.577 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:17:29.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3436760479' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:29.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:29.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3436760479' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:29.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:29.722 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:30.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3436760479' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:30.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:30.169 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:30.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:30.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:30.317 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:30.397 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:30.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:30.504 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:30.571 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:30 smithi181 ceph-mon[196979]: pgmap v803: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 860 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1046/41331 objects degraded (2.531%); 4537/41331 objects misplaced (10.977%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:30.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:17:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:17:30] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-10-09T04:17:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:30.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T04:17:30.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[213841]: pgmap v803: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 860 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1046/41331 objects degraded (2.531%); 4537/41331 objects misplaced (10.977%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: failed to encode map e290 with expected crc 2023-10-09T04:17:30.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:30 smithi163 ceph-mon[221067]: pgmap v803: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 860 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1046/41331 objects degraded (2.531%); 4537/41331 objects misplaced (10.977%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:31.020 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:31.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:31.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:31.213 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:31.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:31.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:31.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:31.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:31.743 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:31.886 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:31.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:31.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:31.949 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:17:31.951 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:17:32.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:32.026 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:32.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:32.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:32.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:32.240 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:32.660 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:32.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:32.768 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:32.818 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:32.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:32.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:32.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:32.877 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:17:32.878 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:17:32.878 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:17:32.878 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:17:32.910 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:32 smithi181 ceph-mon[196979]: pgmap v804: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 717 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 1046/34476 objects degraded (3.034%); 3980/34476 objects misplaced (11.544%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:32.910 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1046/34476 objects degraded (3.034%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:32.970 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:33.046 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:32 smithi163 ceph-mon[213841]: pgmap v804: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 717 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 1046/34476 objects degraded (3.034%); 3980/34476 objects misplaced (11.544%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:33.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1046/34476 objects degraded (3.034%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:33.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:32 smithi163 ceph-mon[221067]: pgmap v804: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 717 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 1046/34476 objects degraded (3.034%); 3980/34476 objects misplaced (11.544%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:33.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1046/34476 objects degraded (3.034%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:33.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:33.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:33.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:17:33.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:33.261 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:33.309 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":804,"stamp":"2023-10-09T04:17:31.742356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":752034957,"num_objects":11492,"num_object_clones":0,"num_object_copies":34476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":3980,"num_objects_unfound":0,"num_objects_dirty":11492,"num_whiteouts":0,"num_read":285,"num_read_kb":519,"num_write":169894,"num_write_kb":5798571,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14815,"num_bytes_recovered":945241717,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133998,"ondisk_log_size":133998,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":333,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6003244,"kb_used_data":2843244,"kb_used_omap":126,"kb_used_meta":3159105,"kb_avail":743957972,"statfs":{"total":767960285184,"available":761812963328,"internally_reserved":0,"allocated":2911481856,"data_stored":2903123259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3234924155},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,6,6,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-381419520,"num_objects":-5820,"num_object_clones":0,"num_object_copies":-17460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1418,"num_objects_unfound":0,"num_objects_dirty":-5820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5820,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":20250624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008493"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483880+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:17:18.483880+0000","last_peered":"2023-10-09T04:17:18.483880+0000","last_clean":"2023-10-09T04:17:18.483880+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:17:18.483880+0000","last_undegraded":"2023-10-09T04:17:18.483880+0000","last_fullsized":"2023-10-09T04:17:18.483880+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":245,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.751994+0000","last_change":"2023-10-09T04:15:25.414174+0000","last_active":"2023-10-09T04:17:05.751994+0000","last_peered":"2023-10-09T04:17:05.751994+0000","last_clean":"2023-10-09T04:17:05.751994+0000","last_became_active":"2023-10-09T04:15:25.413807+0000","last_became_peered":"2023-10-09T04:15:25.413807+0000","last_unstale":"2023-10-09T04:17:05.751994+0000","last_undegraded":"2023-10-09T04:17:05.751994+0000","last_fullsized":"2023-10-09T04:17:05.751994+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:01.968150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":260,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.751772+0000","last_change":"2023-10-09T04:15:25.414811+0000","last_active":"2023-10-09T04:17:05.751772+0000","last_peered":"2023-10-09T04:17:05.751772+0000","last_clean":"2023-10-09T04:17:05.751772+0000","last_became_active":"2023-10-09T04:15:25.414562+0000","last_became_peered":"2023-10-09T04:15:25.414562+0000","last_unstale":"2023-10-09T04:17:05.751772+0000","last_undegraded":"2023-10-09T04:17:05.751772+0000","last_fullsized":"2023-10-09T04:17:05.751772+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:10.270201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":257,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.479675+0000","last_change":"2023-10-09T04:16:24.469036+0000","last_active":"2023-10-09T04:17:18.479675+0000","last_peered":"2023-10-09T04:17:18.479675+0000","last_clean":"2023-10-09T04:17:18.479675+0000","last_became_active":"2023-10-09T04:16:24.468649+0000","last_became_peered":"2023-10-09T04:16:24.468649+0000","last_unstale":"2023-10-09T04:17:18.479675+0000","last_undegraded":"2023-10-09T04:17:18.479675+0000","last_fullsized":"2023-10-09T04:17:18.479675+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:34:53.505550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":246,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.475985+0000","last_change":"2023-10-09T04:06:45.366206+0000","last_active":"2023-10-09T04:17:18.475985+0000","last_peered":"2023-10-09T04:17:18.475985+0000","last_clean":"2023-10-09T04:17:18.475985+0000","last_became_active":"2023-10-09T04:06:45.365817+0000","last_became_peered":"2023-10-09T04:06:45.365817+0000","last_unstale":"2023-10-09T04:17:18.475985+0000","last_undegraded":"2023-10-09T04:17:18.475985+0000","last_fullsized":"2023-10-09T04:17:18.475985+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:16.972883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":223,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.486212+0000","last_change":"2023-10-09T04:06:57.498300+0000","last_active":"2023-10-09T04:17:18.486212+0000","last_peered":"2023-10-09T04:17:18.486212+0000","last_clean":"2023-10-09T04:17:18.486212+0000","last_became_active":"2023-10-09T04:06:57.497976+0000","last_became_peered":"2023-10-09T04:06:57.497976+0000","last_unstale":"2023-10-09T04:17:18.486212+0000","last_undegraded":"2023-10-09T04:17:18.486212+0000","last_fullsized":"2023-10-09T04:17:18.486212+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:47.227178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":247,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.480473+0000","last_change":"2023-10-09T04:03:37.217414+0000","last_active":"2023-10-09T04:17:18.480473+0000","last_peered":"2023-10-09T04:17:18.480473+0000","last_clean":"2023-10-09T04:17:18.480473+0000","last_became_active":"2023-10-09T04:03:37.217140+0000","last_became_peered":"2023-10-09T04:03:37.217140+0000","last_unstale":"2023-10-09T04:17:18.480473+0000","last_undegraded":"2023-10-09T04:17:18.480473+0000","last_fullsized":"2023-10-09T04:17:18.480473+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:05.304057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":270,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.478689+0000","last_change":"2023-10-09T04:03:37.221476+0000","last_active":"2023-10-09T04:17:18.478689+0000","last_peered":"2023-10-09T04:17:18.478689+0000","last_clean":"2023-10-09T04:17:18.478689+0000","last_became_active":"2023-10-09T04:03:37.220781+0000","last_became_peered":"2023-10-09T04:03:37.220781+0000","last_unstale":"2023-10-09T04:17:18.478689+0000","last_undegraded":"2023-10-09T04:17:18.478689+0000","last_fullsized":"2023-10-09T04:17:18.478689+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:43:47.823356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":652,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489684+0000","last_change":"2023-10-09T04:16:46.787881+0000","last_active":"2023-10-09T04:17:18.489684+0000","last_peered":"2023-10-09T04:17:18.489684+0000","last_clean":"2023-10-09T04:17:18.489684+0000","last_became_active":"2023-10-09T04:16:46.787580+0000","last_became_peered":"2023-10-09T04:16:46.787580+0000","last_unstale":"2023-10-09T04:17:18.489684+0000","last_undegraded":"2023-10-09T04:17:18.489684+0000","last_fullsized":"2023-10-09T04:17:18.489684+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.485775+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:17:18.485775+0000","last_peered":"2023-10-09T04:17:18.485775+0000","last_clean":"2023-10-09T04:17:18.485775+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:17:18.485775+0000","last_undegraded":"2023-10-09T04:17:18.485775+0000","last_fullsized":"2023-10-09T04:17:18.485775+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":108,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.477190+0000","last_change":"2023-10-09T04:13:46.101209+0000","last_active":"2023-10-09T04:17:18.477190+0000","last_peered":"2023-10-09T04:17:18.477190+0000","last_clean":"2023-10-09T04:17:18.477190+0000","last_became_active":"2023-10-09T04:13:46.100828+0000","last_became_peered":"2023-10-09T04:13:46.100828+0000","last_unstale":"2023-10-09T04:17:18.477190+0000","last_undegraded":"2023-10-09T04:17:18.477190+0000","last_fullsized":"2023-10-09T04:17:18.477190+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:57:59.155026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":284,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489974+0000","last_change":"2023-10-09T04:15:25.545630+0000","last_active":"2023-10-09T04:17:18.489974+0000","last_peered":"2023-10-09T04:17:18.489974+0000","last_clean":"2023-10-09T04:17:18.489974+0000","last_became_active":"2023-10-09T04:15:25.410551+0000","last_became_peered":"2023-10-09T04:15:25.410551+0000","last_unstale":"2023-10-09T04:17:18.489974+0000","last_undegraded":"2023-10-09T04:17:18.489974+0000","last_fullsized":"2023-10-09T04:17:18.489974+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:15.300784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":60,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.477946+0000","last_change":"2023-10-09T04:16:24.482108+0000","last_active":"2023-10-09T04:17:18.477946+0000","last_peered":"2023-10-09T04:17:18.477946+0000","last_clean":"2023-10-09T04:17:18.477946+0000","last_became_active":"2023-10-09T04:16:24.481813+0000","last_became_peered":"2023-10-09T04:16:24.481813+0000","last_unstale":"2023-10-09T04:17:18.477946+0000","last_undegraded":"2023-10-09T04:17:18.477946+0000","last_fullsized":"2023-10-09T04:17:18.477946+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:51.848459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":44,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.484412+0000","last_change":"2023-10-09T04:15:25.523555+0000","last_active":"2023-10-09T04:17:18.484412+0000","last_peered":"2023-10-09T04:17:18.484412+0000","last_clean":"2023-10-09T04:17:18.484412+0000","last_became_active":"2023-10-09T04:15:25.408994+0000","last_became_peered":"2023-10-09T04:15:25.408994+0000","last_unstale":"2023-10-09T04:17:18.484412+0000","last_undegraded":"2023-10-09T04:17:18.484412+0000","last_fullsized":"2023-10-09T04:17:18.484412+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:39.516242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483825+0000","last_change":"2023-10-09T04:15:25.476381+0000","last_active":"2023-10-09T04:17:18.483825+0000","last_peered":"2023-10-09T04:17:18.483825+0000","last_clean":"2023-10-09T04:17:18.483825+0000","last_became_active":"2023-10-09T04:15:25.475863+0000","last_became_peered":"2023-10-09T04:15:25.475863+0000","last_unstale":"2023-10-09T04:17:18.483825+0000","last_undegraded":"2023-10-09T04:17:18.483825+0000","last_fullsized":"2023-10-09T04:17:18.483825+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:46.820781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":651,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488973+0000","last_change":"2023-10-09T04:16:46.781567+0000","last_active":"2023-10-09T04:17:18.488973+0000","last_peered":"2023-10-09T04:17:18.488973+0000","last_clean":"2023-10-09T04:17:18.488973+0000","last_became_active":"2023-10-09T04:16:46.781202+0000","last_became_peered":"2023-10-09T04:16:46.781202+0000","last_unstale":"2023-10-09T04:17:18.488973+0000","last_undegraded":"2023-10-09T04:17:18.488973+0000","last_fullsized":"2023-10-09T04:17:18.488973+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":283,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489390+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:17:18.489390+0000","last_peered":"2023-10-09T04:17:18.489390+0000","last_clean":"2023-10-09T04:17:18.489390+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:17:18.489390+0000","last_undegraded":"2023-10-09T04:17:18.489390+0000","last_fullsized":"2023-10-09T04:17:18.489390+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":80,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489077+0000","last_change":"2023-10-09T04:16:24.616676+0000","last_active":"2023-10-09T04:17:18.489077+0000","last_peered":"2023-10-09T04:17:18.489077+0000","last_clean":"2023-10-09T04:17:18.489077+0000","last_became_active":"2023-10-09T04:16:24.616320+0000","last_became_peered":"2023-10-09T04:16:24.616320+0000","last_unstale":"2023-10-09T04:17:18.489077+0000","last_undegraded":"2023-10-09T04:17:18.489077+0000","last_fullsized":"2023-10-09T04:17:18.489077+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:32.174105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":653,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488918+0000","last_change":"2023-10-09T04:16:46.782428+0000","last_active":"2023-10-09T04:17:18.488918+0000","last_peered":"2023-10-09T04:17:18.488918+0000","last_clean":"2023-10-09T04:17:18.488918+0000","last_became_active":"2023-10-09T04:16:46.779655+0000","last_became_peered":"2023-10-09T04:16:46.779655+0000","last_unstale":"2023-10-09T04:17:18.488918+0000","last_undegraded":"2023-10-09T04:17:18.488918+0000","last_fullsized":"2023-10-09T04:17:18.488918+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":247,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.476266+0000","last_change":"2023-10-09T04:03:37.219148+0000","last_active":"2023-10-09T04:17:18.476266+0000","last_peered":"2023-10-09T04:17:18.476266+0000","last_clean":"2023-10-09T04:17:18.476266+0000","last_became_active":"2023-10-09T04:03:37.218691+0000","last_became_peered":"2023-10-09T04:03:37.218691+0000","last_unstale":"2023-10-09T04:17:18.476266+0000","last_undegraded":"2023-10-09T04:17:18.476266+0000","last_fullsized":"2023-10-09T04:17:18.476266+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:52.171020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":248,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.487771+0000","last_change":"2023-10-09T04:03:37.220488+0000","last_active":"2023-10-09T04:17:18.487771+0000","last_peered":"2023-10-09T04:17:18.487771+0000","last_clean":"2023-10-09T04:17:18.487771+0000","last_became_active":"2023-10-09T04:03:37.220069+0000","last_became_peered":"2023-10-09T04:03:37.220069+0000","last_unstale":"2023-10-09T04:17:18.487771+0000","last_undegraded":"2023-10-09T04:17:18.487771+0000","last_fullsized":"2023-10-09T04:17:18.487771+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:01.903341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":652,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489474+0000","last_change":"2023-10-09T04:16:46.781804+0000","last_active":"2023-10-09T04:17:18.489474+0000","last_peered":"2023-10-09T04:17:18.489474+0000","last_clean":"2023-10-09T04:17:18.489474+0000","last_became_active":"2023-10-09T04:16:46.778588+0000","last_became_peered":"2023-10-09T04:16:46.778588+0000","last_unstale":"2023-10-09T04:17:18.489474+0000","last_undegraded":"2023-10-09T04:17:18.489474+0000","last_fullsized":"2023-10-09T04:17:18.489474+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483385+0000","last_change":"2023-10-09T04:15:25.409247+0000","last_active":"2023-10-09T04:17:18.483385+0000","last_peered":"2023-10-09T04:17:18.483385+0000","last_clean":"2023-10-09T04:17:18.483385+0000","last_became_active":"2023-10-09T04:15:25.408950+0000","last_became_peered":"2023-10-09T04:15:25.408950+0000","last_unstale":"2023-10-09T04:17:18.483385+0000","last_undegraded":"2023-10-09T04:17:18.483385+0000","last_fullsized":"2023-10-09T04:17:18.483385+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:46.455335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":247,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.479744+0000","last_change":"2023-10-09T04:03:37.223252+0000","last_active":"2023-10-09T04:17:18.479744+0000","last_peered":"2023-10-09T04:17:18.479744+0000","last_clean":"2023-10-09T04:17:18.479744+0000","last_became_active":"2023-10-09T04:03:37.222791+0000","last_became_peered":"2023-10-09T04:03:37.222791+0000","last_unstale":"2023-10-09T04:17:18.479744+0000","last_undegraded":"2023-10-09T04:17:18.479744+0000","last_fullsized":"2023-10-09T04:17:18.479744+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:23.047971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":649,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489104+0000","last_change":"2023-10-09T04:16:46.778289+0000","last_active":"2023-10-09T04:17:18.489104+0000","last_peered":"2023-10-09T04:17:18.489104+0000","last_clean":"2023-10-09T04:17:18.489104+0000","last_became_active":"2023-10-09T04:16:46.777951+0000","last_became_peered":"2023-10-09T04:16:46.777951+0000","last_unstale":"2023-10-09T04:17:18.489104+0000","last_undegraded":"2023-10-09T04:17:18.489104+0000","last_fullsized":"2023-10-09T04:17:18.489104+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":202,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.475904+0000","last_change":"2023-10-09T04:16:24.625253+0000","last_active":"2023-10-09T04:17:18.475904+0000","last_peered":"2023-10-09T04:17:18.475904+0000","last_clean":"2023-10-09T04:17:18.475904+0000","last_became_active":"2023-10-09T04:16:24.617700+0000","last_became_peered":"2023-10-09T04:16:24.617700+0000","last_unstale":"2023-10-09T04:17:18.475904+0000","last_undegraded":"2023-10-09T04:17:18.475904+0000","last_fullsized":"2023-10-09T04:17:18.475904+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:29.757285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.756885+0000","last_change":"2023-10-09T04:16:24.474334+0000","last_active":"2023-10-09T04:17:05.756885+0000","last_peered":"2023-10-09T04:17:05.756885+0000","last_clean":"2023-10-09T04:17:05.756885+0000","last_became_active":"2023-10-09T04:16:24.474003+0000","last_became_peered":"2023-10-09T04:16:24.474003+0000","last_unstale":"2023-10-09T04:17:05.756885+0000","last_undegraded":"2023-10-09T04:17:05.756885+0000","last_fullsized":"2023-10-09T04:17:05.756885+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:38.410877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":649,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489723+0000","last_change":"2023-10-09T04:16:46.780829+0000","last_active":"2023-10-09T04:17:18.489723+0000","last_peered":"2023-10-09T04:17:18.489723+0000","last_clean":"2023-10-09T04:17:18.489723+0000","last_became_active":"2023-10-09T04:16:46.780531+0000","last_became_peered":"2023-10-09T04:16:46.780531+0000","last_unstale":"2023-10-09T04:17:18.489723+0000","last_undegraded":"2023-10-09T04:17:18.489723+0000","last_fullsized":"2023-10-09T04:17:18.489723+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"290'10582","reported_seq":16544,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:30.713901+0000","last_change":"2023-10-09T04:16:59.486729+0000","last_active":"2023-10-09T04:17:30.713901+0000","last_peered":"2023-10-09T04:17:30.713901+0000","last_clean":"2023-10-09T04:17:30.713901+0000","last_became_active":"2023-10-09T04:16:59.466744+0000","last_became_peered":"2023-10-09T04:16:59.466744+0000","last_unstale":"2023-10-09T04:17:30.713901+0000","last_undegraded":"2023-10-09T04:17:30.713901+0000","last_fullsized":"2023-10-09T04:17:30.713901+0000","mapping_epoch":286,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:35.727516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46137344,"num_objects":704,"num_object_clones":0,"num_object_copies":2112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10582,"num_write_kb":361152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2936,"num_bytes_recovered":180289536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":278,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489864+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:17:18.489864+0000","last_peered":"2023-10-09T04:17:18.489864+0000","last_clean":"2023-10-09T04:17:18.489864+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:17:18.489864+0000","last_undegraded":"2023-10-09T04:17:18.489864+0000","last_fullsized":"2023-10-09T04:17:18.489864+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":231,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.485448+0000","last_change":"2023-10-09T03:54:13.390065+0000","last_active":"2023-10-09T04:17:18.485448+0000","last_peered":"2023-10-09T04:17:18.485448+0000","last_clean":"2023-10-09T04:17:18.485448+0000","last_became_active":"2023-10-09T03:54:13.389682+0000","last_became_peered":"2023-10-09T03:54:13.389682+0000","last_unstale":"2023-10-09T04:17:18.485448+0000","last_undegraded":"2023-10-09T04:17:18.485448+0000","last_fullsized":"2023-10-09T04:17:18.485448+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:59:19.579968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":653,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489784+0000","last_change":"2023-10-09T04:16:46.798945+0000","last_active":"2023-10-09T04:17:18.489784+0000","last_peered":"2023-10-09T04:17:18.489784+0000","last_clean":"2023-10-09T04:17:18.489784+0000","last_became_active":"2023-10-09T04:16:46.798622+0000","last_became_peered":"2023-10-09T04:16:46.798622+0000","last_unstale":"2023-10-09T04:17:18.489784+0000","last_undegraded":"2023-10-09T04:17:18.489784+0000","last_fullsized":"2023-10-09T04:17:18.489784+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":16,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"290'10602","reported_seq":16470,"reported_epoch":290,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:17:30.425351+0000","last_change":"2023-10-09T04:16:57.541338+0000","last_active":"2023-10-09T04:17:30.425351+0000","last_peered":"2023-10-09T04:17:30.425351+0000","last_clean":"2023-10-09T04:15:24.138637+0000","last_became_active":"2023-10-09T04:15:26.489098+0000","last_became_peered":"2023-10-09T04:15:26.489098+0000","last_unstale":"2023-10-09T04:17:30.425351+0000","last_undegraded":"2023-10-09T04:17:30.425351+0000","last_fullsized":"2023-10-09T04:17:30.425351+0000","mapping_epoch":266,"log_start":"257'5018","ondisk_log_start":"257'5018","created":241,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:37.061888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47448064,"num_objects":724,"num_object_clones":0,"num_object_copies":2172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":715,"num_objects_unfound":0,"num_objects_dirty":724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10602,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2887,"num_bytes_recovered":189202432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,7,0],"avail_no_missing":["6","0","7"],"object_location_counts":[{"shards":"0,6,7","objects":724}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":282,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.490126+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:17:18.490126+0000","last_peered":"2023-10-09T04:17:18.490126+0000","last_clean":"2023-10-09T04:17:18.490126+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:17:18.490126+0000","last_undegraded":"2023-10-09T04:17:18.490126+0000","last_fullsized":"2023-10-09T04:17:18.490126+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":44,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483455+0000","last_change":"2023-10-09T04:16:24.476123+0000","last_active":"2023-10-09T04:17:18.483455+0000","last_peered":"2023-10-09T04:17:18.483455+0000","last_clean":"2023-10-09T04:17:18.483455+0000","last_became_active":"2023-10-09T04:16:24.475818+0000","last_became_peered":"2023-10-09T04:16:24.475818+0000","last_unstale":"2023-10-09T04:17:18.483455+0000","last_undegraded":"2023-10-09T04:17:18.483455+0000","last_fullsized":"2023-10-09T04:17:18.483455+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:57.051534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":652,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489811+0000","last_change":"2023-10-09T04:16:46.779365+0000","last_active":"2023-10-09T04:17:18.489811+0000","last_peered":"2023-10-09T04:17:18.489811+0000","last_clean":"2023-10-09T04:17:18.489811+0000","last_became_active":"2023-10-09T04:16:46.779035+0000","last_became_peered":"2023-10-09T04:16:46.779035+0000","last_unstale":"2023-10-09T04:17:18.489811+0000","last_undegraded":"2023-10-09T04:17:18.489811+0000","last_fullsized":"2023-10-09T04:17:18.489811+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.6","version":"290'10669","reported_seq":10728,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:31.553304+0000","last_change":"2023-10-09T04:13:42.646173+0000","last_active":"2023-10-09T04:17:31.553304+0000","last_peered":"2023-10-09T04:17:31.553304+0000","last_clean":"2023-10-09T04:17:31.553304+0000","last_became_active":"2023-10-09T04:13:42.645934+0000","last_became_peered":"2023-10-09T04:13:42.645934+0000","last_unstale":"2023-10-09T04:17:31.553304+0000","last_undegraded":"2023-10-09T04:17:31.553304+0000","last_fullsized":"2023-10-09T04:17:31.553304+0000","mapping_epoch":241,"log_start":"257'5145","ondisk_log_start":"257'5145","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.323104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44630016,"num_objects":681,"num_object_clones":0,"num_object_copies":2043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10669,"num_write_kb":363200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":233,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489822+0000","last_change":"2023-10-09T03:54:13.224285+0000","last_active":"2023-10-09T04:17:18.489822+0000","last_peered":"2023-10-09T04:17:18.489822+0000","last_clean":"2023-10-09T04:17:18.489822+0000","last_became_active":"2023-10-09T03:54:13.223979+0000","last_became_peered":"2023-10-09T03:54:13.223979+0000","last_unstale":"2023-10-09T04:17:18.489822+0000","last_undegraded":"2023-10-09T04:17:18.489822+0000","last_fullsized":"2023-10-09T04:17:18.489822+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:06.862546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":210,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.752065+0000","last_change":"2023-10-09T04:06:57.500298+0000","last_active":"2023-10-09T04:17:05.752065+0000","last_peered":"2023-10-09T04:17:05.752065+0000","last_clean":"2023-10-09T04:17:05.752065+0000","last_became_active":"2023-10-09T04:06:57.500017+0000","last_became_peered":"2023-10-09T04:06:57.500017+0000","last_unstale":"2023-10-09T04:17:05.752065+0000","last_undegraded":"2023-10-09T04:17:05.752065+0000","last_fullsized":"2023-10-09T04:17:05.752065+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:19.831877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":652,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489720+0000","last_change":"2023-10-09T04:16:46.787228+0000","last_active":"2023-10-09T04:17:18.489720+0000","last_peered":"2023-10-09T04:17:18.489720+0000","last_clean":"2023-10-09T04:17:18.489720+0000","last_became_active":"2023-10-09T04:16:46.786694+0000","last_became_peered":"2023-10-09T04:16:46.786694+0000","last_unstale":"2023-10-09T04:17:18.489720+0000","last_undegraded":"2023-10-09T04:17:18.489720+0000","last_fullsized":"2023-10-09T04:17:18.489720+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.7","version":"289'10634","reported_seq":10692,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:29.117891+0000","last_change":"2023-10-09T04:13:42.847572+0000","last_active":"2023-10-09T04:17:29.117891+0000","last_peered":"2023-10-09T04:17:29.117891+0000","last_clean":"2023-10-09T04:17:29.117891+0000","last_became_active":"2023-10-09T04:13:42.847257+0000","last_became_peered":"2023-10-09T04:13:42.847257+0000","last_unstale":"2023-10-09T04:17:29.117891+0000","last_undegraded":"2023-10-09T04:17:29.117891+0000","last_fullsized":"2023-10-09T04:17:29.117891+0000","mapping_epoch":241,"log_start":"248'2500","ondisk_log_start":"248'2500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8134,"log_dups_size":0,"ondisk_log_size":8134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:19.215045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49414144,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10634,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.485919+0000","last_change":"2023-10-09T04:03:37.221861+0000","last_active":"2023-10-09T04:17:18.485919+0000","last_peered":"2023-10-09T04:17:18.485919+0000","last_clean":"2023-10-09T04:17:18.485919+0000","last_became_active":"2023-10-09T04:03:37.221501+0000","last_became_peered":"2023-10-09T04:03:37.221501+0000","last_unstale":"2023-10-09T04:17:18.485919+0000","last_undegraded":"2023-10-09T04:17:18.485919+0000","last_fullsized":"2023-10-09T04:17:18.485919+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:53.195765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":265,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.478251+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:17:18.478251+0000","last_peered":"2023-10-09T04:17:18.478251+0000","last_clean":"2023-10-09T04:17:18.478251+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:17:18.478251+0000","last_undegraded":"2023-10-09T04:17:18.478251+0000","last_fullsized":"2023-10-09T04:17:18.478251+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":653,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488978+0000","last_change":"2023-10-09T04:17:01.099989+0000","last_active":"2023-10-09T04:17:18.488978+0000","last_peered":"2023-10-09T04:17:18.488978+0000","last_clean":"2023-10-09T04:17:18.488978+0000","last_became_active":"2023-10-09T04:15:13.426360+0000","last_became_peered":"2023-10-09T04:15:13.426360+0000","last_unstale":"2023-10-09T04:17:18.488978+0000","last_undegraded":"2023-10-09T04:17:18.488978+0000","last_fullsized":"2023-10-09T04:17:18.488978+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:01.099879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:01.099879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:03.454967+0000","scrub_duration":0.0081520749999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.0","version":"290'10581","reported_seq":10638,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:30.679360+0000","last_change":"2023-10-09T04:13:42.847429+0000","last_active":"2023-10-09T04:17:30.679360+0000","last_peered":"2023-10-09T04:17:30.679360+0000","last_clean":"2023-10-09T04:17:30.679360+0000","last_became_active":"2023-10-09T04:13:42.847130+0000","last_became_peered":"2023-10-09T04:13:42.847130+0000","last_unstale":"2023-10-09T04:17:30.679360+0000","last_undegraded":"2023-10-09T04:17:30.679360+0000","last_fullsized":"2023-10-09T04:17:30.679360+0000","mapping_epoch":241,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10081,"log_dups_size":0,"ondisk_log_size":10081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:29.587257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10581,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":222,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.478347+0000","last_change":"2023-10-09T04:15:25.538809+0000","last_active":"2023-10-09T04:17:18.478347+0000","last_peered":"2023-10-09T04:17:18.478347+0000","last_clean":"2023-10-09T04:17:18.478347+0000","last_became_active":"2023-10-09T04:15:25.419652+0000","last_became_peered":"2023-10-09T04:15:25.419652+0000","last_unstale":"2023-10-09T04:17:18.478347+0000","last_undegraded":"2023-10-09T04:17:18.478347+0000","last_fullsized":"2023-10-09T04:17:18.478347+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:02.875159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.484362+0000","last_change":"2023-10-09T04:15:25.427175+0000","last_active":"2023-10-09T04:17:18.484362+0000","last_peered":"2023-10-09T04:17:18.484362+0000","last_clean":"2023-10-09T04:17:18.484362+0000","last_became_active":"2023-10-09T04:15:25.426875+0000","last_became_peered":"2023-10-09T04:15:25.426875+0000","last_unstale":"2023-10-09T04:17:18.484362+0000","last_undegraded":"2023-10-09T04:17:18.484362+0000","last_fullsized":"2023-10-09T04:17:18.484362+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:46.697878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":653,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488914+0000","last_change":"2023-10-09T04:17:00.124487+0000","last_active":"2023-10-09T04:17:18.488914+0000","last_peered":"2023-10-09T04:17:18.488914+0000","last_clean":"2023-10-09T04:17:18.488914+0000","last_became_active":"2023-10-09T04:15:13.421001+0000","last_became_peered":"2023-10-09T04:15:13.421001+0000","last_unstale":"2023-10-09T04:17:18.488914+0000","last_undegraded":"2023-10-09T04:17:18.488914+0000","last_fullsized":"2023-10-09T04:17:18.488914+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:00.124399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:00.124399+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:25.787390+0000","scrub_duration":0.011147822999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.1","version":"289'10704","reported_seq":10762,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:29.079063+0000","last_change":"2023-10-09T04:13:42.633796+0000","last_active":"2023-10-09T04:17:29.079063+0000","last_peered":"2023-10-09T04:17:29.079063+0000","last_clean":"2023-10-09T04:17:29.079063+0000","last_became_active":"2023-10-09T04:13:42.633490+0000","last_became_peered":"2023-10-09T04:13:42.633490+0000","last_unstale":"2023-10-09T04:17:29.079063+0000","last_undegraded":"2023-10-09T04:17:29.079063+0000","last_fullsized":"2023-10-09T04:17:29.079063+0000","mapping_epoch":241,"log_start":"248'2500","ondisk_log_start":"248'2500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8204,"log_dups_size":0,"ondisk_log_size":8204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:06:46.137500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53346304,"num_objects":814,"num_object_clones":0,"num_object_copies":2442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10704,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.484023+0000","last_change":"2023-10-09T04:15:25.432206+0000","last_active":"2023-10-09T04:17:18.484023+0000","last_peered":"2023-10-09T04:17:18.484023+0000","last_clean":"2023-10-09T04:17:18.484023+0000","last_became_active":"2023-10-09T04:15:25.430422+0000","last_became_peered":"2023-10-09T04:15:25.430422+0000","last_unstale":"2023-10-09T04:17:18.484023+0000","last_undegraded":"2023-10-09T04:17:18.484023+0000","last_fullsized":"2023-10-09T04:17:18.484023+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:04.755818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483477+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:17:18.483477+0000","last_peered":"2023-10-09T04:17:18.483477+0000","last_clean":"2023-10-09T04:17:18.483477+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:17:18.483477+0000","last_undegraded":"2023-10-09T04:17:18.483477+0000","last_fullsized":"2023-10-09T04:17:18.483477+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":652,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489216+0000","last_change":"2023-10-09T04:16:46.778683+0000","last_active":"2023-10-09T04:17:18.489216+0000","last_peered":"2023-10-09T04:17:18.489216+0000","last_clean":"2023-10-09T04:17:18.489216+0000","last_became_active":"2023-10-09T04:16:46.778327+0000","last_became_peered":"2023-10-09T04:16:46.778327+0000","last_unstale":"2023-10-09T04:17:18.489216+0000","last_undegraded":"2023-10-09T04:17:18.489216+0000","last_fullsized":"2023-10-09T04:17:18.489216+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":16,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.2","version":"290'10882","reported_seq":16784,"reported_epoch":290,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:17:31.484673+0000","last_change":"2023-10-09T04:15:26.506798+0000","last_active":"2023-10-09T04:17:31.484673+0000","last_peered":"2023-10-09T04:17:31.484673+0000","last_clean":"2023-10-09T04:15:24.385046+0000","last_became_active":"2023-10-09T04:15:26.497446+0000","last_became_peered":"2023-10-09T04:15:26.497446+0000","last_unstale":"2023-10-09T04:17:31.484673+0000","last_undegraded":"2023-10-09T04:17:31.484673+0000","last_fullsized":"2023-10-09T04:17:31.484673+0000","mapping_epoch":266,"log_start":"258'5381","ondisk_log_start":"258'5381","created":241,"last_epoch_clean":263,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:01.043194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45350912,"num_objects":692,"num_object_clones":0,"num_object_copies":2076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":692,"num_objects_unfound":0,"num_objects_dirty":692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10882,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2907,"num_bytes_recovered":190513152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[0,5,3],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":692}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483340+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:17:18.483340+0000","last_peered":"2023-10-09T04:17:18.483340+0000","last_clean":"2023-10-09T04:17:18.483340+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:17:18.483340+0000","last_undegraded":"2023-10-09T04:17:18.483340+0000","last_fullsized":"2023-10-09T04:17:18.483340+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":44,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483839+0000","last_change":"2023-10-09T04:16:24.617626+0000","last_active":"2023-10-09T04:17:18.483839+0000","last_peered":"2023-10-09T04:17:18.483839+0000","last_clean":"2023-10-09T04:17:18.483839+0000","last_became_active":"2023-10-09T04:16:24.617298+0000","last_became_peered":"2023-10-09T04:16:24.617298+0000","last_unstale":"2023-10-09T04:17:18.483839+0000","last_undegraded":"2023-10-09T04:17:18.483839+0000","last_fullsized":"2023-10-09T04:17:18.483839+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:28.718470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":650,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489213+0000","last_change":"2023-10-09T04:17:02.097310+0000","last_active":"2023-10-09T04:17:18.489213+0000","last_peered":"2023-10-09T04:17:18.489213+0000","last_clean":"2023-10-09T04:17:18.489213+0000","last_became_active":"2023-10-09T04:15:13.428029+0000","last_became_peered":"2023-10-09T04:15:13.428029+0000","last_unstale":"2023-10-09T04:17:18.489213+0000","last_undegraded":"2023-10-09T04:17:18.489213+0000","last_fullsized":"2023-10-09T04:17:18.489213+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:02.097241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:02.097241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:47.238973+0000","scrub_duration":0.0088630900000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.3","version":"290'10518","reported_seq":10587,"reported_epoch":290,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:17:30.650774+0000","last_change":"2023-10-09T04:16:25.591418+0000","last_active":"2023-10-09T04:17:30.650774+0000","last_peered":"2023-10-09T04:17:30.650774+0000","last_clean":"2023-10-09T04:16:23.412068+0000","last_became_active":"2023-10-09T04:16:25.564970+0000","last_became_peered":"2023-10-09T04:16:25.564970+0000","last_unstale":"2023-10-09T04:17:30.650774+0000","last_undegraded":"2023-10-09T04:17:30.650774+0000","last_fullsized":"2023-10-09T04:17:30.650774+0000","mapping_epoch":277,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:44.646528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49414144,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":754,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10518,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":754}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.751882+0000","last_change":"2023-10-09T04:16:24.471755+0000","last_active":"2023-10-09T04:17:05.751882+0000","last_peered":"2023-10-09T04:17:05.751882+0000","last_clean":"2023-10-09T04:17:05.751882+0000","last_became_active":"2023-10-09T04:16:24.471509+0000","last_became_peered":"2023-10-09T04:16:24.471509+0000","last_unstale":"2023-10-09T04:17:05.751882+0000","last_undegraded":"2023-10-09T04:17:05.751882+0000","last_fullsized":"2023-10-09T04:17:05.751882+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:57.228210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":261,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.756929+0000","last_change":"2023-10-09T04:15:25.418196+0000","last_active":"2023-10-09T04:17:05.756929+0000","last_peered":"2023-10-09T04:17:05.756929+0000","last_clean":"2023-10-09T04:17:05.756929+0000","last_became_active":"2023-10-09T04:15:25.417900+0000","last_became_peered":"2023-10-09T04:15:25.417900+0000","last_unstale":"2023-10-09T04:17:05.756929+0000","last_undegraded":"2023-10-09T04:17:05.756929+0000","last_fullsized":"2023-10-09T04:17:05.756929+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:19.256359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":663,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.490011+0000","last_change":"2023-10-09T04:17:14.378761+0000","last_active":"2023-10-09T04:17:18.490011+0000","last_peered":"2023-10-09T04:17:18.490011+0000","last_clean":"2023-10-09T04:17:18.490011+0000","last_became_active":"2023-10-09T04:16:46.771545+0000","last_became_peered":"2023-10-09T04:16:46.771545+0000","last_unstale":"2023-10-09T04:17:18.490011+0000","last_undegraded":"2023-10-09T04:17:18.490011+0000","last_fullsized":"2023-10-09T04:17:18.490011+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:58.869092+0000","scrub_duration":0.0049573430000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.e","version":"290'10746","reported_seq":20418,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:31.612583+0000","last_change":"2023-10-09T04:16:21.813363+0000","last_active":"2023-10-09T04:17:31.612583+0000","last_peered":"2023-10-09T04:17:31.612583+0000","last_clean":"2023-10-09T04:17:31.612583+0000","last_became_active":"2023-10-09T04:16:21.793534+0000","last_became_peered":"2023-10-09T04:16:21.793534+0000","last_unstale":"2023-10-09T04:17:31.612583+0000","last_undegraded":"2023-10-09T04:17:31.612583+0000","last_fullsized":"2023-10-09T04:17:31.612583+0000","mapping_epoch":274,"log_start":"257'5236","ondisk_log_start":"257'5236","created":241,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:21:38.188955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10746,"num_write_kb":365120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4794,"num_bytes_recovered":302186496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":267,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:08.933446+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:17:08.933446+0000","last_peered":"2023-10-09T04:17:08.933446+0000","last_clean":"2023-10-09T04:17:08.933446+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:17:08.933446+0000","last_undegraded":"2023-10-09T04:17:08.933446+0000","last_fullsized":"2023-10-09T04:17:08.933446+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.487871+0000","last_change":"2023-10-09T04:03:37.220381+0000","last_active":"2023-10-09T04:17:18.487871+0000","last_peered":"2023-10-09T04:17:18.487871+0000","last_clean":"2023-10-09T04:17:18.487871+0000","last_became_active":"2023-10-09T04:03:37.220043+0000","last_became_peered":"2023-10-09T04:03:37.220043+0000","last_unstale":"2023-10-09T04:17:18.487871+0000","last_undegraded":"2023-10-09T04:17:18.487871+0000","last_fullsized":"2023-10-09T04:17:18.487871+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:36.508628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":240,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488022+0000","last_change":"2023-10-09T04:16:24.473165+0000","last_active":"2023-10-09T04:17:18.488022+0000","last_peered":"2023-10-09T04:17:18.488022+0000","last_clean":"2023-10-09T04:17:18.488022+0000","last_became_active":"2023-10-09T04:16:24.472923+0000","last_became_peered":"2023-10-09T04:16:24.472923+0000","last_unstale":"2023-10-09T04:17:18.488022+0000","last_undegraded":"2023-10-09T04:17:18.488022+0000","last_fullsized":"2023-10-09T04:17:18.488022+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:01.846352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":652,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489080+0000","last_change":"2023-10-09T04:16:46.777750+0000","last_active":"2023-10-09T04:17:18.489080+0000","last_peered":"2023-10-09T04:17:18.489080+0000","last_clean":"2023-10-09T04:17:18.489080+0000","last_became_active":"2023-10-09T04:16:46.777315+0000","last_became_peered":"2023-10-09T04:16:46.777315+0000","last_unstale":"2023-10-09T04:17:18.489080+0000","last_undegraded":"2023-10-09T04:17:18.489080+0000","last_fullsized":"2023-10-09T04:17:18.489080+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.8","version":"290'10601","reported_seq":10658,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:30.669282+0000","last_change":"2023-10-09T04:13:42.847477+0000","last_active":"2023-10-09T04:17:30.669282+0000","last_peered":"2023-10-09T04:17:30.669282+0000","last_clean":"2023-10-09T04:17:30.669282+0000","last_became_active":"2023-10-09T04:13:42.847180+0000","last_became_peered":"2023-10-09T04:13:42.847180+0000","last_unstale":"2023-10-09T04:17:30.669282+0000","last_undegraded":"2023-10-09T04:17:30.669282+0000","last_fullsized":"2023-10-09T04:17:30.669282+0000","mapping_epoch":241,"log_start":"247'600","ondisk_log_start":"247'600","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10001,"log_dups_size":0,"ondisk_log_size":10001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:23.775147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10601,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":253,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488226+0000","last_change":"2023-10-09T04:02:35.089316+0000","last_active":"2023-10-09T04:17:18.488226+0000","last_peered":"2023-10-09T04:17:18.488226+0000","last_clean":"2023-10-09T04:17:18.488226+0000","last_became_active":"2023-10-09T04:02:35.089008+0000","last_became_peered":"2023-10-09T04:02:35.089008+0000","last_unstale":"2023-10-09T04:17:18.488226+0000","last_undegraded":"2023-10-09T04:17:18.488226+0000","last_fullsized":"2023-10-09T04:17:18.488226+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:29.415844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483758+0000","last_change":"2023-10-09T04:15:25.435628+0000","last_active":"2023-10-09T04:17:18.483758+0000","last_peered":"2023-10-09T04:17:18.483758+0000","last_clean":"2023-10-09T04:17:18.483758+0000","last_became_active":"2023-10-09T04:15:25.435144+0000","last_became_peered":"2023-10-09T04:15:25.435144+0000","last_unstale":"2023-10-09T04:17:18.483758+0000","last_undegraded":"2023-10-09T04:17:18.483758+0000","last_fullsized":"2023-10-09T04:17:18.483758+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:50.156419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":277,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:08.933425+0000","last_change":"2023-10-09T04:10:38.332661+0000","last_active":"2023-10-09T04:17:08.933425+0000","last_peered":"2023-10-09T04:17:08.933425+0000","last_clean":"2023-10-09T04:17:08.933425+0000","last_became_active":"2023-10-09T04:10:38.332165+0000","last_became_peered":"2023-10-09T04:10:38.332165+0000","last_unstale":"2023-10-09T04:17:08.933425+0000","last_undegraded":"2023-10-09T04:17:08.933425+0000","last_fullsized":"2023-10-09T04:17:08.933425+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:39:35.989382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":650,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489273+0000","last_change":"2023-10-09T04:16:46.777572+0000","last_active":"2023-10-09T04:17:18.489273+0000","last_peered":"2023-10-09T04:17:18.489273+0000","last_clean":"2023-10-09T04:17:18.489273+0000","last_became_active":"2023-10-09T04:16:46.777232+0000","last_became_peered":"2023-10-09T04:16:46.777232+0000","last_unstale":"2023-10-09T04:17:18.489273+0000","last_undegraded":"2023-10-09T04:17:18.489273+0000","last_fullsized":"2023-10-09T04:17:18.489273+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.b","version":"289'10545","reported_seq":13137,"reported_epoch":289,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:17:29.105892+0000","last_change":"2023-10-09T04:16:57.541026+0000","last_active":"2023-10-09T04:17:29.105892+0000","last_peered":"2023-10-09T04:17:29.105892+0000","last_clean":"2023-10-09T04:14:14.593157+0000","last_became_active":"2023-10-09T04:16:24.620512+0000","last_became_peered":"2023-10-09T04:16:24.620512+0000","last_unstale":"2023-10-09T04:17:29.105892+0000","last_undegraded":"2023-10-09T04:16:24.612691+0000","last_fullsized":"2023-10-09T04:16:24.468624+0000","mapping_epoch":276,"log_start":"248'2400","ondisk_log_start":"248'2400","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:37.251260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":1819,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10545,"num_write_kb":362432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1253,"num_bytes_recovered":82051072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":773},{"shards":"1,3","objects":-1038}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":106,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.480128+0000","last_change":"2023-10-09T04:10:38.329030+0000","last_active":"2023-10-09T04:17:18.480128+0000","last_peered":"2023-10-09T04:17:18.480128+0000","last_clean":"2023-10-09T04:17:18.480128+0000","last_became_active":"2023-10-09T04:10:38.328683+0000","last_became_peered":"2023-10-09T04:10:38.328683+0000","last_unstale":"2023-10-09T04:17:18.480128+0000","last_undegraded":"2023-10-09T04:17:18.480128+0000","last_fullsized":"2023-10-09T04:17:18.480128+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:26.153740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":261,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.751723+0000","last_change":"2023-10-09T04:10:38.330079+0000","last_active":"2023-10-09T04:17:05.751723+0000","last_peered":"2023-10-09T04:17:05.751723+0000","last_clean":"2023-10-09T04:17:05.751723+0000","last_became_active":"2023-10-09T04:10:38.329775+0000","last_became_peered":"2023-10-09T04:10:38.329775+0000","last_unstale":"2023-10-09T04:17:05.751723+0000","last_undegraded":"2023-10-09T04:17:05.751723+0000","last_fullsized":"2023-10-09T04:17:05.751723+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:23.128658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.486132+0000","last_change":"2023-10-09T04:16:24.470074+0000","last_active":"2023-10-09T04:17:18.486132+0000","last_peered":"2023-10-09T04:17:18.486132+0000","last_clean":"2023-10-09T04:17:18.486132+0000","last_became_active":"2023-10-09T04:16:24.469767+0000","last_became_peered":"2023-10-09T04:16:24.469767+0000","last_unstale":"2023-10-09T04:17:18.486132+0000","last_undegraded":"2023-10-09T04:17:18.486132+0000","last_fullsized":"2023-10-09T04:17:18.486132+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.823815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"290'10560","reported_seq":10615,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:30.687320+0000","last_change":"2023-10-09T04:13:42.638373+0000","last_active":"2023-10-09T04:17:30.687320+0000","last_peered":"2023-10-09T04:17:30.687320+0000","last_clean":"2023-10-09T04:17:30.687320+0000","last_became_active":"2023-10-09T04:13:42.637990+0000","last_became_peered":"2023-10-09T04:13:42.637990+0000","last_unstale":"2023-10-09T04:17:30.687320+0000","last_undegraded":"2023-10-09T04:17:30.687320+0000","last_fullsized":"2023-10-09T04:17:30.687320+0000","mapping_epoch":241,"log_start":"247'500","ondisk_log_start":"247'500","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10060,"log_dups_size":0,"ondisk_log_size":10060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:37.637304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45350912,"num_objects":692,"num_object_clones":0,"num_object_copies":2076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10560,"num_write_kb":360064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":273,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.485968+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:17:18.485968+0000","last_peered":"2023-10-09T04:17:18.485968+0000","last_clean":"2023-10-09T04:17:18.485968+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:17:18.485968+0000","last_undegraded":"2023-10-09T04:17:18.485968+0000","last_fullsized":"2023-10-09T04:17:18.485968+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":664,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489089+0000","last_change":"2023-10-09T04:17:11.306229+0000","last_active":"2023-10-09T04:17:18.489089+0000","last_peered":"2023-10-09T04:17:18.489089+0000","last_clean":"2023-10-09T04:17:18.489089+0000","last_became_active":"2023-10-09T04:16:46.771308+0000","last_became_peered":"2023-10-09T04:16:46.771308+0000","last_unstale":"2023-10-09T04:17:18.489089+0000","last_undegraded":"2023-10-09T04:17:18.489089+0000","last_fullsized":"2023-10-09T04:17:18.489089+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:55.150777+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.488119+0000","last_change":"2023-10-09T04:06:45.474277+0000","last_active":"2023-10-09T04:17:18.488119+0000","last_peered":"2023-10-09T04:17:18.488119+0000","last_clean":"2023-10-09T04:17:18.488119+0000","last_became_active":"2023-10-09T04:06:45.473990+0000","last_became_peered":"2023-10-09T04:06:45.473990+0000","last_unstale":"2023-10-09T04:17:18.488119+0000","last_undegraded":"2023-10-09T04:17:18.488119+0000","last_fullsized":"2023-10-09T04:17:18.488119+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:46.721151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483555+0000","last_change":"2023-10-09T04:15:25.409271+0000","last_active":"2023-10-09T04:17:18.483555+0000","last_peered":"2023-10-09T04:17:18.483555+0000","last_clean":"2023-10-09T04:17:18.483555+0000","last_became_active":"2023-10-09T04:15:25.408986+0000","last_became_peered":"2023-10-09T04:15:25.408986+0000","last_unstale":"2023-10-09T04:17:18.483555+0000","last_undegraded":"2023-10-09T04:17:18.483555+0000","last_fullsized":"2023-10-09T04:17:18.483555+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:18.172890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":663,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489305+0000","last_change":"2023-10-09T04:17:05.198063+0000","last_active":"2023-10-09T04:17:18.489305+0000","last_peered":"2023-10-09T04:17:18.489305+0000","last_clean":"2023-10-09T04:17:18.489305+0000","last_became_active":"2023-10-09T04:16:46.761340+0000","last_became_peered":"2023-10-09T04:16:46.761340+0000","last_unstale":"2023-10-09T04:17:18.489305+0000","last_undegraded":"2023-10-09T04:17:18.489305+0000","last_fullsized":"2023-10-09T04:17:18.489305+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:05.197962+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:05.197962+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:26.889072+0000","scrub_duration":0.0029764380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"289'10140","reported_seq":10198,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:29.012436+0000","last_change":"2023-10-09T04:13:42.635883+0000","last_active":"2023-10-09T04:17:29.012436+0000","last_peered":"2023-10-09T04:17:29.012436+0000","last_clean":"2023-10-09T04:17:29.012436+0000","last_became_active":"2023-10-09T04:13:42.635571+0000","last_became_peered":"2023-10-09T04:13:42.635571+0000","last_unstale":"2023-10-09T04:17:29.012436+0000","last_undegraded":"2023-10-09T04:17:29.012436+0000","last_fullsized":"2023-10-09T04:17:29.012436+0000","mapping_epoch":241,"log_start":"247'2000","ondisk_log_start":"247'2000","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:34.753582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47972352,"num_objects":732,"num_object_clones":0,"num_object_copies":2196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10140,"num_write_kb":347904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":270,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489017+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:17:18.489017+0000","last_peered":"2023-10-09T04:17:18.489017+0000","last_clean":"2023-10-09T04:17:18.489017+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:17:18.489017+0000","last_undegraded":"2023-10-09T04:17:18.489017+0000","last_fullsized":"2023-10-09T04:17:18.489017+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":275,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.478395+0000","last_change":"2023-10-09T04:16:24.470732+0000","last_active":"2023-10-09T04:17:18.478395+0000","last_peered":"2023-10-09T04:17:18.478395+0000","last_clean":"2023-10-09T04:17:18.478395+0000","last_became_active":"2023-10-09T04:16:24.470433+0000","last_became_peered":"2023-10-09T04:16:24.470433+0000","last_unstale":"2023-10-09T04:17:18.478395+0000","last_undegraded":"2023-10-09T04:17:18.478395+0000","last_fullsized":"2023-10-09T04:17:18.478395+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:54.719857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":281,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.479515+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:17:18.479515+0000","last_peered":"2023-10-09T04:17:18.479515+0000","last_clean":"2023-10-09T04:17:18.479515+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:17:18.479515+0000","last_undegraded":"2023-10-09T04:17:18.479515+0000","last_fullsized":"2023-10-09T04:17:18.479515+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":663,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.490062+0000","last_change":"2023-10-09T04:17:07.265562+0000","last_active":"2023-10-09T04:17:18.490062+0000","last_peered":"2023-10-09T04:17:18.490062+0000","last_clean":"2023-10-09T04:17:18.490062+0000","last_became_active":"2023-10-09T04:16:46.786290+0000","last_became_peered":"2023-10-09T04:16:46.786290+0000","last_unstale":"2023-10-09T04:17:18.490062+0000","last_undegraded":"2023-10-09T04:17:18.490062+0000","last_fullsized":"2023-10-09T04:17:18.490062+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:07.265457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:07.265457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:58:48.475690+0000","scrub_duration":0.0080754069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.f","version":"290'10913","reported_seq":10971,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:30.281022+0000","last_change":"2023-10-09T04:13:42.847637+0000","last_active":"2023-10-09T04:17:30.281022+0000","last_peered":"2023-10-09T04:17:30.281022+0000","last_clean":"2023-10-09T04:17:30.281022+0000","last_became_active":"2023-10-09T04:13:42.847359+0000","last_became_peered":"2023-10-09T04:13:42.847359+0000","last_unstale":"2023-10-09T04:17:30.281022+0000","last_undegraded":"2023-10-09T04:17:30.281022+0000","last_fullsized":"2023-10-09T04:17:30.281022+0000","mapping_epoch":241,"log_start":"247'900","ondisk_log_start":"247'900","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10013,"log_dups_size":0,"ondisk_log_size":10013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:15:39.956849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45416448,"num_objects":693,"num_object_clones":0,"num_object_copies":2079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10913,"num_write_kb":371392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":273,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489880+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:17:18.489880+0000","last_peered":"2023-10-09T04:17:18.489880+0000","last_clean":"2023-10-09T04:17:18.489880+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:17:18.489880+0000","last_undegraded":"2023-10-09T04:17:18.489880+0000","last_fullsized":"2023-10-09T04:17:18.489880+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":261,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:08.933462+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:17:08.933462+0000","last_peered":"2023-10-09T04:17:08.933462+0000","last_clean":"2023-10-09T04:17:08.933462+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:17:08.933462+0000","last_undegraded":"2023-10-09T04:17:08.933462+0000","last_fullsized":"2023-10-09T04:17:08.933462+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":233,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.478430+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:17:18.478430+0000","last_peered":"2023-10-09T04:17:18.478430+0000","last_clean":"2023-10-09T04:17:18.478430+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:17:18.478430+0000","last_undegraded":"2023-10-09T04:17:18.478430+0000","last_fullsized":"2023-10-09T04:17:18.478430+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":650,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489728+0000","last_change":"2023-10-09T04:16:46.771645+0000","last_active":"2023-10-09T04:17:18.489728+0000","last_peered":"2023-10-09T04:17:18.489728+0000","last_clean":"2023-10-09T04:17:18.489728+0000","last_became_active":"2023-10-09T04:16:46.771347+0000","last_became_peered":"2023-10-09T04:16:46.771347+0000","last_unstale":"2023-10-09T04:17:18.489728+0000","last_undegraded":"2023-10-09T04:17:18.489728+0000","last_fullsized":"2023-10-09T04:17:18.489728+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.c","version":"290'10583","reported_seq":10642,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:31.612916+0000","last_change":"2023-10-09T04:13:42.847596+0000","last_active":"2023-10-09T04:17:31.612916+0000","last_peered":"2023-10-09T04:17:31.612916+0000","last_clean":"2023-10-09T04:17:31.612916+0000","last_became_active":"2023-10-09T04:13:42.847298+0000","last_became_peered":"2023-10-09T04:13:42.847298+0000","last_unstale":"2023-10-09T04:17:31.612916+0000","last_undegraded":"2023-10-09T04:17:31.612916+0000","last_fullsized":"2023-10-09T04:17:31.612916+0000","mapping_epoch":241,"log_start":"257'5055","ondisk_log_start":"257'5055","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":5528,"log_dups_size":0,"ondisk_log_size":5528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:42.275480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44105728,"num_objects":673,"num_object_clones":0,"num_object_copies":2019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10583,"num_write_kb":360192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":194,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489946+0000","last_change":"2023-10-09T04:06:45.471461+0000","last_active":"2023-10-09T04:17:18.489946+0000","last_peered":"2023-10-09T04:17:18.489946+0000","last_clean":"2023-10-09T04:17:18.489946+0000","last_became_active":"2023-10-09T04:06:45.470922+0000","last_became_peered":"2023-10-09T04:06:45.470922+0000","last_unstale":"2023-10-09T04:17:18.489946+0000","last_undegraded":"2023-10-09T04:17:18.489946+0000","last_fullsized":"2023-10-09T04:17:18.489946+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:03.625928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":245,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:17:05.752652+0000","last_change":"2023-10-09T04:15:25.414014+0000","last_active":"2023-10-09T04:17:05.752652+0000","last_peered":"2023-10-09T04:17:05.752652+0000","last_clean":"2023-10-09T04:17:05.752652+0000","last_became_active":"2023-10-09T04:15:25.413619+0000","last_became_peered":"2023-10-09T04:15:25.413619+0000","last_unstale":"2023-10-09T04:17:05.752652+0000","last_undegraded":"2023-10-09T04:17:05.752652+0000","last_fullsized":"2023-10-09T04:17:05.752652+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:13.741581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":230,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489164+0000","last_change":"2023-10-09T04:16:24.469186+0000","last_active":"2023-10-09T04:17:18.489164+0000","last_peered":"2023-10-09T04:17:18.489164+0000","last_clean":"2023-10-09T04:17:18.489164+0000","last_became_active":"2023-10-09T04:16:24.468836+0000","last_became_peered":"2023-10-09T04:16:24.468836+0000","last_unstale":"2023-10-09T04:17:18.489164+0000","last_undegraded":"2023-10-09T04:17:18.489164+0000","last_fullsized":"2023-10-09T04:17:18.489164+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:42.902638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":653,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.489891+0000","last_change":"2023-10-09T04:16:46.782032+0000","last_active":"2023-10-09T04:17:18.489891+0000","last_peered":"2023-10-09T04:17:18.489891+0000","last_clean":"2023-10-09T04:17:18.489891+0000","last_became_active":"2023-10-09T04:16:46.778146+0000","last_became_peered":"2023-10-09T04:16:46.778146+0000","last_unstale":"2023-10-09T04:17:18.489891+0000","last_undegraded":"2023-10-09T04:17:18.489891+0000","last_fullsized":"2023-10-09T04:17:18.489891+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"290'10437","reported_seq":10495,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:17:30.761527+0000","last_change":"2023-10-09T04:13:42.646443+0000","last_active":"2023-10-09T04:17:30.761527+0000","last_peered":"2023-10-09T04:17:30.761527+0000","last_clean":"2023-10-09T04:17:30.761527+0000","last_became_active":"2023-10-09T04:13:42.646138+0000","last_became_peered":"2023-10-09T04:13:42.646138+0000","last_unstale":"2023-10-09T04:17:30.761527+0000","last_undegraded":"2023-10-09T04:17:30.761527+0000","last_fullsized":"2023-10-09T04:17:30.761527+0000","mapping_epoch":241,"log_start":"247'400","ondisk_log_start":"247'400","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:13:41.612996+0000","last_clean_scrub_stamp":"2023-10-09T04:13:41.612996+0000","objects_scrubbed":0,"log_size":10037,"log_dups_size":0,"ondisk_log_size":10037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:56.607273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44433432,"num_objects":679,"num_object_clones":0,"num_object_copies":2037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":679,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10437,"num_write_kb":355649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":111,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.479981+0000","last_change":"2023-10-09T04:13:46.100979+0000","last_active":"2023-10-09T04:17:18.479981+0000","last_peered":"2023-10-09T04:17:18.479981+0000","last_clean":"2023-10-09T04:17:18.479981+0000","last_became_active":"2023-10-09T04:13:46.100727+0000","last_became_peered":"2023-10-09T04:13:46.100727+0000","last_unstale":"2023-10-09T04:17:18.479981+0000","last_undegraded":"2023-10-09T04:17:18.479981+0000","last_fullsized":"2023-10-09T04:17:18.479981+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:11.755013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":281,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.479599+0000","last_change":"2023-10-09T04:16:20.109259+0000","last_active":"2023-10-09T04:17:18.479599+0000","last_peered":"2023-10-09T04:17:18.479599+0000","last_clean":"2023-10-09T04:17:18.479599+0000","last_became_active":"2023-10-09T04:15:25.496763+0000","last_became_peered":"2023-10-09T04:15:25.496763+0000","last_unstale":"2023-10-09T04:17:18.479599+0000","last_undegraded":"2023-10-09T04:17:18.479599+0000","last_fullsized":"2023-10-09T04:17:18.479599+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:15.807543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483421+0000","last_change":"2023-10-09T04:15:25.406909+0000","last_active":"2023-10-09T04:17:18.483421+0000","last_peered":"2023-10-09T04:17:18.483421+0000","last_clean":"2023-10-09T04:17:18.483421+0000","last_became_active":"2023-10-09T04:15:25.406617+0000","last_became_peered":"2023-10-09T04:15:25.406617+0000","last_unstale":"2023-10-09T04:17:18.483421+0000","last_undegraded":"2023-10-09T04:17:18.483421+0000","last_fullsized":"2023-10-09T04:17:18.483421+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:34.910221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":254,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.478622+0000","last_change":"2023-10-09T04:15:25.414363+0000","last_active":"2023-10-09T04:17:18.478622+0000","last_peered":"2023-10-09T04:17:18.478622+0000","last_clean":"2023-10-09T04:17:18.478622+0000","last_became_active":"2023-10-09T04:15:25.413987+0000","last_became_peered":"2023-10-09T04:15:25.413987+0000","last_unstale":"2023-10-09T04:17:18.478622+0000","last_undegraded":"2023-10-09T04:17:18.478622+0000","last_fullsized":"2023-10-09T04:17:18.478622+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:45.394691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":105,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.484117+0000","last_change":"2023-10-09T04:10:38.333635+0000","last_active":"2023-10-09T04:17:18.484117+0000","last_peered":"2023-10-09T04:17:18.484117+0000","last_clean":"2023-10-09T04:17:18.484117+0000","last_became_active":"2023-10-09T04:10:38.333314+0000","last_became_peered":"2023-10-09T04:10:38.333314+0000","last_unstale":"2023-10-09T04:17:18.484117+0000","last_undegraded":"2023-10-09T04:17:18.484117+0000","last_fullsized":"2023-10-09T04:17:18.484117+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:48.354417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":279,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.485996+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:17:18.485996+0000","last_peered":"2023-10-09T04:17:18.485996+0000","last_clean":"2023-10-09T04:17:18.485996+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:17:18.485996+0000","last_undegraded":"2023-10-09T04:17:18.485996+0000","last_fullsized":"2023-10-09T04:17:18.485996+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483893+0000","last_change":"2023-10-09T04:15:25.476221+0000","last_active":"2023-10-09T04:17:18.483893+0000","last_peered":"2023-10-09T04:17:18.483893+0000","last_clean":"2023-10-09T04:17:18.483893+0000","last_became_active":"2023-10-09T04:15:25.475459+0000","last_became_peered":"2023-10-09T04:15:25.475459+0000","last_unstale":"2023-10-09T04:17:18.483893+0000","last_undegraded":"2023-10-09T04:17:18.483893+0000","last_fullsized":"2023-10-09T04:17:18.483893+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:34.779580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":262,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.487888+0000","last_change":"2023-10-09T04:15:25.525420+0000","last_active":"2023-10-09T04:17:18.487888+0000","last_peered":"2023-10-09T04:17:18.487888+0000","last_clean":"2023-10-09T04:17:18.487888+0000","last_became_active":"2023-10-09T04:15:25.406419+0000","last_became_peered":"2023-10-09T04:15:25.406419+0000","last_unstale":"2023-10-09T04:17:18.487888+0000","last_undegraded":"2023-10-09T04:17:18.487888+0000","last_fullsized":"2023-10-09T04:17:18.487888+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:10.680233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483954+0000","last_change":"2023-10-09T04:15:25.435313+0000","last_active":"2023-10-09T04:17:18.483954+0000","last_peered":"2023-10-09T04:17:18.483954+0000","last_clean":"2023-10-09T04:17:18.483954+0000","last_became_active":"2023-10-09T04:15:25.434996+0000","last_became_peered":"2023-10-09T04:15:25.434996+0000","last_unstale":"2023-10-09T04:17:18.483954+0000","last_undegraded":"2023-10-09T04:17:18.483954+0000","last_fullsized":"2023-10-09T04:17:18.483954+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:22.816036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":106,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.479354+0000","last_change":"2023-10-09T04:10:38.369547+0000","last_active":"2023-10-09T04:17:18.479354+0000","last_peered":"2023-10-09T04:17:18.479354+0000","last_clean":"2023-10-09T04:17:18.479354+0000","last_became_active":"2023-10-09T04:10:38.369170+0000","last_became_peered":"2023-10-09T04:10:38.369170+0000","last_unstale":"2023-10-09T04:17:18.479354+0000","last_undegraded":"2023-10-09T04:17:18.479354+0000","last_fullsized":"2023-10-09T04:17:18.479354+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:36.602594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.480203+0000","last_change":"2023-10-09T04:10:38.329452+0000","last_active":"2023-10-09T04:17:18.480203+0000","last_peered":"2023-10-09T04:17:18.480203+0000","last_clean":"2023-10-09T04:17:18.480203+0000","last_became_active":"2023-10-09T04:10:38.328692+0000","last_became_peered":"2023-10-09T04:10:38.328692+0000","last_unstale":"2023-10-09T04:17:18.480203+0000","last_undegraded":"2023-10-09T04:17:18.480203+0000","last_fullsized":"2023-10-09T04:17:18.480203+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:03.644308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":282,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.480549+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:17:18.480549+0000","last_peered":"2023-10-09T04:17:18.480549+0000","last_clean":"2023-10-09T04:17:18.480549+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:17:18.480549+0000","last_undegraded":"2023-10-09T04:17:18.480549+0000","last_fullsized":"2023-10-09T04:17:18.480549+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:17:18.483188+0000","last_change":"2023-10-09T04:15:25.423355+0000","last_active":"2023-10-09T04:17:18.483188+0000","last_peered":"2023-10-09T04:17:18.483188+0000","last_clean":"2023-10-09T04:17:18.483188+0000","last_became_active":"2023-10-09T04:15:25.423070+0000","last_became_peered":"2023-10-09T04:15:25.423070+0000","last_unstale":"2023-10-09T04:17:18.483188+0000","last_undegraded":"2023-10-09T04:17:18.483188+0000","last_fullsized":"2023-10-09T04:17:18.483188+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:56.629007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6134,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":751435800,"num_objects":11467,"num_object_clones":0,"num_object_copies":34401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":3980,"num_objects_unfound":0,"num_objects_dirty":11467,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169697,"num_write_kb":5797185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14777,"num_bytes_recovered":944242688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2831953920,"data_stored":2831941704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130562,"ondisk_log_size":130562,"up":48,"acting":47,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":593653,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":284,"num_read_kb":518,"num_write":136,"num_write_kb":1357,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":992895,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201580,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":693156,"kb_used_data":383580,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93051996,"statfs":{"total":95995035648,"available":95285243904,"internally_reserved":0,"allocated":392785920,"data_stored":391912441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":3.5139999999999998,"15min":2.5830000000000002},"min":{"1min":0.33900000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":9.4220000000000006,"5min":88.144999999999996,"15min":126.752},"last":0.222},{"interface":"front","average":{"1min":2.8479999999999999,"5min":3.371,"15min":2.5569999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.332000000000001,"5min":89.284999999999997,"15min":128.88399999999999},"last":4.6260000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:17:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1890000000000001,"5min":4.7720000000000002,"15min":3.7789999999999999},"min":{"1min":0.38,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":6.774,"5min":103.324,"15min":245.28700000000001},"last":5.4580000000000002},{"interface":"front","average":{"1min":2.9079999999999999,"5min":4.5060000000000002,"15min":3.6469999999999998},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":10.913,"5min":104.248,"15min":244.626},"last":4.2910000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":2.1139999999999999,"15min":1.79},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.27400000000000002},"max":{"1min":4.6779999999999999,"5min":76.774000000000001,"15min":76.774000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.774,"5min":2.6240000000000001,"15min":2.3580000000000001},"min":{"1min":0.39700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":10.291,"5min":76.715999999999994,"15min":99.372},"last":4.1840000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:17:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":3.1480000000000001,"15min":2.415},"min":{"1min":0.37,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":6.9420000000000002,"5min":85.575000000000003,"15min":99.134},"last":1.3959999999999999},{"interface":"front","average":{"1min":2.4750000000000001,"5min":3.4620000000000002,"15min":2.589},"min":{"1min":0.40000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.9269999999999996,"5min":85.528000000000006,"15min":99.117999999999995},"last":4.2309999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.59899999999999998,"15min":0.73899999999999999},"min":{"1min":0.25800000000000001,"5min":0.252,"15min":0.23400000000000001},"max":{"1min":4.7110000000000003,"5min":4.7110000000000003,"15min":16.195},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.3399999999999999,"5min":1.0249999999999999,"15min":0.88400000000000001},"min":{"1min":0.34699999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.282,"5min":9.282,"15min":12.442},"last":4.2610000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.089,"15min":1.133},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27400000000000002},"max":{"1min":8.4130000000000003,"5min":120.066,"15min":120.066},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.9079999999999999,"15min":1.1399999999999999},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.4580000000000002,"5min":120.09699999999999,"15min":120.09699999999999},"last":0.46300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5390000000000001,"5min":1.121,"15min":0.876},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.452,"5min":8.452,"15min":8.452},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.357,"5min":0.76700000000000002,"15min":0.81000000000000005},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.0990000000000002,"5min":5.0990000000000002,"15min":5.9039999999999999},"last":0.62}]}]},{"osd":6,"up_from":47,"seq":201863463217,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":666640,"kb_used_data":352968,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93078512,"statfs":{"total":95995035648,"available":95312396288,"internally_reserved":0,"allocated":361439232,"data_stored":360585740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:28 2023","interfaces":[{"interface":"back","average":{"1min":6.6130000000000004,"5min":3.4940000000000002,"15min":3.2240000000000002},"min":{"1min":0.22500000000000001,"5min":0.193,"15min":0.193},"max":{"1min":87.927999999999997,"5min":120.55200000000001,"15min":120.55200000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":7.1130000000000004,"5min":3.7570000000000001,"15min":3.5870000000000002},"min":{"1min":0.23699999999999999,"5min":0.223,"15min":0.223},"max":{"1min":87.238,"5min":120.252,"15min":120.252},"last":6.056}]},{"osd":1,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":4.5110000000000001,"15min":3.0249999999999999},"min":{"1min":0.318,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.526999999999999,"5min":143.22499999999999,"15min":143.22499999999999},"last":0.436},{"interface":"front","average":{"1min":2.1909999999999998,"5min":4.0650000000000004,"15min":2.98},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.8129999999999997,"5min":143.18700000000001,"15min":143.18700000000001},"last":3.8069999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:17:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":3.2770000000000001,"15min":2.9409999999999998},"min":{"1min":0.40799999999999997,"5min":0.23899999999999999,"15min":0.218},"max":{"1min":6.6070000000000002,"5min":139.06899999999999,"15min":139.06899999999999},"last":1.4339999999999999},{"interface":"front","average":{"1min":2.4119999999999999,"5min":3.3109999999999999,"15min":2.9849999999999999},"min":{"1min":0.434,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":6.6390000000000002,"5min":139.173,"15min":139.173},"last":1.4530000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":2.464,"15min":2.6509999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":7.0099999999999998,"5min":51.082000000000001,"15min":57.509999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.9020000000000001,"5min":2.3500000000000001,"15min":2.617},"min":{"1min":0.39800000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.2669999999999995,"5min":51.031999999999996,"15min":57.453000000000003},"last":8.7620000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.93300000000000005,"15min":0.64800000000000002},"min":{"1min":0.28000000000000003,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":5.9829999999999997,"5min":5.9829999999999997,"15min":5.9829999999999997},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.579,"5min":0.97699999999999998,"15min":0.80900000000000005},"min":{"1min":0.307,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":7.7439999999999998,"5min":7.7439999999999998,"15min":7.7439999999999998},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:17:04 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.0629999999999999,"15min":1.0389999999999999},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.8040000000000003,"5min":7.6310000000000002,"15min":54.773000000000003},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.1299999999999999,"15min":1.147},"min":{"1min":0.35099999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.0289999999999999,"5min":7.0289999999999999,"15min":54.853999999999999},"last":0.501}]},{"osd":7,"last update":"Mon Oct 9 04:17:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.2,"15min":0.84399999999999997},"min":{"1min":0.377,"5min":0.34000000000000002,"15min":0.313},"max":{"1min":15.917,"5min":15.917,"15min":15.917},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.897,"5min":1.321,"15min":0.95099999999999996},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28699999999999998},"max":{"1min":16.914999999999999,"5min":16.914999999999999,"15min":16.914999999999999},"last":0.64300000000000002}]}]},{"osd":1,"up_from":89,"seq":382252089525,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":749704,"kb_used_data":363136,"kb_used_omap":9,"kb_used_meta":386486,"kb_avail":92995448,"statfs":{"total":95995035648,"available":95227338752,"internally_reserved":0,"allocated":371851264,"data_stored":370644837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":395762665},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":4.5979999999999999,"15min":3.2519999999999998},"min":{"1min":0.217,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":24.111999999999998,"5min":68.048000000000002,"15min":85.977999999999994},"last":2.9630000000000001},{"interface":"front","average":{"1min":4.4989999999999997,"5min":4.2240000000000002,"15min":3.629},"min":{"1min":0.26600000000000001,"5min":0.182,"15min":0.182},"max":{"1min":21.323,"5min":58.027000000000001,"15min":99.831999999999994},"last":12.436}]},{"osd":2,"last update":"Mon Oct 9 04:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.3359999999999999,"15min":2.2010000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.191},"max":{"1min":7.6349999999999998,"5min":25.577000000000002,"15min":121.54600000000001},"last":2.9169999999999998},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.8799999999999999,"15min":2.54},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":13.595000000000001,"5min":15.298999999999999,"15min":99.899000000000001},"last":12.340999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":4.0289999999999999,"15min":3.3479999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.6600000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":2.984},{"interface":"front","average":{"1min":3.7669999999999999,"5min":4.9669999999999996,"15min":4.1589999999999998},"min":{"1min":0.28799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":21.859000000000002,"5min":68.563999999999993,"15min":138.49799999999999},"last":12.821}]},{"osd":4,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":2.3359999999999999,"15min":2.0459999999999998},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.254},"max":{"1min":10.09,"5min":11.973000000000001,"15min":99.989999999999995},"last":3.145},{"interface":"front","average":{"1min":2.0019999999999998,"5min":2.4990000000000001,"15min":1.9770000000000001},"min":{"1min":0.32600000000000001,"5min":0.318,"15min":0.25900000000000001},"max":{"1min":9.2929999999999993,"5min":25.957999999999998,"15min":100.249},"last":2.7549999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":2.4350000000000001,"15min":2.0510000000000002},"min":{"1min":0.35799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":14.314,"5min":14.314,"15min":99.933999999999997},"last":3.1099999999999999},{"interface":"front","average":{"1min":2.427,"5min":2.7269999999999999,"15min":2.1619999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.307},"max":{"1min":9.3350000000000009,"5min":25.890999999999998,"15min":100.34099999999999},"last":2.7839999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":2.7229999999999999,"15min":2.258},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30299999999999999},"max":{"1min":9.3640000000000008,"5min":25.931000000000001,"15min":100.309},"last":2.8279999999999998},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.8959999999999999,"15min":1.9019999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":3.9079999999999999,"5min":13.446,"15min":100.143},"last":2.9399999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.585,"5min":2.7650000000000001,"15min":2.242},"min":{"1min":0.434,"5min":0.374,"15min":0.35799999999999998},"max":{"1min":9.391,"5min":25.984999999999999,"15min":100.214},"last":2.8050000000000002},{"interface":"front","average":{"1min":2.0470000000000002,"5min":2.2269999999999999,"15min":2.0289999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":11.023,"5min":13.471,"15min":100.113},"last":2.8820000000000001}]}]},{"osd":0,"up_from":84,"seq":360777253052,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":742296,"kb_used_data":323280,"kb_used_omap":28,"kb_used_meta":418531,"kb_avail":93002856,"statfs":{"total":95995035648,"available":95234924544,"internally_reserved":0,"allocated":331038720,"data_stored":329763905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":428576592},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:16:56 2023","interfaces":[{"interface":"back","average":{"1min":10.387,"5min":5.298,"15min":3.6240000000000001},"min":{"1min":0.217,"5min":0.187,"15min":0.17000000000000001},"max":{"1min":120.511,"5min":120.511,"15min":120.511},"last":0.23799999999999999},{"interface":"front","average":{"1min":9.8219999999999992,"5min":5.8760000000000003,"15min":4.0460000000000003},"min":{"1min":0.30299999999999999,"5min":0.25,"15min":0.214},"max":{"1min":120.934,"5min":120.934,"15min":120.934},"last":0.25700000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":2.786,"15min":2.7320000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":4.944,"5min":13.108000000000001,"15min":118.54000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":4.7089999999999996,"5min":3.3570000000000002,"15min":3.0680000000000001},"min":{"1min":0.23100000000000001,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":14.304,"5min":14.304,"15min":118.727},"last":0.34399999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:16:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":4.4909999999999997,"15min":4.9669999999999996},"min":{"1min":0.219,"5min":0.219,"15min":0.20999999999999999},"max":{"1min":9.9529999999999994,"5min":99.709000000000003,"15min":156.488},"last":1.585},{"interface":"front","average":{"1min":2.5009999999999999,"5min":4.516,"15min":5.0469999999999997},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.22500000000000001},"max":{"1min":18.823,"5min":99.560000000000002,"15min":156.50700000000001},"last":0.28899999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.8700000000000001,"15min":2.0529999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":5.6550000000000002,"5min":9.7270000000000003,"15min":118.633},"last":0.35999999999999999},{"interface":"front","average":{"1min":4.4039999999999999,"5min":2.375,"15min":2.4860000000000002},"min":{"1min":0.378,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.201000000000001,"5min":13.201000000000001,"15min":118.913},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":5.4029999999999996,"5min":4.5289999999999999,"15min":3.718},"min":{"1min":0.42599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":13.246,"5min":156.96100000000001,"15min":182.68899999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.377,"5min":3.5009999999999999,"15min":3.258},"min":{"1min":0.31,"5min":0.28399999999999997,"15min":0.28100000000000003},"max":{"1min":5.8029999999999999,"5min":156.97300000000001,"15min":182.66300000000001},"last":1.498}]},{"osd":6,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.5289999999999999,"15min":2.5009999999999999},"min":{"1min":0.34300000000000003,"5min":0.33400000000000002,"15min":0.30599999999999999},"max":{"1min":4.7880000000000003,"5min":10.797000000000001,"15min":137.36099999999999},"last":1.532},{"interface":"front","average":{"1min":1.6299999999999999,"5min":2.0859999999999999,"15min":2.5470000000000002},"min":{"1min":0.39200000000000002,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":5.6849999999999996,"5min":12.496,"15min":136.88999999999999},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 04:16:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.611,"15min":3.157},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.32100000000000001},"max":{"1min":3.8010000000000002,"5min":10.77,"15min":185.024},"last":1.5600000000000001},{"interface":"front","average":{"1min":1.514,"5min":2.1680000000000001,"15min":2.8180000000000001},"min":{"1min":0.41599999999999998,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":5.7089999999999996,"5min":12.417,"15min":184.98599999999999},"last":0.47199999999999998}]}]},{"osd":2,"up_from":94,"seq":403726925999,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792796,"kb_used_data":324884,"kb_used_omap":81,"kb_used_meta":467822,"kb_avail":92952356,"statfs":{"total":95995035648,"available":95183212544,"internally_reserved":0,"allocated":332681216,"data_stored":331461836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":479049946},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:24 2023","interfaces":[{"interface":"back","average":{"1min":8.6549999999999994,"5min":5.782,"15min":3.2839999999999998},"min":{"1min":0.215,"5min":0.20499999999999999,"15min":0.17399999999999999},"max":{"1min":81.385999999999996,"5min":81.385999999999996,"15min":81.385999999999996},"last":4.8109999999999999},{"interface":"front","average":{"1min":8.3610000000000007,"5min":5.1459999999999999,"15min":3.323},"min":{"1min":0.33900000000000002,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":79.453999999999994,"5min":79.453999999999994,"15min":79.453999999999994},"last":4.766}]},{"osd":1,"last update":"Mon Oct 9 04:17:24 2023","interfaces":[{"interface":"back","average":{"1min":8.2870000000000008,"5min":3.964,"15min":3.1699999999999999},"min":{"1min":0.38900000000000001,"5min":0.191,"15min":0.191},"max":{"1min":75.067999999999998,"5min":75.067999999999998,"15min":75.067999999999998},"last":4.726},{"interface":"front","average":{"1min":10.215999999999999,"5min":4.157,"15min":3.4060000000000001},"min":{"1min":0.28000000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":80.168000000000006,"5min":80.168000000000006,"15min":80.168000000000006},"last":2.9449999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:16:53 2023","interfaces":[{"interface":"back","average":{"1min":3.0979999999999999,"5min":2.6789999999999998,"15min":2.1139999999999999},"min":{"1min":0.22700000000000001,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":8.6859999999999999,"5min":20.242999999999999,"15min":27.562000000000001},"last":8.9969999999999999},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.6869999999999998,"15min":2.3210000000000002},"min":{"1min":0.23899999999999999,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":15.333,"5min":17.076000000000001,"15min":46.286999999999999},"last":7.4630000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:17:24 2023","interfaces":[{"interface":"back","average":{"1min":3.5009999999999999,"5min":1.8340000000000001,"15min":1.857},"min":{"1min":0.313,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":19.225999999999999,"5min":19.225999999999999,"15min":46.075000000000003},"last":1.478},{"interface":"front","average":{"1min":2.6190000000000002,"5min":1.857,"15min":1.655},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":11.891999999999999,"5min":11.891999999999999,"15min":20.134},"last":4.5640000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:17:24 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":2.0619999999999998,"15min":1.946},"min":{"1min":0.38900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":15.31,"5min":15.31,"15min":46.265000000000001},"last":3.379},{"interface":"front","average":{"1min":3.274,"5min":1.883,"15min":2.0150000000000001},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.28100000000000003},"max":{"1min":19.274999999999999,"5min":19.274999999999999,"15min":46.106000000000002},"last":2.113}]},{"osd":6,"last update":"Mon Oct 9 04:17:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":2.0249999999999999,"15min":2.137},"min":{"1min":0.318,"5min":0.318,"15min":0.311},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":114.14},"last":4.5990000000000002},{"interface":"front","average":{"1min":3.0449999999999999,"5min":1.9350000000000001,"15min":2.274},"min":{"1min":0.35499999999999998,"5min":0.34599999999999997,"15min":0.33300000000000002},"max":{"1min":15.239000000000001,"5min":15.239000000000001,"15min":114.123},"last":3.4220000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:17:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.097,"15min":2.339},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34300000000000003},"max":{"1min":11.837999999999999,"5min":11.837999999999999,"15min":117.32599999999999},"last":4.6699999999999999},{"interface":"front","average":{"1min":3.157,"5min":2.0390000000000001,"15min":2.4249999999999998},"min":{"1min":0.42499999999999999,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":15.284000000000001,"5min":15.284000000000001,"15min":117.28700000000001},"last":3.4540000000000002}]}]},{"osd":3,"up_from":99,"seq":425201762473,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":966624,"kb_used_data":322520,"kb_used_omap":7,"kb_used_meta":643896,"kb_avail":92778528,"statfs":{"total":95995035648,"available":95005212672,"internally_reserved":0,"allocated":330260480,"data_stored":329037474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":659350120},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":6.2850000000000001,"5min":6.0449999999999999,"15min":5.0620000000000003},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":78.771000000000001,"5min":125.765,"15min":125.765},"last":7.585},{"interface":"front","average":{"1min":6.5449999999999999,"5min":6.5250000000000004,"15min":5.274},"min":{"1min":0.35999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":78.853999999999999,"5min":125.46899999999999,"15min":125.46899999999999},"last":7.8949999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":7.9779999999999998,"5min":6.359,"15min":4.2590000000000003},"min":{"1min":0.28000000000000003,"5min":0.22600000000000001,"15min":0.19800000000000001},"max":{"1min":77.599999999999994,"5min":186.21299999999999,"15min":186.21299999999999},"last":4.6059999999999999},{"interface":"front","average":{"1min":7.0780000000000003,"5min":6.1079999999999997,"15min":4.0209999999999999},"min":{"1min":0.23000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":78.810000000000002,"5min":191.78,"15min":191.78},"last":7.5369999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":3.1579999999999999,"15min":3.097},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":12.593,"5min":72.343999999999994,"15min":106.105},"last":0.58799999999999997},{"interface":"front","average":{"1min":6.9119999999999999,"5min":4.5990000000000002,"15min":3.931},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":80.676000000000002,"5min":80.676000000000002,"15min":106.70399999999999},"last":7.8209999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.0899999999999999,"15min":2.4260000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1529999999999996,"5min":27.539999999999999,"15min":106.85899999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.8239999999999998,"5min":2.4329999999999998,"15min":1.9630000000000001},"min":{"1min":0.34999999999999998,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":8.8550000000000004,"5min":26.663,"15min":37.442},"last":4.6980000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5009999999999999,"5min":2.5880000000000001,"15min":1.9079999999999999},"min":{"1min":0.315,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":11.785,"5min":17.901,"15min":17.901},"last":4.6740000000000004},{"interface":"front","average":{"1min":2.3849999999999998,"5min":2.2509999999999999,"15min":1.8899999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":8.2309999999999999,"5min":18.068000000000001,"15min":18.068000000000001},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":2.133,"15min":2.2440000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.758,"5min":14.976000000000001,"15min":106.036},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.1309999999999998,"5min":2.2629999999999999,"15min":2.3279999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.1069999999999993,"5min":14.529,"15min":106.741},"last":0.64600000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9430000000000001,"5min":2.1160000000000001,"15min":2.3220000000000001},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.34399999999999997},"max":{"1min":7.1280000000000001,"5min":14.291,"15min":106.82599999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.6400000000000001,"15min":2.3809999999999998},"min":{"1min":0.51300000000000001,"5min":0.38900000000000001,"15min":0.35999999999999999},"max":{"1min":11.539999999999999,"5min":20.757999999999999,"15min":106.084},"last":4.641}]}]},{"osd":4,"up_from":31,"seq":133143986489,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":715284,"kb_used_data":401612,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":93029868,"statfs":{"total":95995035648,"available":95262584832,"internally_reserved":0,"allocated":411250688,"data_stored":410392690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:25 2023","interfaces":[{"interface":"back","average":{"1min":8.2710000000000008,"5min":8.1400000000000006,"15min":4.1689999999999996},"min":{"1min":0.27400000000000002,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":92.072000000000003,"5min":154.62100000000001,"15min":154.62100000000001},"last":11.952999999999999},{"interface":"front","average":{"1min":9.3879999999999999,"5min":8.4529999999999994,"15min":4.2229999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.23400000000000001},"max":{"1min":92.233999999999995,"5min":154.37299999999999,"15min":154.37299999999999},"last":6.9850000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":12.978,"5min":6.6230000000000002,"15min":3.4649999999999999},"min":{"1min":0.29799999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":96.861999999999995,"5min":117.81,"15min":117.81},"last":0.88300000000000001},{"interface":"front","average":{"1min":12.298999999999999,"5min":6.3849999999999998,"15min":3.3799999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":96.787000000000006,"5min":117.848,"15min":117.848},"last":9.5999999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.8159999999999998,"15min":1.835},"min":{"1min":0.308,"5min":0.255,"15min":0.245},"max":{"1min":7.5460000000000003,"5min":123.598,"15min":123.598},"last":6.5759999999999996},{"interface":"front","average":{"1min":1.5529999999999999,"5min":2.7029999999999998,"15min":1.756},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":5.0220000000000002,"5min":123.619,"15min":123.619},"last":6.5620000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.827,"5min":3.298,"15min":2.02},"min":{"1min":0.36699999999999999,"5min":0.308,"15min":0.27100000000000002},"max":{"1min":9.5310000000000006,"5min":125.90300000000001,"15min":125.90300000000001},"last":2.1800000000000002},{"interface":"front","average":{"1min":2.855,"5min":3.3900000000000001,"15min":2.1669999999999998},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.29099999999999998},"max":{"1min":8.1389999999999993,"5min":127.90000000000001,"15min":127.90000000000001},"last":8.9169999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:16:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":3.1869999999999998,"15min":1.4810000000000001},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":7.5049999999999999,"5min":111.809,"15min":111.809},"last":4.0860000000000003},{"interface":"front","average":{"1min":1.792,"5min":3.2669999999999999,"15min":1.637},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.25},"max":{"1min":6.6520000000000001,"5min":112.002,"15min":112.002},"last":2.9980000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.085,"15min":1.254},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":5.5519999999999996,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.8799999999999999,"15min":1.1100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.5430000000000001,"5min":78.421999999999997,"15min":78.421999999999997},"last":2.1589999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:17:25 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.119,"15min":0.89500000000000002},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.2610000000000001,"5min":34.039000000000001,"15min":34.039000000000001},"last":4.117},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.216,"15min":0.97599999999999998},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.5890000000000004,"5min":33.433999999999997,"15min":33.433999999999997},"last":0.44}]}]},{"osd":5,"up_from":39,"seq":167503724853,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":676744,"kb_used_data":371264,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93068408,"statfs":{"total":95995035648,"available":95302049792,"internally_reserved":0,"allocated":380174336,"data_stored":379324336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":8.5860000000000003,"5min":10.26,"15min":4.7969999999999997},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":111.312,"5min":151.23599999999999,"15min":151.23599999999999},"last":0.21299999999999999},{"interface":"front","average":{"1min":9.0990000000000002,"5min":10.897,"15min":5.1420000000000003},"min":{"1min":0.23000000000000001,"5min":0.22,"15min":0.22},"max":{"1min":111.096,"5min":151.566,"15min":151.566},"last":6.7039999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":7.6699999999999999,"5min":5.8490000000000002,"15min":3.472},"min":{"1min":0.34000000000000002,"5min":0.217,"15min":0.217},"max":{"1min":111.194,"5min":111.194,"15min":111.194},"last":6.2679999999999998},{"interface":"front","average":{"1min":8.3539999999999992,"5min":6.2759999999999998,"15min":3.8140000000000001},"min":{"1min":0.25700000000000001,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":111.247,"5min":112.226,"15min":112.226},"last":3.2509999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:16:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7549999999999999,"5min":4.4320000000000004,"15min":2.8079999999999998},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":9.8640000000000008,"5min":106.508,"15min":106.508},"last":3.476},{"interface":"front","average":{"1min":2.6219999999999999,"5min":4.0990000000000002,"15min":2.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":8.5280000000000005,"5min":106.462,"15min":106.462},"last":3.3180000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":2.5099999999999998,"15min":2.343},"min":{"1min":0.38,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.544,"5min":27.925000000000001,"15min":91.834999999999994},"last":0.57399999999999995},{"interface":"front","average":{"1min":3.4460000000000002,"5min":2.609,"15min":2.4129999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.122999999999999,"5min":27.939,"15min":91.802000000000007},"last":0.61399999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":1.1539999999999999,"15min":0.88500000000000001},"min":{"1min":0.251,"5min":0.246,"15min":0.246},"max":{"1min":10.162000000000001,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.254},{"interface":"front","average":{"1min":1.9770000000000001,"5min":0.94799999999999995,"15min":0.86499999999999999},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.297000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.29099999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:16:58 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":0.96099999999999997,"15min":1.2609999999999999},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":12.068,"5min":12.068,"15min":123.239},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.913,"5min":1.2709999999999999,"15min":1.373},"min":{"1min":0.33800000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.7720000000000002,"5min":9.7720000000000002,"15min":123.014},"last":0.307}]},{"osd":7,"last update":"Mon Oct 9 04:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":0.98199999999999998,"15min":0.92000000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.096,"5min":12.096,"15min":12.096},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.5369999999999999,"5min":1.2370000000000001,"15min":1.016},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.307},"max":{"1min":9.9760000000000009,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":311361536,"data_stored":311361536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":355991552,"data_stored":355991552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":327815168,"data_stored":327811096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":319422464,"data_stored":319422464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":405143552,"data_stored":405143552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":370216960,"data_stored":370212888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":356646912,"data_stored":356646912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":385355776,"data_stored":385351704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:17:33.316 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:17:33.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-10-09T04:17:33.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:33.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:33.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:33.510 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:33.695 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:17:33.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:33.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:33.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:33.782 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:17:33.839 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:17:33.860 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:33.876 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:17:33.930 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:17:33.968 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:17:33.968 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:34.007 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:17:34.009 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:34.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:33 smithi181 ceph-mon[196979]: from='client.55420 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:17:34.024 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/89981322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:34.048 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:17:34.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:34.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:33 smithi163 ceph-mon[221067]: from='client.55420 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:17:34.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/89981322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:34.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:33 smithi163 ceph-mon[213841]: from='client.55420 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:17:34.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/89981322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:34.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:34.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:34.092 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.122 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:17:34.150 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:34.153 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:34.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:34.248 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:34.431 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:34.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:34.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:34.587 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:34.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:34 smithi163 ceph-mon[221067]: pgmap v805: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 680 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 1067/32688 objects degraded (3.264%); 3839/32688 objects misplaced (11.744%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:17:34.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:34.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:34 smithi163 ceph-mon[213841]: pgmap v805: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 680 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 1067/32688 objects degraded (3.264%); 3839/32688 objects misplaced (11.744%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:17:34.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:34.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:34.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:34 smithi181 ceph-mon[196979]: pgmap v805: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 680 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 1067/32688 objects degraded (3.264%); 3839/32688 objects misplaced (11.744%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:17:34.842 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:34.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:34.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:34.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:34.983 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:35.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:35.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:35.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:35.254 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:35.515 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:35.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:35.624 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:35.669 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:35.853 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:35.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:35.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:36.067 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:36.110 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:17:36.110 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:17:36.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:36.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:36.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:37.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:17:37.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:37.112 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:36 smithi163 ceph-mon[221067]: pgmap v806: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 658 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1067/31614 objects degraded (3.375%); 3651/31614 objects misplaced (11.549%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:37.112 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:37.112 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:36 smithi163 ceph-mon[213841]: pgmap v806: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 658 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1067/31614 objects degraded (3.375%); 3651/31614 objects misplaced (11.549%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:37.112 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:37.115 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:37.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:36 smithi181 ceph-mon[196979]: pgmap v806: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 658 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1067/31614 objects degraded (3.375%); 3651/31614 objects misplaced (11.549%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:37.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:37.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:37.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:37.652 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:17:37.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:37.772 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:37.819 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:38.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1067/31614 objects degraded (3.375%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:38.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1067/31614 objects degraded (3.375%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:38.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1067/31614 objects degraded (3.375%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:38.129 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:38.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:38.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:38.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:38.291 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:38.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:38.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:38.804 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:38.868 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:38.928 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:39.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:38 smithi181 ceph-mon[196979]: pgmap v807: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 547 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1067/26310 objects degraded (4.055%); 3288/26310 objects misplaced (12.497%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3170622396' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T04:17:39.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:38 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T04:17:39.091 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:39.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:39.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:39.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:39.254 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:39.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:38 smithi163 ceph-mon[213841]: pgmap v807: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 547 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1067/26310 objects degraded (4.055%); 3288/26310 objects misplaced (12.497%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:39.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3170622396' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T04:17:39.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:38 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T04:17:39.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:38 smithi163 ceph-mon[221067]: pgmap v807: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 547 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 1067/26310 objects degraded (4.055%); 3288/26310 objects misplaced (12.497%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:17:39.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3170622396' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T04:17:39.313 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:38 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T04:17:39.539 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:17:39.546 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 1 pg_num to 31 2023-10-09T04:17:39.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:39.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:39.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:39.774 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:39.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3246419555' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:17:39.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:17:39.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:39.876 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T04:17:39.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:17:39.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T04:17:39.877 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T04:17:40.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:40.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:40.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:40.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3246419555' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:17:40.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3246419555' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:17:40.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:17:40.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:17:40.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T04:17:40.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:17:40.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T04:17:40.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T04:17:40.213 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:40.310 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:17:40.310 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:17:41.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: pgmap v809: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 506 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 1067/24324 objects degraded (4.387%); 3110/24324 objects misplaced (12.786%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: osdmap e292: 8 total, 8 up, 8 in 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: failed to encode map e292 with expected crc 2023-10-09T04:17:41.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[221067]: failed to encode map e292 with expected crc 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:17:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:17:40] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: pgmap v809: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 506 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 1067/24324 objects degraded (4.387%); 3110/24324 objects misplaced (12.786%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T04:17:41.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: osdmap e292: 8 total, 8 up, 8 in 2023-10-09T04:17:41.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: failed to encode map e292 with expected crc 2023-10-09T04:17:41.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:40 smithi163 ceph-mon[213841]: failed to encode map e292 with expected crc 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: failed to encode map e291 with expected crc 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: pgmap v809: 109 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 506 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 1067/24324 objects degraded (4.387%); 3110/24324 objects misplaced (12.786%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: osdmap e292: 8 total, 8 up, 8 in 2023-10-09T04:17:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: failed to encode map e292 with expected crc 2023-10-09T04:17:41.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:40 smithi181 ceph-mon[196979]: failed to encode map e292 with expected crc 2023-10-09T04:17:41.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:17:42.069 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:41 smithi181 ceph-mon[196979]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:41 smithi181 ceph-mon[196979]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:41 smithi181 ceph-mon[196979]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:41 smithi181 ceph-mon[196979]: osdmap e293: 8 total, 8 up, 8 in 2023-10-09T04:17:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:41 smithi181 ceph-mon[196979]: failed to encode map e293 with expected crc 2023-10-09T04:17:42.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[213841]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[213841]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[213841]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[213841]: osdmap e293: 8 total, 8 up, 8 in 2023-10-09T04:17:42.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[213841]: failed to encode map e293 with expected crc 2023-10-09T04:17:42.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[221067]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[221067]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[221067]: failed to encode map e292 with expected crc 2023-10-09T04:17:42.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[221067]: osdmap e293: 8 total, 8 up, 8 in 2023-10-09T04:17:42.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:41 smithi163 ceph-mon[221067]: failed to encode map e293 with expected crc 2023-10-09T04:17:42.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:42.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:42.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:42.647 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:42.798 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:42 smithi181 ceph-mon[196979]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:42 smithi181 ceph-mon[196979]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:42 smithi181 ceph-mon[196979]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:42 smithi181 ceph-mon[196979]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:42 smithi181 ceph-mon[196979]: pgmap v812: 119 pgs: 10 peering, 10 unknown, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 375 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 1067/18066 objects degraded (5.906%); 2559/18066 objects misplaced (14.165%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:17:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1067/24324 objects degraded (4.387%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:43.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:43.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[221067]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[221067]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[221067]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[221067]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[221067]: pgmap v812: 119 pgs: 10 peering, 10 unknown, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 375 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 1067/18066 objects degraded (5.906%); 2559/18066 objects misplaced (14.165%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:17:43.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1067/24324 objects degraded (4.387%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:43.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[213841]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[213841]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[213841]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[213841]: failed to encode map e293 with expected crc 2023-10-09T04:17:43.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[213841]: pgmap v812: 119 pgs: 10 peering, 10 unknown, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 375 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 1067/18066 objects degraded (5.906%); 2559/18066 objects misplaced (14.165%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:17:43.070 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1067/24324 objects degraded (4.387%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:43.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:43.192 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:43.231 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:43.463 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:17:43.505 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:17:43.566 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:17:43.605 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:17:43.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:43.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:43.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:17:43.669 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:17:43.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:43.741 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:17:43.804 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:17:43.820 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:43.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.833 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.834 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.834 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.834 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.834 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.834 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.834 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:17:43.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2406663348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:44.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:44.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:44.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:44.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2406663348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:44.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2406663348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:44.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177513 8.8 8:118f0be3:::benchmark_data_smithi163_404719_object86929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.204747+0000", 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.020430772, 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.020461170000000001, 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:17:44.229 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177513 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.204747+0000", 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.204747+0000", 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.204746+0000", 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.204749+0000", 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6690000000000002e-06 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.204754+0000", 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4410000000000001e-06 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.204758+0000", 2023-10-09T04:17:44.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3739999999999998e-06 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.214868+0000", 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010110295 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.214990+0000", 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012217900000000001 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.215105+0000", 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115179 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.221351+0000", 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062455640000000003 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:44.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.224590+0000", 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032387409999999998 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177515 8.3 8:c890612a:::benchmark_data_smithi163_404719_object86931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.211908+0000", 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.013269644000000001, 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013394469000000001, 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.235 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177515 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211908+0000", 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211908+0000", 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211907+0000", 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211911+0000", 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7960000000000002e-06 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211919+0000", 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6240000000000008e-06 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211923+0000", 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5640000000000001e-06 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211960+0000", 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6690999999999998e-05 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212076+0000", 2023-10-09T04:17:44.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116906 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212197+0000", 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012010600000000001 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.222585+0000", 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010388656 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.240 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177516 8.9 8:94c86138:::benchmark_data_smithi163_404719_object86932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.211929+0000", 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.013248483, 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013452331999999999, 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177516 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211929+0000", 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211929+0000", 2023-10-09T04:17:44.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211929+0000", 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211931+0000", 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3379999999999999e-06 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211935+0000", 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.578e-06 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211938+0000", 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6759999999999999e-06 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211971+0000", 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3039e-05 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212058+0000", 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7083000000000006e-05 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:17:44.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212131+0000", 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2723000000000002e-05 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.216664+0000", 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045331729999999997 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177519 8.3 8:c58cd497:::benchmark_data_smithi163_404719_object86935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.211946+0000", 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.013231681, 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013516187000000001, 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177519 2023-10-09T04:17:44.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211946+0000", 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211946+0000", 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211945+0000", 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211948+0000", 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3779999999999999e-06 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211953+0000", 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4929999999999998e-06 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211957+0000", 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8689999999999997e-06 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.217209+0000", 2023-10-09T04:17:44.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052521620000000003 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.217341+0000", 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013158500000000001 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.217466+0000", 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124991 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.224561+0000", 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0070953869999999999 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177522 8.d 8:b4af013f:::benchmark_data_smithi163_404719_object86938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.211962+0000", 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.013215222, 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013595205000000001, 2023-10-09T04:17:44.252 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177522 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211962+0000", 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211962+0000", 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211962+0000", 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211964+0000", 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2309999999999999e-06 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211970+0000", 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3389999999999998e-06 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.211974+0000", 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.551e-06 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212021+0000", 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7435000000000002e-05 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:44.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212133+0000", 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011204 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.212239+0000", 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010558899999999999 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177524 8.9 8:910c8419:::benchmark_data_smithi163_404719_object86940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.218915+0000", 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0062619099999999999, 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00671376, 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177524 2023-10-09T04:17:44.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218915+0000", 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218915+0000", 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218914+0000", 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218919+0000", 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.826e-06 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218928+0000", 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1880000000000008e-06 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218933+0000", 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9830000000000004e-06 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.218973+0000", 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9378999999999997e-05 2023-10-09T04:17:44.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.219080+0000", 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107837 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.219190+0000", 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010912300000000001 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.224577+0000", 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005387233 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:177525 8.8 8.2e8e418 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e293)", 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:44.225426+0000", 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 4294967295.9997516, 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000280143, 2023-10-09T04:17:44.263 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 177525 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.225426+0000", 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.225426+0000", 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.225425+0000", 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.225432+0000", 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8019999999999992e-06 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.225450+0000", 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7277000000000001e-05 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:44.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:44.225457+0000", 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1910000000000003e-06 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 7 2023-10-09T04:17:44.267 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:17:44.268 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:44.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:17:44.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:17:45.157 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:45.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:45.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:45.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:44 smithi181 ceph-mon[196979]: pgmap v813: 119 pgs: 10 peering, 10 unknown, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 327 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 1067/15732 objects degraded (6.782%); 2362/15732 objects misplaced (15.014%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:17:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:44 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:44 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:45.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:44 smithi163 ceph-mon[213841]: pgmap v813: 119 pgs: 10 peering, 10 unknown, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 327 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 1067/15732 objects degraded (6.782%); 2362/15732 objects misplaced (15.014%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:17:45.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:44 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:45.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:44 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:45.272 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:44 smithi163 ceph-mon[221067]: pgmap v813: 119 pgs: 10 peering, 10 unknown, 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 327 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 1067/15732 objects degraded (6.782%); 2362/15732 objects misplaced (15.014%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:17:45.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:44 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:45.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:44 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:45.372 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:45.522 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:45.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:45.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:45.694 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:45.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:45.801 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:45.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:45.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:46.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:45 smithi181 ceph-mon[196979]: 1.1 scrub starts 2023-10-09T04:17:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:45 smithi181 ceph-mon[196979]: 1.1 scrub ok 2023-10-09T04:17:46.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:45 smithi181 ceph-mon[196979]: pgmap v814: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 306 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 1067/14733 objects degraded (7.242%); 2201/14733 objects misplaced (14.939%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:17:46.152 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:46.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:45 smithi163 ceph-mon[213841]: 1.1 scrub starts 2023-10-09T04:17:46.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:45 smithi163 ceph-mon[213841]: 1.1 scrub ok 2023-10-09T04:17:46.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:45 smithi163 ceph-mon[213841]: pgmap v814: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 306 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 1067/14733 objects degraded (7.242%); 2201/14733 objects misplaced (14.939%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:17:46.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:45 smithi163 ceph-mon[221067]: 1.1 scrub starts 2023-10-09T04:17:46.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:45 smithi163 ceph-mon[221067]: 1.1 scrub ok 2023-10-09T04:17:46.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:45 smithi163 ceph-mon[221067]: pgmap v814: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 306 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 1067/14733 objects degraded (7.242%); 2201/14733 objects misplaced (14.939%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:17:46.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:46.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:46.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:46.396 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:46.476 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:46.708 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:46.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:46.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:46.900 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:47.283 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:17:47.283 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:17:47.284 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:17:47.284 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:17:47.284 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:17:47.285 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:17:47.571 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:17:47.571 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:17:47.571 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2023-10-09T04:17:47.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T04:17:47.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:47 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1067/14733 objects degraded (7.242%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:47.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:47 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1067/14733 objects degraded (7.242%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:47.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:47 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1067/14733 objects degraded (7.242%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:48.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:17:48.137 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:48.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:48.247 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:48.282 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:48.351 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:48.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:48.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:48.479 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:48.553 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:48.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[213841]: pgmap v815: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 188 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1067/9072 objects degraded (11.761%); 1818/9072 objects misplaced (20.040%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:17:48.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:48.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:48.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[213841]: 1.2 scrub starts 2023-10-09T04:17:48.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[213841]: 1.2 scrub ok 2023-10-09T04:17:48.559 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[221067]: pgmap v815: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 188 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1067/9072 objects degraded (11.761%); 1818/9072 objects misplaced (20.040%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:17:48.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:48.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:48.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[221067]: 1.2 scrub starts 2023-10-09T04:17:48.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:48 smithi163 ceph-mon[221067]: 1.2 scrub ok 2023-10-09T04:17:48.609 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:48.774 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:48 smithi181 ceph-mon[196979]: pgmap v815: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 188 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1067/9072 objects degraded (11.761%); 1818/9072 objects misplaced (20.040%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:17:48.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:48.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:48 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:48.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:48 smithi181 ceph-mon[196979]: 1.2 scrub starts 2023-10-09T04:17:48.775 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:48 smithi181 ceph-mon[196979]: 1.2 scrub ok 2023-10-09T04:17:48.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:48.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:48.954 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:49.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:49.045 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:49.157 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:49.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:49.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:49.296 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:49.349 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:49.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:49.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:49.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:49.488 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:49.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:49.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:49.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:49.565 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:49.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3109174377' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:49.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:49.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:49.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:49.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:49.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:49.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:49.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3109174377' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:49.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:49.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:49.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:17:49.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:49.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3109174377' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:17:49.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:49.944 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:50.194 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:17:50.290 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:17:50.290 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:17:50.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36612 7.0 7:006250d9:::benchmark_data_smithi163_346082_object36611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 481.99555990800002, 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97418880500000005, 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:17:50.291 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36612 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290014+0000", 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290012+0000", 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290059+0000", 2023-10-09T04:17:50.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6390000000000001e-05 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290079+0000", 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0370999999999999e-05 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.290088+0000", 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.183e-06 2023-10-09T04:17:50.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291399+0000", 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013106369999999999 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.291468+0000", 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9116999999999996e-05 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264113+0000", 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97264483000000002 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.264203+0000", 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0143999999999996e-05 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.296 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80494 7.e 7:79ff37ea:::benchmark_data_smithi163_346082_object80493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 432.83594642700001, 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.00624114, 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:50.297 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80494 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:17:50.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449627+0000", 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449685+0000", 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8378000000000002e-05 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449697+0000", 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2427e-05 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449703+0000", 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449751+0000", 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8476999999999997e-05 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449761+0000", 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0378e-05 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441447+0000", 2023-10-09T04:17:50.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99168547399999996 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441622+0000", 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017500600000000001 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441827+0000", 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020516999999999999 2023-10-09T04:17:50.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451473+0000", 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096461649999999999 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452805+0000", 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013312770000000001 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454609+0000", 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018042959999999999 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454649+0000", 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0256999999999998e-05 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.455869+0000", 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001219649 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80509 7.e 7:77548b6a:::benchmark_data_smithi163_346082_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 432.81710845999999, 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0024772179999999, 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.305 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80509 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468464+0000", 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468512+0000", 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615000000000001e-05 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468527+0000", 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4778e-05 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468534+0000", 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2049999999999996e-06 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468595+0000", 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0865000000000001e-05 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468603+0000", 2023-10-09T04:17:50.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2439999999999996e-06 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445422+0000", 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97681920499999997 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445635+0000", 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000213382 2023-10-09T04:17:50.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445836+0000", 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020043 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451507+0000", 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056710220000000004 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452890+0000", 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013838699999999999 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469631+0000", 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01674059 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469678+0000", 2023-10-09T04:17:50.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6551999999999997e-05 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.470943+0000", 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00126515 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.313 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80511 7.e 7:74ae5f43:::benchmark_data_smithi163_346082_object80510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 432.81554615699997, 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002226042, 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80511 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470027+0000", 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470082+0000", 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5106999999999999e-05 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470093+0000", 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0738e-05 2023-10-09T04:17:50.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470098+0000", 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5820000000000003e-06 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470142+0000", 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4261000000000002e-05 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470148+0000", 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0170000000000002e-06 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.447968+0000", 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97782021900000005 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448099+0000", 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013048700000000001 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448204+0000", 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105266 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451513+0000", 2023-10-09T04:17:50.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033096240000000002 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.456169+0000", 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046552060000000003 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471004+0000", 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014834926999999999 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471062+0000", 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8873999999999999e-05 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472254+0000", 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011913939999999999 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80513 7.e 7:768b3fe4:::benchmark_data_smithi163_346082_object80512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 432.81362044700001, 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0014613109999999, 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:17:50.322 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80513 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471952+0000", 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472006+0000", 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3981999999999999e-05 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472018+0000", 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1924e-05 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472023+0000", 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6999999999999999e-06 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472070+0000", 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7218000000000001e-05 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472078+0000", 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8830000000000006e-06 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449809+0000", 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97773131000000002 2023-10-09T04:17:50.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449917+0000", 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107543 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.450027+0000", 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010959399999999999 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451621+0000", 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015948360000000001 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.466881+0000", 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015260029 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472336+0000", 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054546960000000002 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472393+0000", 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6829000000000001e-05 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.473415+0000", 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001021852 2023-10-09T04:17:50.329 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131725 8.3 8:c2b90b78:::benchmark_data_smithi163_404719_object41141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.811038018999994, 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1240774949999999, 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.330 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131725 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474538+0000", 2023-10-09T04:17:50.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4329999999999998e-06 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474549+0000", 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0927000000000001e-05 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474553+0000", 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8800000000000001e-06 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474594+0000", 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1332999999999999e-05 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474602+0000", 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468816+0000", 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99421445799999997 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468827+0000", 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1093e-05 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.586018+0000", 2023-10-09T04:17:50.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.117190543 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.588171+0000", 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021532040000000001 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.588321+0000", 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015008999999999999 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598261+0000", 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099394879999999998 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598299+0000", 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8522000000000001e-05 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598361+0000", 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1239000000000001e-05 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598457+0000", 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6027999999999996e-05 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598531+0000", 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4678999999999994e-05 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598613+0000", 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2201999999999997e-05 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.339 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131729 8.3 8:c6b15366:::benchmark_data_smithi163_404719_object41145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.800952995000003, 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1141904230000002, 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131729 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484620+0000", 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484626+0000", 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0870000000000003e-06 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484639+0000", 2023-10-09T04:17:50.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3502e-05 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484647+0000", 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484692+0000", 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5667999999999997e-05 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484702+0000", 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5850000000000006e-06 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468853+0000", 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98415053600000002 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468860+0000", 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1189999999999999e-06 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591501+0000", 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1226411199999999 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591590+0000", 2023-10-09T04:17:50.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9573000000000005e-05 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591667+0000", 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7033000000000006e-05 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598305+0000", 2023-10-09T04:17:50.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066379239999999999 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598404+0000", 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9100999999999999e-05 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598650+0000", 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024551400000000001 2023-10-09T04:17:50.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598726+0000", 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6175999999999997e-05 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598754+0000", 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7898999999999998e-05 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598811+0000", 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7370999999999997e-05 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131731 8.3 8:c392b46b:::benchmark_data_smithi163_404719_object41147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:17:50.349 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.800881398000001, 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1142496469999998, 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131731 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484692+0000", 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:50.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484695+0000", 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7630000000000001e-06 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484701+0000", 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1909999999999997e-06 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484705+0000", 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9700000000000001e-06 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484752+0000", 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6964e-05 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484761+0000", 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5590000000000006e-06 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468882+0000", 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98412151699999995 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468889+0000", 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8260000000000004e-06 2023-10-09T04:17:50.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594688+0000", 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1257991060000001 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594798+0000", 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010983 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594900+0000", 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102451 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598656+0000", 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037554839999999999 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598683+0000", 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6998000000000001e-05 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598703+0000", 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9925999999999999e-05 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598843+0000", 2023-10-09T04:17:50.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014050299999999999 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598872+0000", 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8965000000000001e-05 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598942+0000", 2023-10-09T04:17:50.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0003e-05 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131738 8.3 8:cc1c5c5f:::benchmark_data_smithi163_404719_object41154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.790256997, 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1194174960000001, 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.359 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131738 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:17:50.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:17:50.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495319+0000", 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9379999999999999e-06 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495324+0000", 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1200000000000001e-06 2023-10-09T04:17:50.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495326+0000", 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4820000000000001e-06 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495378+0000", 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1295000000000003e-05 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495384+0000", 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4320000000000004e-06 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468935+0000", 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97355068700000003 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468944+0000", 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.816e-06 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598966+0000", 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.130022938 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.599049+0000", 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2161000000000002e-05 2023-10-09T04:17:50.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.599124+0000", 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4906000000000004e-05 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614430+0000", 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015306491 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614476+0000", 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6394999999999999e-05 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614536+0000", 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9799000000000001e-05 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614566+0000", 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0008000000000001e-05 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614623+0000", 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6487999999999999e-05 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614735+0000", 2023-10-09T04:17:50.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111797 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131739 8.3 8:ccd4c823:::benchmark_data_smithi163_404719_object41155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.769555080000003, 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.098935633, 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.369 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131739 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:17:50.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516018+0000", 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516024+0000", 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.421e-06 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516038+0000", 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3558e-05 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516044+0000", 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4400000000000002e-06 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516099+0000", 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4361999999999999e-05 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516110+0000", 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1661000000000001e-05 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468972+0000", 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95286181000000003 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468979+0000", 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6800000000000004e-06 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602170+0000", 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.133191676 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602304+0000", 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013313999999999999 2023-10-09T04:17:50.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602429+0000", 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012518499999999999 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614498+0000", 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012068972000000001 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614780+0000", 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000281978 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614813+0000", 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3337000000000003e-05 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614843+0000", 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0256999999999998e-05 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614872+0000", 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.898e-05 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614955+0000", 2023-10-09T04:17:50.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2304000000000003e-05 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131750 8.3 8:caad7466:::benchmark_data_smithi163_404719_object41166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.752749731999998, 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0823976559999999, 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131750 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532826+0000", 2023-10-09T04:17:50.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.779e-06 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532829+0000", 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4290000000000001e-06 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532832+0000", 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3879999999999998e-06 2023-10-09T04:17:50.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532884+0000", 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2642999999999997e-05 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532892+0000", 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6290000000000001e-06 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469000+0000", 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93610853400000005 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469007+0000", 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.605999+0000", 2023-10-09T04:17:50.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.136991761 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.606144+0000", 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014551200000000001 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.606301+0000", 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000156458 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614505+0000", 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082041619999999992 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614978+0000", 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047337600000000002 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615009+0000", 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0454999999999999e-05 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615040+0000", 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1758000000000003e-05 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615086+0000", 2023-10-09T04:17:50.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5494999999999997e-05 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615222+0000", 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013613699999999999 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.388 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131757 8.3 8:cde86114:::benchmark_data_smithi163_404719_object41173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.734466843999996, 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0643897739999999, 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131757 2023-10-09T04:17:50.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:17:50.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551105+0000", 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551114+0000", 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2609999999999993e-06 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551130+0000", 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5934e-05 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551137+0000", 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1019999999999999e-06 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551191+0000", 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4138000000000002e-05 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551202+0000", 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.164e-05 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469028+0000", 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91782591899999999 2023-10-09T04:17:50.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469035+0000", 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2580000000000001e-06 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610215+0000", 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1411807170000001 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610340+0000", 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012489900000000001 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610434+0000", 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4154999999999996e-05 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614745+0000", 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043102599999999998 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615262+0000", 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00051697899999999998 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615294+0000", 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2675000000000001e-05 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615328+0000", 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3495999999999999e-05 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615374+0000", 2023-10-09T04:17:50.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6514000000000002e-05 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615497+0000", 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122583 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.398 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131773 8.3 8:cc396e7f:::benchmark_data_smithi163_404719_object41189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 86.701143524000003, 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0561476569999999, 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131773 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584429+0000", 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584435+0000", 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4090000000000002e-06 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584448+0000", 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.361e-05 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584456+0000", 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6389999999999992e-06 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584515+0000", 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8638999999999999e-05 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584525+0000", 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0314e-05 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469134+0000", 2023-10-09T04:17:50.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88460910100000001 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469142+0000", 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0639999999999994e-06 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615534+0000", 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1463917850000001 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615640+0000", 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010655800000000001 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615751+0000", 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011056 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627128+0000", 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011377376 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631239+0000", 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041103110000000002 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631277+0000", 2023-10-09T04:17:50.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8482000000000002e-05 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640379+0000", 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0091021850000000005 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640436+0000", 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6545000000000002e-05 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640578+0000", 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142261 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131846 8.3 8:c021821e:::benchmark_data_smithi163_404719_object41262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:17:50.408 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.497209886999997, 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.852533604, 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131846 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788363+0000", 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788368+0000", 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5560000000000001e-06 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788375+0000", 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1559999999999998e-06 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788379+0000", 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0430000000000001e-06 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788435+0000", 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5859000000000002e-05 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788441+0000", 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.156e-06 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619638+0000", 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.831196571 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619758+0000", 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120253 2023-10-09T04:17:50.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619848+0000", 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9886e-05 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631195+0000", 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011347183 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631309+0000", 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113992 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640695+0000", 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093864970000000006 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640734+0000", 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8285999999999999e-05 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640782+0000", 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615999999999997e-05 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640898+0000", 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011548900000000001 2023-10-09T04:17:50.416 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131856 8.3 8:c67caa71:::benchmark_data_smithi163_404719_object41272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.466051929000002, 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82161764000000004, 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131856 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819524+0000", 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1560000000000002e-06 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819529+0000", 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2159999999999997e-06 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819532+0000", 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.79e-06 2023-10-09T04:17:50.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819581+0000", 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8748999999999997e-05 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819589+0000", 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8879999999999997e-06 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623021+0000", 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80343243799999997 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623126+0000", 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105138 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623256+0000", 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129383 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631246+0000", 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079899900000000006 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631345+0000", 2023-10-09T04:17:50.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8989999999999999e-05 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640971+0000", 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096260070000000007 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641005+0000", 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4100000000000002e-05 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641043+0000", 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8825000000000002e-05 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641140+0000", 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6241000000000002e-05 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.425 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131865 8.3 8:c397b806:::benchmark_data_smithi163_404719_object41281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.444146205999999, 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79990704499999998, 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.426 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131865 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841427+0000", 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841432+0000", 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7679999999999997e-06 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841444+0000", 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1588e-05 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841449+0000", 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0769999999999998e-06 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841505+0000", 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6144000000000003e-05 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841514+0000", 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6100000000000006e-06 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627156+0000", 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78564268000000004 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627290+0000", 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013397600000000001 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627401+0000", 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011101 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640330+0000", 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012928057 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640641+0000", 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031165099999999997 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641173+0000", 2023-10-09T04:17:50.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00053144399999999995 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641202+0000", 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9847000000000001e-05 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641241+0000", 2023-10-09T04:17:50.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8713e-05 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641335+0000", 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3648999999999997e-05 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131881 8.3 8:cd41d762:::benchmark_data_smithi163_404719_object41297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.434 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.407462019999997, 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79379251699999998, 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131881 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878111+0000", 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:50.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878118+0000", 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.956e-06 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878131+0000", 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3141e-05 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878138+0000", 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8109999999999997e-06 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878215+0000", 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7330000000000007e-05 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878228+0000", 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3209e-05 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631369+0000", 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75314052899999995 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631495+0000", 2023-10-09T04:17:50.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125823 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631615+0000", 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120013 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640655+0000", 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0090400859999999993 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640937+0000", 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028247100000000001 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649876+0000", 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089392050000000004 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671738+0000", 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021861968999999998 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671788+0000", 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9493999999999997e-05 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671905+0000", 2023-10-09T04:17:50.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011675699999999999 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131884 8.3 8:c4f20fc8:::benchmark_data_smithi163_404719_object41300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.402258298000007, 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78877238199999999, 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.443 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131884 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883314+0000", 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883321+0000", 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0639999999999996e-06 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883334+0000", 2023-10-09T04:17:50.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2595000000000001e-05 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883340+0000", 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3409999999999999e-06 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883406+0000", 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6045999999999998e-05 2023-10-09T04:17:50.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883416+0000", 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0387000000000001e-05 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635386+0000", 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75196957900000005 2023-10-09T04:17:50.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635512+0000", 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012591300000000001 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635633+0000", 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121377 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640702+0000", 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050688790000000001 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641370+0000", 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00066799299999999997 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649915+0000", 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085445550000000006 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671963+0000", 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022048742 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671998+0000", 2023-10-09T04:17:50.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4585000000000002e-05 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672088+0000", 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0163999999999999e-05 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131932 8.3 8:c8cbc02f:::benchmark_data_smithi163_404719_object41348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.277441562999996, 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.66412386599999995, 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.453 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131932 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:17:50.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008131+0000", 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008140+0000", 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.901e-06 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008151+0000", 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008158+0000", 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3949999999999998e-06 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008225+0000", 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7541999999999998e-05 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008237+0000", 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1732999999999999e-05 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641390+0000", 2023-10-09T04:17:50.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.633153454 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641535+0000", 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014420899999999999 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641667+0000", 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205199999999999 2023-10-09T04:17:50.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649884+0000", 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082173850000000007 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649944+0000", 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0229000000000002e-05 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649976+0000", 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1479000000000001e-05 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672141+0000", 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022165625000000001 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672185+0000", 2023-10-09T04:17:50.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3373e-05 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672256+0000", 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1606000000000005e-05 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131959 8.3 8:c13e43ce:::benchmark_data_smithi163_404719_object41375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:17:50.461 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 85.208730512000002, 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.59562893100000003, 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131959 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:17:50.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:17:50.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076848+0000", 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076858+0000", 2023-10-09T04:17:50.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0603e-05 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076863+0000", 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4159999999999997e-06 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076923+0000", 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9760999999999999e-05 2023-10-09T04:17:50.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076933+0000", 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0849999999999999e-05 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645730+0000", 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.56879615900000002 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645860+0000", 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013080899999999999 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645984+0000", 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012326399999999999 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658944+0000", 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012960704999999999 2023-10-09T04:17:50.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658951+0000", 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0469999999999996e-06 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658992+0000", 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1028000000000001e-05 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672327+0000", 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013334278999999999 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672372+0000", 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4783000000000001e-05 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:17:50.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672472+0000", 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010089 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:17:50.470 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:17:50.471 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 90581 objects 2023-10-09T04:17:50.471 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :150.06 2023-10-09T04:17:50.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:50.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:50.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:50.614 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:50.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:50.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:50.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:50.788 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:50.831 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:17:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:17:50] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2023-10-09T04:17:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[221067]: pgmap v816: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 143 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1067/6918 objects degraded (15.424%); 1608/6918 objects misplaced (23.244%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T04:17:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:17:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3109174377' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:50.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[221067]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T04:17:50.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[213841]: pgmap v816: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 143 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1067/6918 objects degraded (15.424%); 1608/6918 objects misplaced (23.244%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T04:17:50.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:17:50.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3109174377' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:50.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:50 smithi163 ceph-mon[213841]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T04:17:50.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:50 smithi181 ceph-mon[196979]: pgmap v816: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 143 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1067/6918 objects degraded (15.424%); 1608/6918 objects misplaced (23.244%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T04:17:50.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:50 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:17:50.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3109174377' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:17:50.834 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:50 smithi181 ceph-mon[196979]: osdmap e294: 8 total, 8 up, 8 in 2023-10-09T04:17:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:50.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:50.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:50.963 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:51.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:17:51.113 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:51.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:51.218 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:51.275 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:51.467 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:51 smithi163 ceph-mon[221067]: failed to encode map e294 with expected crc 2023-10-09T04:17:51.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:51 smithi163 ceph-mon[221067]: failed to encode map e294 with expected crc 2023-10-09T04:17:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:51 smithi163 ceph-mon[213841]: failed to encode map e294 with expected crc 2023-10-09T04:17:51.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:51 smithi163 ceph-mon[213841]: failed to encode map e294 with expected crc 2023-10-09T04:17:51.960 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:51 smithi181 ceph-mon[196979]: failed to encode map e294 with expected crc 2023-10-09T04:17:51.961 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:51 smithi181 ceph-mon[196979]: failed to encode map e294 with expected crc 2023-10-09T04:17:52.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:52.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:52.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:52.708 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:52.721 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:52 smithi181 ceph-mon[196979]: pgmap v818: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 21 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 716 B/s rd, 1.9 KiB/s wr, 567 op/s; 1067/1059 objects degraded (100.755%); 1093/1059 objects misplaced (103.211%); 2.5 MiB/s, 41 objects/s recovering 2023-10-09T04:17:52.721 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1067/6918 objects degraded (15.424%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:52.764 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:52.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:52.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:52.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:52 smithi163 ceph-mon[221067]: pgmap v818: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 21 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 716 B/s rd, 1.9 KiB/s wr, 567 op/s; 1067/1059 objects degraded (100.755%); 1093/1059 objects misplaced (103.211%); 2.5 MiB/s, 41 objects/s recovering 2023-10-09T04:17:52.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1067/6918 objects degraded (15.424%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:52.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:52 smithi163 ceph-mon[213841]: pgmap v818: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 21 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 716 B/s rd, 1.9 KiB/s wr, 567 op/s; 1067/1059 objects degraded (100.755%); 1093/1059 objects misplaced (103.211%); 2.5 MiB/s, 41 objects/s recovering 2023-10-09T04:17:52.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1067/6918 objects degraded (15.424%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:52.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:17:52.869 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:17:52.873 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:17:52.891 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:17:52.901 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:52.912 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:17:52.931 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:17:52.931 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:17:52.932 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:17:52.932 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:17:52.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:17:52.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:17:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:52.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:17:52.954 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:17:52.975 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:17:52.995 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:17:52.997 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:17:53.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:53.078 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:53.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:53.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:53.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:53.311 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:53.362 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:53.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:53.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:17:53.494 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:53.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:17:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:53.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:17:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:53 smithi163 conmon[213817]: 2023-10-09T04:17:53.381+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e294 definitely_dead 0 2023-10-09T04:17:53.603 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.3. 2023-10-09T04:17:53.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:17:53.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:53.692 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:53.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:53.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:53.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:53.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2956290827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:53.875 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/56959117' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:17:53.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:17:53.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[221067]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:17:53.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2956290827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:53.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/56959117' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:17:53.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:17:53.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:53 smithi163 ceph-mon[213841]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:17:53.878 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:53.988 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:54.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2956290827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:17:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/56959117' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:17:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:53 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:17:54.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:53 smithi181 ceph-mon[196979]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:17:54.308 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T04:17:54.309 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T04:17:54.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:54.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:54.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:17:54.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[221067]: failed to encode map e295 with expected crc 2023-10-09T04:17:54.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[221067]: failed to encode map e295 with expected crc 2023-10-09T04:17:54.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:17:54.680 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[221067]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:17:54.681 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[221067]: pgmap v820: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 597 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 798/78 objects degraded (1023.077%); 798/78 objects misplaced (1023.077%); 1.8 MiB/s, 84 objects/s recovering 2023-10-09T04:17:54.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[213841]: failed to encode map e295 with expected crc 2023-10-09T04:17:54.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[213841]: failed to encode map e295 with expected crc 2023-10-09T04:17:54.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:17:54.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[213841]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:17:54.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:54 smithi163 ceph-mon[213841]: pgmap v820: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 597 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 798/78 objects degraded (1023.077%); 798/78 objects misplaced (1023.077%); 1.8 MiB/s, 84 objects/s recovering 2023-10-09T04:17:54.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:54.709 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:54 smithi181 ceph-mon[196979]: failed to encode map e295 with expected crc 2023-10-09T04:17:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:54 smithi181 ceph-mon[196979]: failed to encode map e295 with expected crc 2023-10-09T04:17:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:17:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:54 smithi181 ceph-mon[196979]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:17:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:54 smithi181 ceph-mon[196979]: pgmap v820: 119 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 115 active+clean; 597 KiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 798/78 objects degraded (1023.077%); 798/78 objects misplaced (1023.077%); 1.8 MiB/s, 84 objects/s recovering 2023-10-09T04:17:55.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:55.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:55.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:17:55.498 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:55.513 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:55.546 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:55.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:17:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:55.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:17:55.657 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.657 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.658 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.658 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.658 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:17:55.658 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.658 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:55 smithi181 ceph-mon[196979]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.723 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[213841]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.724 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:55 smithi163 ceph-mon[221067]: failed to encode map e296 with expected crc 2023-10-09T04:17:55.731 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:55.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:55.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:55.902 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:17:55.902 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:17:55.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:55.957 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:56.158 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:56.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:56.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:56.288 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:56.331 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:17:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:56.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:17:56.435 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:56.463 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:56.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:17:56.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 294'11388 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 294'11606 2023-10-09T04:17:56.818 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 294'11256 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: 8.b starting backfill to osd.7 from (0'0,0'0] MAX to 294'11326 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 294'11308 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 294'11304 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[221067]: pgmap v823: 119 pgs: 1 remapped+peering, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 114 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 798/75 objects degraded (1064.000%); 798/75 objects misplaced (1064.000%); 283 KiB/s, 83 objects/s recovering 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.819 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 294'11388 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 294'11606 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 294'11256 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: 8.b starting backfill to osd.7 from (0'0,0'0] MAX to 294'11326 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 294'11308 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 294'11304 2023-10-09T04:17:56.820 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:56 smithi163 ceph-mon[213841]: pgmap v823: 119 pgs: 1 remapped+peering, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 114 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 798/75 objects degraded (1064.000%); 798/75 objects misplaced (1064.000%); 283 KiB/s, 83 objects/s recovering 2023-10-09T04:17:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:56.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:56.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 294'11388 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: failed to encode map e297 with expected crc 2023-10-09T04:17:56.948 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 294'11606 2023-10-09T04:17:56.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: 8.c starting backfill to osd.5 from (0'0,0'0] MAX to 294'11256 2023-10-09T04:17:56.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: 8.b starting backfill to osd.7 from (0'0,0'0] MAX to 294'11326 2023-10-09T04:17:56.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 294'11308 2023-10-09T04:17:56.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 294'11304 2023-10-09T04:17:56.949 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:56 smithi181 ceph-mon[196979]: pgmap v823: 119 pgs: 1 remapped+peering, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 114 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 798/75 objects degraded (1064.000%); 798/75 objects misplaced (1064.000%); 283 KiB/s, 83 objects/s recovering 2023-10-09T04:17:56.951 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:57.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:57.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:57.041 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:57.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:57.136 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:57.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:57.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:57.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:57.362 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:57.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:17:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[213841]: failed to encode map e298 with expected crc 2023-10-09T04:17:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[213841]: failed to encode map e298 with expected crc 2023-10-09T04:17:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[213841]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:17:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[213841]: failed to encode map e298 with expected crc 2023-10-09T04:17:57.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 798/75 objects degraded (1064.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:57.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[221067]: failed to encode map e298 with expected crc 2023-10-09T04:17:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[221067]: failed to encode map e298 with expected crc 2023-10-09T04:17:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[221067]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:17:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[221067]: failed to encode map e298 with expected crc 2023-10-09T04:17:57.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 798/75 objects degraded (1064.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:58.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:57 smithi181 ceph-mon[196979]: failed to encode map e298 with expected crc 2023-10-09T04:17:58.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:57 smithi181 ceph-mon[196979]: failed to encode map e298 with expected crc 2023-10-09T04:17:58.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:57 smithi181 ceph-mon[196979]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:17:58.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:57 smithi181 ceph-mon[196979]: failed to encode map e298 with expected crc 2023-10-09T04:17:58.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 798/75 objects degraded (1064.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:17:58.444 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:17:58.445 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:17:58.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:58.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:58.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:58.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:17:58.653 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:58.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:17:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:58.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:17:58.760 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:58.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:58.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[213841]: failed to encode map e299 with expected crc 2023-10-09T04:17:58.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[213841]: failed to encode map e299 with expected crc 2023-10-09T04:17:58.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[213841]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:17:58.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[213841]: failed to encode map e299 with expected crc 2023-10-09T04:17:58.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[213841]: pgmap v826: 119 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 112 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 798/75 objects degraded (1064.000%); 798/75 objects misplaced (1064.000%); 22 B/s, 1 objects/s recovering 2023-10-09T04:17:58.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[221067]: failed to encode map e299 with expected crc 2023-10-09T04:17:58.825 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[221067]: failed to encode map e299 with expected crc 2023-10-09T04:17:58.825 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[221067]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:17:58.825 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[221067]: failed to encode map e299 with expected crc 2023-10-09T04:17:58.825 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:58 smithi163 ceph-mon[221067]: pgmap v826: 119 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 112 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 798/75 objects degraded (1064.000%); 798/75 objects misplaced (1064.000%); 22 B/s, 1 objects/s recovering 2023-10-09T04:17:58.837 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:58 smithi181 ceph-mon[196979]: failed to encode map e299 with expected crc 2023-10-09T04:17:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:58 smithi181 ceph-mon[196979]: failed to encode map e299 with expected crc 2023-10-09T04:17:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:58 smithi181 ceph-mon[196979]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:17:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:58 smithi181 ceph-mon[196979]: failed to encode map e299 with expected crc 2023-10-09T04:17:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:58 smithi181 ceph-mon[196979]: pgmap v826: 119 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 112 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 798/75 objects degraded (1064.000%); 798/75 objects misplaced (1064.000%); 22 B/s, 1 objects/s recovering 2023-10-09T04:17:59.038 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:59.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:59.142 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:17:59.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:17:59.178 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:59.287 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:17:59.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:59.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:17:59.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:17:59.428 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:59.494 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:17:59.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:17:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:59.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:17:59.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/219223466' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:17:59.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3051083975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:59.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:17:59 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:59.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/219223466' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:17:59.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3051083975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:59.624 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:17:59 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:59.630 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:17:59.637 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_2' removed 2023-10-09T04:17:59.638 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:17:59.685 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:17:59.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:17:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:17:59.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:17:59.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:17:59.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/219223466' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:17:59.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3051083975' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:59.887 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:17:59 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:17:59.889 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:00.085 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:00.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:00.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:00.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:00.217 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:00.426 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 6, 7, 4] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:18:00.427 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:18:00.427 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T04:18:00.427 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:18:00.427 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:18:00.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T04:18:00.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: failed to encode map e300 with expected crc 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/219223466' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: failed to encode map e300 with expected crc 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: pgmap v828: 103 pgs: 103 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T04:18:00.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T04:18:00.977 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:18:00.977 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:18:01.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:18:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:18:00] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2023-10-09T04:18:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: failed to encode map e300 with expected crc 2023-10-09T04:18:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/219223466' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:18:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: failed to encode map e300 with expected crc 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: pgmap v828: 103 pgs: 103 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: failed to encode map e300 with expected crc 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/219223466' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: failed to encode map e300 with expected crc 2023-10-09T04:18:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: pgmap v828: 103 pgs: 103 active+clean; 533 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T04:18:01.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T04:18:01.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:01.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:01.218 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:01.247 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:01.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:01.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:01.395 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:01.421 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:01.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:01.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:01.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:01.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:01.593 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:01.729 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:18:01.729 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:18:01.730 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T04:18:01.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T04:18:01.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:18:01.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:01.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:01.811 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:01.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:01.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 798/75 objects degraded (1064.000%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:18:01.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[221067]: failed to encode map e301 with expected crc 2023-10-09T04:18:01.879 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T04:18:01.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[221067]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:18:01.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 798/75 objects degraded (1064.000%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:18:01.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[213841]: failed to encode map e301 with expected crc 2023-10-09T04:18:01.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T04:18:01.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:01 smithi163 ceph-mon[213841]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:18:01.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:01 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 798/75 objects degraded (1064.000%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:18:01.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:01 smithi181 ceph-mon[196979]: failed to encode map e301 with expected crc 2023-10-09T04:18:01.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:01 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T04:18:01.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:01 smithi181 ceph-mon[196979]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:18:01.898 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:01.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:01.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:01.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:02.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:02.079 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:02.116 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:18:02.127 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:02.148 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:02.168 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:02.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:02.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:02.191 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:02.209 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:02.227 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:02.246 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:02.264 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:18:02.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:02.318 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:02.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:02.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:02.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:02.502 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:02.645 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.645 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T04:18:02.645 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.646 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.646 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.646 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.646 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: pgmap v831: 103 pgs: 24 remapped+peering, 4 peering, 75 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 95 KiB/s rd, 49 op/s 2023-10-09T04:18:02.646 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/70425294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T04:18:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: pgmap v831: 103 pgs: 24 remapped+peering, 4 peering, 75 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 95 KiB/s rd, 49 op/s 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/70425294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: failed to encode map e302 with expected crc 2023-10-09T04:18:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: pgmap v831: 103 pgs: 24 remapped+peering, 4 peering, 75 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 95 KiB/s rd, 49 op/s 2023-10-09T04:18:02.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/70425294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:03.406 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:03.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:03.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:03.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:03.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:03.580 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:18:03.580 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:18:03.580 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:18:03.580 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:18:03.581 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:18:03.587 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:03.632 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:03.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:03.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:03.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:18:03.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:03.762 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:03.807 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:03.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:03.910 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:03.938 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 24 pgs peering (PG_AVAILABILITY) 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[221067]: failed to encode map e303 with expected crc 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[221067]: failed to encode map e303 with expected crc 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[221067]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[221067]: failed to encode map e303 with expected crc 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 24 pgs peering (PG_AVAILABILITY) 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[213841]: failed to encode map e303 with expected crc 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[213841]: failed to encode map e303 with expected crc 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[213841]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T04:18:03.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:03 smithi163 ceph-mon[213841]: failed to encode map e303 with expected crc 2023-10-09T04:18:03.941 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:03 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 24 pgs peering (PG_AVAILABILITY) 2023-10-09T04:18:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:03 smithi181 ceph-mon[196979]: failed to encode map e303 with expected crc 2023-10-09T04:18:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:03 smithi181 ceph-mon[196979]: failed to encode map e303 with expected crc 2023-10-09T04:18:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:03 smithi181 ceph-mon[196979]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T04:18:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:03 smithi181 ceph-mon[196979]: failed to encode map e303 with expected crc 2023-10-09T04:18:04.029 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:18:04.029 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T04:18:04.030 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:18:04.282 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:18:04.625 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:18:04.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:04.899 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: failed to encode map e303 with expected crc 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: pgmap v834: 103 pgs: 24 remapped+peering, 4 peering, 75 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 99 KiB/s rd, 265 KiB/s wr, 77 op/s; 190 KiB/s, 0 objects/s recovering 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:04.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: failed to encode map e303 with expected crc 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: failed to encode map e304 with expected crc 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: pgmap v834: 103 pgs: 24 remapped+peering, 4 peering, 75 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 99 KiB/s rd, 265 KiB/s wr, 77 op/s; 190 KiB/s, 0 objects/s recovering 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:04.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:04.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:05.003 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:05.011 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: failed to encode map e303 with expected crc 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: failed to encode map e304 with expected crc 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: failed to encode map e304 with expected crc 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: failed to encode map e304 with expected crc 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: failed to encode map e304 with expected crc 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: failed to encode map e304 with expected crc 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: pgmap v834: 103 pgs: 24 remapped+peering, 4 peering, 75 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 99 KiB/s rd, 265 KiB/s wr, 77 op/s; 190 KiB/s, 0 objects/s recovering 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:05.012 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:05.036 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:05.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:05.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:05.170 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:05.191 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:05.220 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:05.270 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:05.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:05.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:05.402 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:05.435 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:18:05.454 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:05.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:05.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:05.587 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:05.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:05.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:05.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:05.824 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:05.910 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.910 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.910 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T04:18:05.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: failed to encode map e305 with expected crc 2023-10-09T04:18:05.911 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:05.919 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:05.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:06.023 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:06.051 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:06.158 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:18:06.158 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:58923 7.15 7:af81a40f:::benchmark_data_smithi163_346082_object58922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.281027+0000", 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 471.87423901599999, 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301018, 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.159 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 58923 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281027+0000", 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281027+0000", 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281026+0000", 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281079+0000", 2023-10-09T04:18:06.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2969000000000001e-05 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281094+0000", 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4844999999999999e-05 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281101+0000", 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2219999999999996e-06 2023-10-09T04:18:06.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281157+0000", 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5467999999999998e-05 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.281165+0000", 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3739999999999998e-06 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.308095+0000", 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.026930361 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.308204+0000", 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108232 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.308306+0000", 2023-10-09T04:18:06.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010252299999999999 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310963+0000", 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026565629999999998 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.310971+0000", 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3459999999999994e-06 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311061+0000", 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9554000000000004e-05 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311091+0000", 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0043e-05 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311129+0000", 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8433999999999998e-05 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:58929 7.15 7:abb9695c:::benchmark_data_smithi163_346082_object58928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e190)", 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:14.287449+0000", 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 471.86781723199999, 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.025253508, 2023-10-09T04:18:06.167 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 58929 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287449+0000", 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287449+0000", 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287448+0000", 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287560+0000", 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112825 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287575+0000", 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4942e-05 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287582+0000", 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3300000000000004e-06 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287706+0000", 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123996 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:14.287715+0000", 2023-10-09T04:18:06.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5380000000000008e-06 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.309383+0000", 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0216676739999999 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.309460+0000", 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7250999999999996e-05 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.309569+0000", 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109252 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.311856+0000", 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022868570000000002 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312607+0000", 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00075095399999999997 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312640+0000", 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2662e-05 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312668+0000", 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7776e-05 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:15.312703+0000", 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4949e-05 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80483 7.5 7:a6133132:::benchmark_data_smithi163_346082_object80482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.158958+0000", 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 448.99630813499999, 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2345513130000001, 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.176 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80483 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.158958+0000", 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.158958+0000", 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.158957+0000", 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159013+0000", 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6461e-05 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159025+0000", 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1671000000000001e-05 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159030+0000", 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7999999999999998e-06 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159069+0000", 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9007000000000001e-05 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159189+0000", 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011971 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.159306+0000", 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011739900000000001 2023-10-09T04:18:06.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.160632+0000", 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013256570000000001 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.160849+0000", 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000217123 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310638+0000", 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14978875999999999 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310645+0000", 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7449999999999995e-06 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.393397+0000", 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.082751867 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.393510+0000", 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112316 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.183 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80484 7.15 7:ade26b62:::benchmark_data_smithi163_346082_object80483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 448.990015185, 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.249257176, 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80484 2023-10-09T04:18:06.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165250+0000", 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165302+0000", 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2011000000000001e-05 2023-10-09T04:18:06.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165315+0000", 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2316e-05 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165321+0000", 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0920000000000003e-06 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165361+0000", 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9743000000000002e-05 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165448+0000", 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7162000000000003e-05 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165538+0000", 2023-10-09T04:18:06.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0328999999999995e-05 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.166751+0000", 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012134100000000001 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.167144+0000", 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000392896 2023-10-09T04:18:06.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310638+0000", 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14349324699999999 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310649+0000", 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1039999999999999e-05 2023-10-09T04:18:06.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326699+0000", 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.016050111 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326711+0000", 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2063e-05 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414399+0000", 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0876884 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414508+0000", 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109128 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.192 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80500 7.15 7:a9cb5e2a:::benchmark_data_smithi163_346082_object80499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.457432+0000", 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 448.697834043, 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9663766489999999, 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.193 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80500 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457432+0000", 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457432+0000", 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457431+0000", 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457464+0000", 2023-10-09T04:18:06.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2923999999999999e-05 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457477+0000", 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3018e-05 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457490+0000", 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2914e-05 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457548+0000", 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8184999999999999e-05 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.457557+0000", 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9069999999999996e-06 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326812+0000", 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86925475900000004 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326821+0000", 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.611e-06 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414770+0000", 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0879485680000001 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414935+0000", 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016542 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1,2", 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.415125+0000", 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018943599999999999 2023-10-09T04:18:06.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420602+0000", 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054771819999999997 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420616+0000", 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3730000000000001e-05 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.420656+0000", 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0658999999999998e-05 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423666+0000", 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030100000000000001 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423736+0000", 2023-10-09T04:18:06.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9595e-05 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.423809+0000", 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2886000000000007e-05 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.202 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183833 7.15 7:adbb6c64:::benchmark_data_smithi163_346082_object90563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.360634+0000", 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.79463245800002, 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6201656880000002, 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.203 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183833 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360634+0000", 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360634+0000", 2023-10-09T04:18:06.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360632+0000", 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360641+0000", 2023-10-09T04:18:06.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4169999999999992e-06 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360658+0000", 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7110999999999999e-05 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360665+0000", 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4189999999999996e-06 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360727+0000", 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1489999999999996e-05 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.360739+0000", 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2537999999999999e-05 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771501+0000", 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.41076157099999999 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771512+0000", 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0951999999999999e-05 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.954286+0000", 2023-10-09T04:18:06.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.182774204 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.954403+0000", 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116797 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.954504+0000", 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101217 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962102+0000", 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0075979860000000001 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962229+0000", 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000126823 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.980478+0000", 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018249263000000002 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.980530+0000", 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2188000000000002e-05 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.980800+0000", 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026928500000000002 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183855 7.15 7:ae67ab08:::benchmark_data_smithi163_346082_object90585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.389781+0000", 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.76548498900002, 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5912880889999999, 2023-10-09T04:18:06.212 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183855 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389781+0000", 2023-10-09T04:18:06.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389781+0000", 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389780+0000", 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389789+0000", 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1929999999999999e-06 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389805+0000", 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5602000000000001e-05 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389811+0000", 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.55e-06 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389891+0000", 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9732000000000002e-05 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.389900+0000", 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0839999999999998e-06 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771543+0000", 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.38164336500000001 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771548+0000", 2023-10-09T04:18:06.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2020000000000003e-06 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.957125+0000", 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1855760879999999 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.957226+0000", 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010152100000000001 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.957319+0000", 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2567999999999999e-05 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962235+0000", 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0049163360000000003 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.965210+0000", 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029749009999999998 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.980853+0000", 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01564337 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.980892+0000", 2023-10-09T04:18:06.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8631000000000003e-05 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.981069+0000", 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017751999999999999 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183900 7.15 7:ae7020d7:::benchmark_data_smithi163_346082_object90630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.221 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.466438+0000", 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.688828699, 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.516501388, 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183900 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466438+0000", 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466438+0000", 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466437+0000", 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466441+0000", 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4549999999999997e-06 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466451+0000", 2023-10-09T04:18:06.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0132999999999999e-05 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466456+0000", 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4349999999999999e-06 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466517+0000", 2023-10-09T04:18:06.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1463000000000004e-05 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.466527+0000", 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.611e-06 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771575+0000", 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.30504851300000002 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771580+0000", 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5990000000000004e-06 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.958968+0000", 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1873877130000001 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.959097+0000", 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012985399999999999 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.959185+0000", 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7651999999999994e-05 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962240+0000", 2023-10-09T04:18:06.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030553860000000002 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970571+0000", 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0083302210000000005 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982727+0000", 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012156063999999999 2023-10-09T04:18:06.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982761+0000", 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4279999999999997e-05 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982939+0000", 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000177949 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.230 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183916 7.15 7:aa53daba:::benchmark_data_smithi163_346082_object90646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.490223+0000", 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.66504380700002, 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.49778655, 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.231 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183916 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490223+0000", 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490223+0000", 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490221+0000", 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490228+0000", 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0389999999999999e-06 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490241+0000", 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2405000000000001e-05 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490247+0000", 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490553+0000", 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030512600000000002 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.490562+0000", 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1470000000000007e-06 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771600+0000", 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28103856500000002 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771604+0000", 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7799999999999998e-06 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.960704+0000", 2023-10-09T04:18:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1890995689999999 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.960801+0000", 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7596999999999994e-05 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.960888+0000", 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6960999999999996e-05 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970494+0000", 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096060009999999994 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.975172+0000", 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004678237 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.987718+0000", 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012545581 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.987756+0000", 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8099000000000002e-05 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.988009+0000", 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00025295599999999998 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183920 7.15 7:af24fcdf:::benchmark_data_smithi163_346082_object90650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.499807+0000", 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.65545951899998, 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4950453930000001, 2023-10-09T04:18:06.240 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183920 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499807+0000", 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499807+0000", 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499805+0000", 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499815+0000", 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4890000000000001e-06 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499829+0000", 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3844e-05 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499834+0000", 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499888+0000", 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4580999999999998e-05 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.499898+0000", 2023-10-09T04:18:06.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1069999999999994e-06 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771622+0000", 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.271724931 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771626+0000", 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.715e-06 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962279+0000", 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.190652845 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962422+0000", 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142559 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.962512+0000", 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0729000000000005e-05 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970500+0000", 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079872269999999995 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982615+0000", 2023-10-09T04:18:06.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012115791000000001 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.994509+0000", 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011893655 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.994570+0000", 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1307000000000001e-05 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.994852+0000", 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028187200000000001 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183947 7.15 7:afcfd3ea:::benchmark_data_smithi163_346082_object90677:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.545580+0000", 2023-10-09T04:18:06.249 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.609686297, 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.449627177, 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183947 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545580+0000", 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545580+0000", 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545578+0000", 2023-10-09T04:18:06.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545585+0000", 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8260000000000004e-06 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545599+0000", 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3356000000000001e-05 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545604+0000", 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3750000000000002e-06 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545656+0000", 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2482000000000003e-05 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.545667+0000", 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0186999999999999e-05 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771645+0000", 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.225977923 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771648+0000", 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6720000000000002e-06 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.963774+0000", 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1921258779999999 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.963891+0000", 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011660899999999999 2023-10-09T04:18:06.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.963982+0000", 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1420000000000002e-05 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970506+0000", 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0065237869999999996 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982657+0000", 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012150644 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.994912+0000", 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012255492 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.994969+0000", 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6740999999999998e-05 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.995207+0000", 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000238416 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183966 7.15 7:ab233976:::benchmark_data_smithi163_346082_object90696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.584556+0000", 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.57071067999999, 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.419087161, 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.259 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183966 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584556+0000", 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584556+0000", 2023-10-09T04:18:06.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584554+0000", 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584565+0000", 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0933e-05 2023-10-09T04:18:06.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584592+0000", 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7265999999999999e-05 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584604+0000", 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2449999999999999e-05 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584726+0000", 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012157100000000001 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.584746+0000", 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.003e-05 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771666+0000", 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18692030000000001 2023-10-09T04:18:06.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771670+0000", 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4479999999999999e-06 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.965249+0000", 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1935790589999999 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.965349+0000", 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9970000000000007e-05 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.965435+0000", 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6544000000000004e-05 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.972011+0000", 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.006575371 2023-10-09T04:18:06.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998064+0000", 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026053702000000001 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.003300+0000", 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052357769999999996 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.003341+0000", 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0413000000000001e-05 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.003643+0000", 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030229699999999998 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:183990 7.15 7:ae3bfad7:::benchmark_data_smithi163_346082_object90720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.618616+0000", 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.53665064099999, 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4115554210000001, 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 183990 2023-10-09T04:18:06.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618616+0000", 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618616+0000", 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618615+0000", 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618620+0000", 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.203e-06 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618630+0000", 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0159999999999999e-05 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618636+0000", 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0909999999999998e-06 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618691+0000", 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5226999999999998e-05 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.618700+0000", 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0620000000000007e-06 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771691+0000", 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.152990865 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771695+0000", 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6009999999999999e-06 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.966673+0000", 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1949785129999999 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.966812+0000", 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013845599999999999 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.966925+0000", 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011319600000000001 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.972017+0000", 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050919040000000004 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998112+0000", 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026094831999999998 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.029715+0000", 2023-10-09T04:18:06.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.031602793999999997 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.029777+0000", 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2302000000000001e-05 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.030171+0000", 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00039426000000000001 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.276 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184054 7.15 7:afdf5533:::benchmark_data_smithi163_346082_object90784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.744820+0000", 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.41044628200001, 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2954266409999999, 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.277 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184054 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744820+0000", 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744820+0000", 2023-10-09T04:18:06.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744819+0000", 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744826+0000", 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6479999999999997e-06 2023-10-09T04:18:06.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744841+0000", 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5829e-05 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744850+0000", 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9709999999999993e-06 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744909+0000", 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8872000000000001e-05 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.744917+0000", 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4760000000000001e-06 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771707+0000", 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.026790609999999999 2023-10-09T04:18:06.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.771711+0000", 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5410000000000001e-06 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.968443+0000", 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.196731808 2023-10-09T04:18:06.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.968607+0000", 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000164566 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.968763+0000", 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015533699999999999 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.975107+0000", 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0063445619999999998 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998135+0000", 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023027506 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.039922+0000", 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.041787589 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.039971+0000", 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8380000000000001e-05 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040247+0000", 2023-10-09T04:18:06.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00027606799999999997 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184087 7.15 7:aec86aa6:::benchmark_data_smithi163_346082_object90817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.811186+0000", 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.34407991299997, 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.229263182, 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.286 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184087 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811186+0000", 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811186+0000", 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811185+0000", 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811192+0000", 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1239999999999999e-06 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811208+0000", 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.613e-05 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811215+0000", 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7299999999999999e-06 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811265+0000", 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0328999999999999e-05 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.811272+0000", 2023-10-09T04:18:06.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1099999999999997e-06 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970604+0000", 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1593321569999999 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970702+0000", 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7391000000000003e-05 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.970786+0000", 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3897e-05 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.977272+0000", 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064860569999999999 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998176+0000", 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.020904750999999999 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040285+0000", 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.042108963999999999 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040317+0000", 2023-10-09T04:18:06.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1479999999999997e-05 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040450+0000", 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132822 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.294 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184093 7.15 7:a847d1d6:::benchmark_data_smithi163_346082_object90823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.821768+0000", 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.33349819599999, 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2189133750000001, 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184093 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.295 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821768+0000", 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821768+0000", 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821767+0000", 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821775+0000", 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8310000000000001e-06 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821786+0000", 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1079e-05 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821791+0000", 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5929999999999999e-06 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821835+0000", 2023-10-09T04:18:06.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4277000000000003e-05 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.821843+0000", 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5499999999999995e-06 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.972063+0000", 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.15021956 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.972191+0000", 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012793700000000001 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.972283+0000", 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2136e-05 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.977279+0000", 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0049958579999999997 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998198+0000", 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.020919172 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040497+0000", 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.042298918999999997 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040529+0000", 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1936000000000001e-05 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040682+0000", 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152597 2023-10-09T04:18:06.302 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184110 7.15 7:aea4b8a5:::benchmark_data_smithi163_346082_object90840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.851676+0000", 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.30359002199998, 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1892785969999999, 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.303 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184110 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851676+0000", 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851676+0000", 2023-10-09T04:18:06.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851675+0000", 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851680+0000", 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000004e-06 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851690+0000", 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0461e-05 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851694+0000", 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1529999999999999e-06 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851756+0000", 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1156999999999997e-05 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.851764+0000", 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7020000000000007e-06 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.973594+0000", 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1218300919999999 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.973696+0000", 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101652 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.973784+0000", 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7795999999999998e-05 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.981082+0000", 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0072982289999999998 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998241+0000", 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017158797999999999 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040801+0000", 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.042560424999999999 2023-10-09T04:18:06.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040835+0000", 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3599999999999997e-05 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040955+0000", 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120062 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.310 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184128 7.15 7:aebda5a1:::benchmark_data_smithi163_346082_object90858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.909673+0000", 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.24559353000001, 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.131538232, 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.311 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184128 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909673+0000", 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909673+0000", 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909671+0000", 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909681+0000", 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0360000000000001e-05 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909700+0000", 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9230000000000001e-05 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909709+0000", 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.687e-06 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.314 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909760+0000", 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0748000000000003e-05 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.909768+0000", 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2390000000000002e-06 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.975218+0000", 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0654500840000001 2023-10-09T04:18:06.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.975370+0000", 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152106 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.975531+0000", 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016097799999999999 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982548+0000", 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0070167299999999997 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998270+0000", 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015721557000000001 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.040989+0000", 2023-10-09T04:18:06.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.042719762000000001 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.041031+0000", 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1381999999999998e-05 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.041211+0000", 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018031699999999999 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.318 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184141 7.15 7:aeef34f5:::benchmark_data_smithi163_346082_object90871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.941230+0000", 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.21403660200002, 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1029498879999999, 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.319 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184141 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941230+0000", 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941230+0000", 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941228+0000", 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941235+0000", 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3160000000000002e-06 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941247+0000", 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1901e-05 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941252+0000", 2023-10-09T04:18:06.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.716e-06 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941306+0000", 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3711999999999997e-05 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.941316+0000", 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0026e-05 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.977325+0000", 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0360091220000001 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.977464+0000", 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138691 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.977557+0000", 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3351999999999997e-05 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982558+0000", 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050009750000000004 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998312+0000", 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015753471000000002 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.043911+0000", 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.045599558999999998 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.043946+0000", 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4561000000000003e-05 2023-10-09T04:18:06.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.044180+0000", 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023384100000000001 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:184155 7.15 7:aaad25c0:::benchmark_data_smithi163_346082_object90885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e234)", 2023-10-09T04:18:06.327 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:13:10.993842+0000", 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 295.16142432300001, 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.050600719, 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 184155 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993842+0000", 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993842+0000", 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:18:06.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993841+0000", 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993847+0000", 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.686e-06 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993862+0000", 2023-10-09T04:18:06.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4814000000000001e-05 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993867+0000", 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1800000000000004e-06 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993910+0000", 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2750999999999998e-05 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:10.993917+0000", 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1509999999999998e-06 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.979039+0000", 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98512127400000005 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:18:06.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.979161+0000", 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012213 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.979248+0000", 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7266000000000001e-05 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.982567+0000", 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033188950000000001 2023-10-09T04:18:06.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:11.998333+0000", 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015765858000000001 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.044238+0000", 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.045905552000000002 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.044290+0000", 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1357000000000003e-05 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:13:12.044443+0000", 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015314699999999999 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:18:06.335 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:18:06.336 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:18:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:06.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:06.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:06.409 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:06.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:06.467 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:06.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:06.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:06.620 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:06.648 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:06.674 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_3' created 2023-10-09T04:18:06.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:06.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:06.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:18:06.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:06.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/214144338' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:18:06.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:06 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:18:06.871 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:06 smithi163 ceph-mon[221067]: pgmap v836: 103 pgs: 19 remapped+peering, 2 peering, 82 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 96 KiB/s rd, 257 KiB/s wr, 75 op/s; 184 KiB/s, 0 objects/s recovering 2023-10-09T04:18:06.871 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/214144338' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:18:06.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:06 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:18:06.872 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:06 smithi163 ceph-mon[213841]: pgmap v836: 103 pgs: 19 remapped+peering, 2 peering, 82 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 96 KiB/s rd, 257 KiB/s wr, 75 op/s; 184 KiB/s, 0 objects/s recovering 2023-10-09T04:18:06.874 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/214144338' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:18:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:06 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:18:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:06 smithi181 ceph-mon[196979]: pgmap v836: 103 pgs: 19 remapped+peering, 2 peering, 82 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 96 KiB/s rd, 257 KiB/s wr, 75 op/s; 184 KiB/s, 0 objects/s recovering 2023-10-09T04:18:07.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:07.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:07.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:07.159 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:07.552 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:07.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:07.656 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:07.674 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:18:07.697 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:07.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:07.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:07.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[221067]: failed to encode map e306 with expected crc 2023-10-09T04:18:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[221067]: failed to encode map e306 with expected crc 2023-10-09T04:18:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:18:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[221067]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T04:18:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2764276589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[213841]: failed to encode map e306 with expected crc 2023-10-09T04:18:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[213841]: failed to encode map e306 with expected crc 2023-10-09T04:18:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:18:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[213841]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T04:18:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2764276589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:07.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:07.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:07.878 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:07.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:07.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:07 smithi181 ceph-mon[196979]: failed to encode map e306 with expected crc 2023-10-09T04:18:07.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:07 smithi181 ceph-mon[196979]: failed to encode map e306 with expected crc 2023-10-09T04:18:07.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:18:07.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:07 smithi181 ceph-mon[196979]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T04:18:07.927 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2764276589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:07.928 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:07.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:08.030 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:08.056 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:08.098 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:08.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:08.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T04:18:08.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:08.271 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:08.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:08.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:08.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:08.576 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:08.592 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:08.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:08.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:08.726 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:08.740 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:08.753 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:08.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:08.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: osdmap e307: 8 total, 8 up, 7 in 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: pgmap v839: 119 pgs: 12 unknown, 4 creating+peering, 5 remapped+peering, 2 peering, 96 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 140 KiB/s, 0 objects/s recovering 2023-10-09T04:18:08.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[213841]: osdmap e308: 8 total, 8 up, 7 in 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: osdmap e307: 8 total, 8 up, 7 in 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: failed to encode map e307 with expected crc 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: pgmap v839: 119 pgs: 12 unknown, 4 creating+peering, 5 remapped+peering, 2 peering, 96 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 140 KiB/s, 0 objects/s recovering 2023-10-09T04:18:08.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:08 smithi163 ceph-mon[221067]: osdmap e308: 8 total, 8 up, 7 in 2023-10-09T04:18:08.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:08.932 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: osdmap e307: 8 total, 8 up, 7 in 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: failed to encode map e307 with expected crc 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: failed to encode map e307 with expected crc 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: failed to encode map e307 with expected crc 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: failed to encode map e307 with expected crc 2023-10-09T04:18:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: pgmap v839: 119 pgs: 12 unknown, 4 creating+peering, 5 remapped+peering, 2 peering, 96 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 140 KiB/s, 0 objects/s recovering 2023-10-09T04:18:09.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:08 smithi181 ceph-mon[196979]: osdmap e308: 8 total, 8 up, 7 in 2023-10-09T04:18:09.582 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:09.591 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:09.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:09.627 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:09.646 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:09.665 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:09.684 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:09.702 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.705 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:18:09.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:09.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:09.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[221067]: failed to encode map e308 with expected crc 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[221067]: Health check update: Reduced data availability: 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[221067]: failed to encode map e308 with expected crc 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2268422584' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[213841]: failed to encode map e308 with expected crc 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[213841]: Health check update: Reduced data availability: 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[213841]: failed to encode map e308 with expected crc 2023-10-09T04:18:09.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2268422584' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:09.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:10.009 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:09 smithi181 ceph-mon[196979]: failed to encode map e308 with expected crc 2023-10-09T04:18:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:09 smithi181 ceph-mon[196979]: Health check update: Reduced data availability: 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:18:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:09 smithi181 ceph-mon[196979]: failed to encode map e308 with expected crc 2023-10-09T04:18:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2268422584' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:10.039 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 6, 7, 4] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:18:10.039 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:18:10.039 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T04:18:10.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T04:18:10.560 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:18:10.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:10.655 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:18:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:18:10] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2023-10-09T04:18:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:10.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:10.757 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:10.790 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:10.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:10.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:10.957 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:10 smithi163 ceph-mon[213841]: pgmap v841: 119 pgs: 9 unknown, 4 creating+peering, 106 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2023-10-09T04:18:10.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:10 smithi163 ceph-mon[221067]: pgmap v841: 119 pgs: 9 unknown, 4 creating+peering, 106 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2023-10-09T04:18:10.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:11.017 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:10 smithi181 ceph-mon[196979]: pgmap v841: 119 pgs: 9 unknown, 4 creating+peering, 106 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2023-10-09T04:18:11.020 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:11.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:11.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:11.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:11.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:11.205 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:11.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:11.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:11.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:11.271 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:18:11.271 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:18:11.356 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:11.382 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:11.423 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:11.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:11.487 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:11.529 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:11.555 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:11.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:11.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:11.702 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T04:18:11.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:11.728 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:11.770 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:11.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:11.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:11.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:11.903 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:11.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:11.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:11 smithi181 ceph-mon[196979]: failed to encode map e308 with expected crc 2023-10-09T04:18:11.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:11 smithi181 ceph-mon[196979]: failed to encode map e308 with expected crc 2023-10-09T04:18:11.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:11 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs peering) 2023-10-09T04:18:11.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3467103682' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:11.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:11 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:11.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:11.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:12.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[213841]: failed to encode map e308 with expected crc 2023-10-09T04:18:12.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[213841]: failed to encode map e308 with expected crc 2023-10-09T04:18:12.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs peering) 2023-10-09T04:18:12.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3467103682' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:12.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:12.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[221067]: failed to encode map e308 with expected crc 2023-10-09T04:18:12.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[221067]: failed to encode map e308 with expected crc 2023-10-09T04:18:12.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs peering) 2023-10-09T04:18:12.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3467103682' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:12.018 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:11 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:12.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:12.078 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:12.119 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:12.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:12.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:12.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:18:12.260 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:12.311 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:12.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:12.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:12.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:12.444 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:12.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:12.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:12.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:12.620 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:12.662 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:12.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:12.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:12.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:12.798 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:12.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:18:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:12 smithi181 ceph-mon[196979]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:12 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:18:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:12 smithi181 ceph-mon[196979]: osdmap e309: 8 total, 8 up, 7 in 2023-10-09T04:18:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:12 smithi181 ceph-mon[196979]: pgmap v843: 119 pgs: 119 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-09T04:18:13.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:12 smithi181 ceph-mon[196979]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[213841]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:18:13.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[213841]: osdmap e309: 8 total, 8 up, 7 in 2023-10-09T04:18:13.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[213841]: pgmap v843: 119 pgs: 119 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-09T04:18:13.050 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[213841]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[221067]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:18:13.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[221067]: osdmap e309: 8 total, 8 up, 7 in 2023-10-09T04:18:13.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[221067]: pgmap v843: 119 pgs: 119 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-09T04:18:13.051 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:12 smithi163 ceph-mon[221067]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:13.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:13.522 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:13.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:13.631 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:13.675 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:13.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:13.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:13.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:13.816 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:13.972 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:13 smithi163 ceph-mon[221067]: failed to encode map e309 with expected crc 2023-10-09T04:18:13.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:13 smithi163 ceph-mon[213841]: failed to encode map e309 with expected crc 2023-10-09T04:18:14.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:13 smithi181 ceph-mon[196979]: failed to encode map e309 with expected crc 2023-10-09T04:18:14.118 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:14.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:14.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:14.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:14.253 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:14.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:14.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:14.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:14.439 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:14.461 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T04:18:14.464 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:18:14.464 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:18:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:14 smithi163 conmon[213817]: 2023-10-09T04:18:14.506+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e309 definitely_dead 0 2023-10-09T04:18:14.675 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:14.733 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.3. 2023-10-09T04:18:14.824 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T04:18:14.824 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:18:14.824 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_464650 2023-10-09T04:18:14.825 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:18:14.825 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:18:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:14 smithi181 ceph-mon[196979]: pgmap v844: 119 pgs: 119 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T04:18:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1914239656' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:18:15.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:14 smithi163 ceph-mon[221067]: pgmap v844: 119 pgs: 119 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T04:18:15.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1914239656' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:18:15.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:14 smithi163 ceph-mon[213841]: pgmap v844: 119 pgs: 119 active+clean; 529 KiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-10-09T04:18:15.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1914239656' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:18:15.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:15.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:15.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:15.802 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:15.824 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 236 220 13.7494 13.75 0.00152789 0.0718802 2023-10-09T04:18:15.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:15.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:15.909 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1914239656' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1914239656' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: failed to encode map e310 with expected crc 2023-10-09T04:18:15.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:15 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:15.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:16.004 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: failed to encode map e310 with expected crc 2023-10-09T04:18:16.004 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: failed to encode map e310 with expected crc 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: failed to encode map e310 with expected crc 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: failed to encode map e310 with expected crc 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1914239656' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: osdmap e310: 8 total, 8 up, 8 in 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: failed to encode map e310 with expected crc 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: failed to encode map e310 with expected crc 2023-10-09T04:18:16.005 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:15 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:16.007 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:16.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:16.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:16.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:16.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:16.264 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:16.278 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T04:18:16.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:16.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:16.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:16.563 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:16.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:16.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:16.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:16.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:16.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T04:18:16.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: failed to encode map e311 with expected crc 2023-10-09T04:18:16.813 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:16 smithi181 ceph-mon[196979]: pgmap v847: 119 pgs: 1 remapped+peering, 7 peering, 2 active+recovery_wait+degraded, 109 active+clean; 1.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 149 KiB/s wr, 2 op/s; 3/114 objects degraded (2.632%) 2023-10-09T04:18:16.816 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:16.824 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 3936 3920 122.492 231.25 0.00266473 0.00813766 2023-10-09T04:18:16.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:16.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:16.967 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[221067]: pgmap v847: 119 pgs: 1 remapped+peering, 7 peering, 2 active+recovery_wait+degraded, 109 active+clean; 1.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 149 KiB/s wr, 2 op/s; 3/114 objects degraded (2.632%) 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: osdmap e311: 8 total, 8 up, 8 in 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.006 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: failed to encode map e311 with expected crc 2023-10-09T04:18:17.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:16 smithi163 ceph-mon[213841]: pgmap v847: 119 pgs: 1 remapped+peering, 7 peering, 2 active+recovery_wait+degraded, 109 active+clean; 1.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 149 KiB/s wr, 2 op/s; 3/114 objects degraded (2.632%) 2023-10-09T04:18:17.007 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:17.467 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:17.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:17.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:17.612 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:17.693 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:17.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:17.718 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:18:17.719 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 3/114 objects degraded (2.632%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[213841]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T04:18:17.740 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 3/114 objects degraded (2.632%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.741 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: failed to encode map e312 with expected crc 2023-10-09T04:18:17.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:17 smithi163 ceph-mon[221067]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T04:18:17.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:17.825 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 6479 6463 134.619 158.938 0.00370755 0.00741849 2023-10-09T04:18:17.870 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:17.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:17.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:17.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 3/114 objects degraded (2.632%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:18:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: osdmap e312: 8 total, 8 up, 8 in 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: failed to encode map e312 with expected crc 2023-10-09T04:18:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:17 smithi181 ceph-mon[196979]: osdmap e313: 8 total, 8 up, 8 in 2023-10-09T04:18:18.080 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:18.130 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:18.516 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:18:18.563 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:18.564 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:18.571 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:18.610 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:18.623 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:18.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:18.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:18.648 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:18.682 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:18.721 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:18.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:18.757 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:18:18.762 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.762 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.762 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.762 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.763 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:18:18.768 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:18.825 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 9362 9346 145.994 180.188 0.00274936 0.00684044 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: pgmap v850: 119 pgs: 6 remapped+peering, 7 peering, 2 active+recovery_wait+degraded, 104 active+clean; 60 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 235 op/s; 3/2904 objects degraded (0.103%); 0 B/s, 1 objects/s recovering 2023-10-09T04:18:18.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3951394110' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1967614275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3951394110' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[213841]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: failed to encode map e313 with expected crc 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: pgmap v850: 119 pgs: 6 remapped+peering, 7 peering, 2 active+recovery_wait+degraded, 104 active+clean; 60 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 235 op/s; 3/2904 objects degraded (0.103%); 0 B/s, 1 objects/s recovering 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3951394110' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1967614275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3951394110' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:18:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:18 smithi163 ceph-mon[221067]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T04:18:18.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:18.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:18.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: failed to encode map e313 with expected crc 2023-10-09T04:18:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: failed to encode map e313 with expected crc 2023-10-09T04:18:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: failed to encode map e313 with expected crc 2023-10-09T04:18:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: failed to encode map e313 with expected crc 2023-10-09T04:18:19.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: failed to encode map e313 with expected crc 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: failed to encode map e313 with expected crc 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: pgmap v850: 119 pgs: 6 remapped+peering, 7 peering, 2 active+recovery_wait+degraded, 104 active+clean; 60 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 235 op/s; 3/2904 objects degraded (0.103%); 0 B/s, 1 objects/s recovering 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3951394110' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1967614275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3951394110' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:18:19.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:18 smithi181 ceph-mon[196979]: osdmap e314: 8 total, 8 up, 8 in 2023-10-09T04:18:19.024 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:19.085 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:19.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:19.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:19.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:19.241 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:19.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:19.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:19.458 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:19.513 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:19.751 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.751 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:19.808 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.809 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.810 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.810 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: failed to encode map e314 with expected crc 2023-10-09T04:18:19.810 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:19.810 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:19.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:19.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:19.826 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 12198 12182 152.235 177.25 0.00297759 0.00656254 2023-10-09T04:18:19.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:19.961 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: failed to encode map e314 with expected crc 2023-10-09T04:18:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: failed to encode map e314 with expected crc 2023-10-09T04:18:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: failed to encode map e314 with expected crc 2023-10-09T04:18:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:20.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:20.355 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:20.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:20.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:20.509 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:20.561 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:20.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:20.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:20.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:20.721 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:20.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:20.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:20.826 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 14024 14008 145.875 114.125 0.105197 0.00678867 2023-10-09T04:18:20.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:20.900 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:20.974 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:20.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:20.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:20 smithi181 ceph-mon[196979]: 1.1e scrub starts 2023-10-09T04:18:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:20 smithi181 ceph-mon[196979]: 1.1e scrub ok 2023-10-09T04:18:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:20 smithi181 ceph-mon[196979]: 1.13 scrub starts 2023-10-09T04:18:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:20 smithi181 ceph-mon[196979]: 1.13 scrub ok 2023-10-09T04:18:21.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:20 smithi181 ceph-mon[196979]: pgmap v852: 119 pgs: 6 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 103 active+clean; 146 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s; 3/7071 objects degraded (0.042%); 0 B/s, 1 objects/s recovering 2023-10-09T04:18:21.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[221067]: 1.1e scrub starts 2023-10-09T04:18:21.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[221067]: 1.1e scrub ok 2023-10-09T04:18:21.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[221067]: 1.13 scrub starts 2023-10-09T04:18:21.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[221067]: 1.13 scrub ok 2023-10-09T04:18:21.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[221067]: pgmap v852: 119 pgs: 6 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 103 active+clean; 146 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s; 3/7071 objects degraded (0.042%); 0 B/s, 1 objects/s recovering 2023-10-09T04:18:21.032 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:18:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:18:20] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-10-09T04:18:21.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:18:21.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[213841]: 1.1e scrub starts 2023-10-09T04:18:21.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[213841]: 1.1e scrub ok 2023-10-09T04:18:21.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[213841]: 1.13 scrub starts 2023-10-09T04:18:21.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[213841]: 1.13 scrub ok 2023-10-09T04:18:21.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:20 smithi163 ceph-mon[213841]: pgmap v852: 119 pgs: 6 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 103 active+clean; 146 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s; 3/7071 objects degraded (0.042%); 0 B/s, 1 objects/s recovering 2023-10-09T04:18:21.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:21.149 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:21.278 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 6, 7, 4, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:18:21.278 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:18:21.279 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 5, 6, 7, 4, 3] 2023-10-09T04:18:21.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T04:18:21.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:21.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:21.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:21.472 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:21.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:21.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:21.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:21.713 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:21.744 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:21.790 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:21.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:21.828 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 16309 16293 145.395 142.812 0.00389992 0.0068717 2023-10-09T04:18:21.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:21.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[213841]: 1.14 scrub starts 2023-10-09T04:18:21.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[213841]: 1.14 scrub ok 2023-10-09T04:18:21.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[213841]: 1.4 scrub starts 2023-10-09T04:18:21.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[213841]: 1.4 scrub ok 2023-10-09T04:18:21.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[213841]: 1.b scrub starts 2023-10-09T04:18:21.952 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[213841]: 1.b scrub ok 2023-10-09T04:18:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[221067]: 1.14 scrub starts 2023-10-09T04:18:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[221067]: 1.14 scrub ok 2023-10-09T04:18:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[221067]: 1.4 scrub starts 2023-10-09T04:18:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[221067]: 1.4 scrub ok 2023-10-09T04:18:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[221067]: 1.b scrub starts 2023-10-09T04:18:21.954 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:21 smithi163 ceph-mon[221067]: 1.b scrub ok 2023-10-09T04:18:21.955 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:21 smithi181 ceph-mon[196979]: 1.14 scrub starts 2023-10-09T04:18:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:21 smithi181 ceph-mon[196979]: 1.14 scrub ok 2023-10-09T04:18:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:21 smithi181 ceph-mon[196979]: 1.4 scrub starts 2023-10-09T04:18:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:21 smithi181 ceph-mon[196979]: 1.4 scrub ok 2023-10-09T04:18:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:21 smithi181 ceph-mon[196979]: 1.b scrub starts 2023-10-09T04:18:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:21 smithi181 ceph-mon[196979]: 1.b scrub ok 2023-10-09T04:18:22.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:22.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:22.165 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:22.222 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:22.253 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:22.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:22.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:22.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:22.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:22.566 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:22.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:22.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:22.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:22.822 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:22.828 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 18876 18860 147.273 160.438 0.00147145 0.00677644 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: 1.8 scrub starts 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: 1.8 scrub ok 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: 1.3 scrub starts 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: 1.3 scrub ok 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: pgmap v853: 119 pgs: 1 peering, 118 active+clean; 830 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 38 KiB/s, 1 objects/s recovering 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: 1.1c scrub starts 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: 1.1c scrub ok 2023-10-09T04:18:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:22 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3/7071 objects degraded (0.042%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: 1.8 scrub starts 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: 1.8 scrub ok 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: 1.3 scrub starts 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: 1.3 scrub ok 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: pgmap v853: 119 pgs: 1 peering, 118 active+clean; 830 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 38 KiB/s, 1 objects/s recovering 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: 1.1c scrub starts 2023-10-09T04:18:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: 1.1c scrub ok 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3/7071 objects degraded (0.042%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: 1.8 scrub starts 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: 1.8 scrub ok 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: 1.3 scrub starts 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: 1.3 scrub ok 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: pgmap v853: 119 pgs: 1 peering, 118 active+clean; 830 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 38 KiB/s, 1 objects/s recovering 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: 1.1c scrub starts 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: 1.1c scrub ok 2023-10-09T04:18:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:22 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3/7071 objects degraded (0.042%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:18:23.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:18:23.304 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:18:23.482 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:23.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:23.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:23.631 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:23.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:23.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:23.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:23.828 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 20483 20467 142.07 100.438 0.0126035 0.00703274 2023-10-09T04:18:23.877 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:24.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:23 smithi181 ceph-mon[196979]: 1.7 scrub starts 2023-10-09T04:18:24.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:23 smithi181 ceph-mon[196979]: 1.7 scrub ok 2023-10-09T04:18:24.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:23 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/7071 objects degraded (0.042%), 2 pgs degraded) 2023-10-09T04:18:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:23 smithi163 ceph-mon[213841]: 1.7 scrub starts 2023-10-09T04:18:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:23 smithi163 ceph-mon[213841]: 1.7 scrub ok 2023-10-09T04:18:24.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:23 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/7071 objects degraded (0.042%), 2 pgs degraded) 2023-10-09T04:18:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:23 smithi163 ceph-mon[221067]: 1.7 scrub starts 2023-10-09T04:18:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:23 smithi163 ceph-mon[221067]: 1.7 scrub ok 2023-10-09T04:18:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:23 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/7071 objects degraded (0.042%), 2 pgs degraded) 2023-10-09T04:18:24.688 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:18:24.688 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:18:24.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80549 7.12 7:4a26220b:::benchmark_data_smithi163_346082_object80548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 466.20824201099998, 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5311827440000001, 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.689 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80549 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470415+0000", 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470446+0000", 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0809000000000003e-05 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470453+0000", 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6939999999999995e-06 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470459+0000", 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8830000000000003e-06 2023-10-09T04:18:24.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470525+0000", 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5624999999999996e-05 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.470537+0000", 2023-10-09T04:18:24.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1918e-05 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.968283+0000", 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4977461619999999 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.972146+0000", 2023-10-09T04:18:24.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038629369999999999 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.978159+0000", 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060134689999999996 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.994408+0000", 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016248729999999999 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.995092+0000", 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068414799999999998 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.995313+0000", 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000220638 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.996444+0000", 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011307999999999999 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.001598+0000", 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051543429999999996 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80579 7.12 7:4e07a344:::benchmark_data_smithi163_346082_object80578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:18:24.699 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 466.158412752, 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4902949560000001, 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80579 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.700 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520245+0000", 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520243+0000", 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520271+0000", 2023-10-09T04:18:24.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8158999999999999e-05 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.520288+0000", 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7629999999999999e-05 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523866+0000", 2023-10-09T04:18:24.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035770730000000001 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524028+0000", 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016236000000000001 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524050+0000", 2023-10-09T04:18:24.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1882000000000002e-05 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.982571+0000", 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.458521492 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.985387+0000", 2023-10-09T04:18:24.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028160400000000001 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:41.989941+0000", 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045534929999999996 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.001841+0000", 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011900571 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.002750+0000", 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090909300000000003 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.003526+0000", 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00077512299999999996 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.004995+0000", 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001469143 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.010540+0000", 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055447580000000003 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80582 7.12 7:4a340bf1:::benchmark_data_smithi163_346082_object80581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:18:24.709 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 466.15473924299999, 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5356164219999999, 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80582 2023-10-09T04:18:24.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523918+0000", 2023-10-09T04:18:24.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523917+0000", 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523945+0000", 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7841999999999999e-05 2023-10-09T04:18:24.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523957+0000", 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2449000000000001e-05 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.523974+0000", 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7101000000000001e-05 2023-10-09T04:18:24.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524082+0000", 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107395 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524090+0000", 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8189999999999998e-06 2023-10-09T04:18:24.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.010809+0000", 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4867180810000002 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.014231+0000", 2023-10-09T04:18:24.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003422043 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.019765+0000", 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055343019999999996 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.037504+0000", 2023-10-09T04:18:24.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017739296000000002 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.038374+0000", 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086940900000000002 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.052524+0000", 2023-10-09T04:18:24.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014150563999999999 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.053699+0000", 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011743890000000001 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.059534+0000", 2023-10-09T04:18:24.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058359709999999997 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80583 7.12 7:4f777d92:::benchmark_data_smithi163_346082_object80582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 466.15434871600002, 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5426957880000001, 2023-10-09T04:18:24.719 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80583 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524309+0000", 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524308+0000", 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524363+0000", 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5349999999999997e-05 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524377+0000", 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4075e-05 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524384+0000", 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1230000000000002e-06 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524440+0000", 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5284999999999997e-05 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:38.524453+0000", 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3614999999999999e-05 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.024073+0000", 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.499619434 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.027624+0000", 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003551102 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.032819+0000", 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051955509999999996 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.039280+0000", 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064610379999999997 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.052274+0000", 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012993754 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.059828+0000", 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075536780000000003 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.061217+0000", 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001389033 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.067004+0000", 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005787665 2023-10-09T04:18:24.728 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80604 7.12 7:4ae32c24:::benchmark_data_smithi163_346082_object80603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 465.24899617, 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.646247169, 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.729 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80604 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429661+0000", 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429660+0000", 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429684+0000", 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4735e-05 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429700+0000", 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6118999999999999e-05 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.429710+0000", 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4110000000000002e-06 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.039654+0000", 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6099443359999999 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.042687+0000", 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030327050000000001 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.048100+0000", 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054131359999999998 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.067276+0000", 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019175580000000001 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.068101+0000", 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082490899999999997 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.068965+0000", 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086460499999999995 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.070415+0000", 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001449547 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.075908+0000", 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054935929999999997 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.737 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80621 7.12 7:4a7d6f10:::benchmark_data_smithi163_346082_object80620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 465.226104083, 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6714043699999999, 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.738 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80621 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:18:24.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452553+0000", 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452552+0000", 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452577+0000", 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5196000000000001e-05 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452588+0000", 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1482e-05 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.452595+0000", 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8430000000000004e-06 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.076258+0000", 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6236622060000001 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.079917+0000", 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003659425 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.085706+0000", 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057886170000000002 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.115020+0000", 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.029313949999999998 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.115996+0000", 2023-10-09T04:18:24.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00097686999999999991 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.116902+0000", 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090546300000000001 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.118150+0000", 2023-10-09T04:18:24.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001248377 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.123958+0000", 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058073669999999999 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80659 7.12 7:4da2b418:::benchmark_data_smithi163_346082_object80658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 465.18320640000002, 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.638010832, 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.747 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80659 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495451+0000", 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495449+0000", 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495472+0000", 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2648999999999999e-05 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495487+0000", 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4714e-05 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.495494+0000", 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6179999999999997e-06 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.089728+0000", 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5942332260000001 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.092886+0000", 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031585609999999998 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.098139+0000", 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052528790000000002 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.124279+0000", 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.026139367 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.125465+0000", 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011861739999999999 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.126393+0000", 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092876999999999999 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.127602+0000", 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012081450000000001 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.133462+0000", 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058601410000000001 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80699 7.12 7:49a7773c:::benchmark_data_smithi163_346082_object80698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:18:24.755 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 465.13659548999999, 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.601076436, 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80699 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542062+0000", 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542061+0000", 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542082+0000", 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0676999999999999e-05 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542089+0000", 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1969999999999999e-06 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.542095+0000", 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8889999999999999e-06 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.101752+0000", 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.559657235 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.105743+0000", 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039902699999999998 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.111032+0000", 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052895010000000003 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.124581+0000", 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013548701999999999 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.134215+0000", 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096345360000000008 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.135040+0000", 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00082435799999999997 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.136244+0000", 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00120403 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.143138+0000", 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006894608 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.763 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80726 7.12 7:4d882541:::benchmark_data_smithi163_346082_object80725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 465.09983229099998, 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6025676880000002, 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.764 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80726 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578825+0000", 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578824+0000", 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578846+0000", 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2328e-05 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578858+0000", 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1749999999999999e-05 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.578866+0000", 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.542e-06 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.143524+0000", 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.564657902 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.146910+0000", 2023-10-09T04:18:24.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033861049999999999 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.152321+0000", 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054116600000000004 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.172638+0000", 2023-10-09T04:18:24.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020316186999999999 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.172924+0000", 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028628699999999998 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.173994+0000", 2023-10-09T04:18:24.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010702839999999999 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.175304+0000", 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013095439999999999 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.181393+0000", 2023-10-09T04:18:24.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060889680000000002 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35115.0:80738 7.12 7:4f007b37:::benchmark_data_smithi163_346082_object80737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 465.07715040300002, 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5902670159999999, 2023-10-09T04:18:24.772 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35115", 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 80738 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601507+0000", 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601505+0000", 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601561+0000", 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6643e-05 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601585+0000", 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3541e-05 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:39.601597+0000", 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2396e-05 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.156776+0000", 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5551789820000002 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.160320+0000", 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035439960000000002 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.165689+0000", 2023-10-09T04:18:24.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053691839999999999 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.173228+0000", 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075388089999999996 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.182616+0000", 2023-10-09T04:18:24.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093880849999999991 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.183537+0000", 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00092063400000000004 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.185023+0000", 2023-10-09T04:18:24.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014860819999999999 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:10:42.191774+0000", 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067509279999999998 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:146186 8.6 8:66c3ba9a:::benchmark_data_smithi163_404719_object55602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:49.133555+0000", 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.545101959999997, 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26869774499999999, 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:18:24.781 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 146186 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.133555+0000", 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.133555+0000", 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.133554+0000", 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.133560+0000", 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6210000000000001e-06 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.133572+0000", 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1851000000000001e-05 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.133580+0000", 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9449999999999993e-06 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150604+0000", 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.017024589999999999 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150699+0000", 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4828999999999996e-05 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150793+0000", 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4063999999999999e-05 2023-10-09T04:18:24.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.395287+0000", 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.244493511 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.395356+0000", 2023-10-09T04:18:24.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9628000000000005e-05 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.402159+0000", 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0068022050000000004 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.402206+0000", 2023-10-09T04:18:24.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7697000000000001e-05 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.402253+0000", 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6703999999999999e-05 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.789 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:146199 8.e 8:7ed46b49:::benchmark_data_smithi163_404719_object55615:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:49.149991+0000", 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.528666603000005, 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25845849100000001, 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 146199 2023-10-09T04:18:24.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.149991+0000", 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.149991+0000", 2023-10-09T04:18:24.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.149990+0000", 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.149993+0000", 2023-10-09T04:18:24.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2579999999999998e-06 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.149998+0000", 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8520000000000003e-06 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150002+0000", 2023-10-09T04:18:24.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1670000000000001e-06 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.395539+0000", 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24553724499999999 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.395643+0000", 2023-10-09T04:18:24.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103645 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.395744+0000", 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101897 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.405618+0000", 2023-10-09T04:18:24.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0098735109999999997 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.405720+0000", 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101554 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.408359+0000", 2023-10-09T04:18:24.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026390570000000002 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.408407+0000", 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7988999999999997e-05 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.408449+0000", 2023-10-09T04:18:24.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2619e-05 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:18833 9.9 9:96dc207e:::benchmark_data_smithi163_464650_object18832:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.808524+0000", 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.8701334869999999, 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.296220751, 2023-10-09T04:18:24.798 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18833 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808524+0000", 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808524+0000", 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808523+0000", 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808548+0000", 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5106999999999999e-05 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808558+0000", 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0275000000000001e-05 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808570+0000", 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2065999999999999e-05 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808599+0000", 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9509000000000002e-05 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808732+0000", 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013303800000000001 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808861+0000", 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012874100000000001 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104566+0000", 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29570467900000003 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104608+0000", 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2481999999999997e-05 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104654+0000", 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6010000000000002e-05 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104705+0000", 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0529999999999999e-05 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104745+0000", 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9632000000000003e-05 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.806 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:18836 9.8 9:15b9b6f2:::benchmark_data_smithi163_464650_object18835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.808996+0000", 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.869661571, 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29789586200000001, 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:18:24.807 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18836 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808996+0000", 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808996+0000", 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.808995+0000", 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809019+0000", 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3856999999999999e-05 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809030+0000", 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1588e-05 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809038+0000", 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.52e-06 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809078+0000", 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0664999999999998e-05 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809205+0000", 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126952 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809342+0000", 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000136639 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104818+0000", 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29547595199999999 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104855+0000", 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7357999999999997e-05 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106813+0000", 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0019580230000000001 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106856+0000", 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2558000000000001e-05 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106892+0000", 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5707000000000002e-05 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:18839 9.8 9:10e4b68c:::benchmark_data_smithi163_464650_object18838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.809053+0000", 2023-10-09T04:18:24.815 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.869603948, 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29792777999999998, 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18839 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809053+0000", 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809053+0000", 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809053+0000", 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809096+0000", 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3593000000000001e-05 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809105+0000", 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7620000000000001e-06 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809111+0000", 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9309999999999996e-06 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103314+0000", 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29420306000000002 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103495+0000", 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018085599999999999 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:18:24.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103638+0000", 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000143335 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106762+0000", 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031235379999999999 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106802+0000", 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0497000000000002e-05 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106905+0000", 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102805 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106948+0000", 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3129e-05 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106981+0000", 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2863e-05 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.823 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:18840 9.5 9:a66cc69b:::benchmark_data_smithi163_464650_object18839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.809120+0000", 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.8695368299999999, 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29437070599999998, 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.824 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18840 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809120+0000", 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809120+0000", 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809120+0000", 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809146+0000", 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6222000000000001e-05 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809154+0000", 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6639999999999998e-06 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809159+0000", 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.604e-06 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809207+0000", 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7525000000000003e-05 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809343+0000", 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000136098 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809479+0000", 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013562599999999999 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103334+0000", 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29385482699999999 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103372+0000", 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8915999999999998e-05 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103405+0000", 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2759999999999998e-05 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103448+0000", 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2688999999999997e-05 2023-10-09T04:18:24.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103491+0000", 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3257000000000003e-05 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:18841 9.9 9:916a40f5:::benchmark_data_smithi163_464650_object18840:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:18:24.832 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.809173+0000", 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.86948392, 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29763753399999998, 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18841 2023-10-09T04:18:24.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809173+0000", 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809173+0000", 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809173+0000", 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809195+0000", 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2339000000000001e-05 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809202+0000", 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2350000000000002e-06 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.809208+0000", 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2959999999999998e-06 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.810013+0000", 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080517900000000005 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103285+0000", 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29327189100000001 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103492+0000", 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00020672900000000001 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106684+0000", 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003192009 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106719+0000", 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5380999999999999e-05 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106728+0000", 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.9309999999999997e-06 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106774+0000", 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5741999999999997e-05 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.106811+0000", 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7273e-05 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.55504.0:18833 9.9 e314/306 ondisk, result = 0)", 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.810791+0000", 2023-10-09T04:18:24.841 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.8678660740000002, 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.293824223, 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.810791+0000", 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.810791+0000", 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.810791+0000", 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.810795+0000", 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6110000000000004e-06 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103217+0000", 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29242138600000001 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103238+0000", 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1542000000000001e-05 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104594+0000", 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001356381 2023-10-09T04:18:24.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104603+0000", 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.089e-06 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104615+0000", 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2926e-05 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.849 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.55504.0:18833 9.9 e314/306 ondisk, result = 0)", 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.811504+0000", 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.867152994, 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29324936200000001, 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.811504+0000", 2023-10-09T04:18:24.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.811504+0000", 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.811504+0000", 2023-10-09T04:18:24.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.811509+0000", 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2020000000000003e-06 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103218+0000", 2023-10-09T04:18:24.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29170939299999998 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103238+0000", 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9664e-05 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104643+0000", 2023-10-09T04:18:24.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001405505 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104649+0000", 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0719999999999996e-06 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104754+0000", 2023-10-09T04:18:24.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000104248 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:18847 9.9 9:971b2252:::benchmark_data_smithi163_464650_object18846:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:22.813199+0000", 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 1.865458048, 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29791304899999999, 2023-10-09T04:18:24.855 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18847 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:18:24.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.813199+0000", 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.813199+0000", 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:18:24.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.813198+0000", 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:22.813224+0000", 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6444e-05 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103231+0000", 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.290006667 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.103246+0000", 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5702e-05 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:18:24.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104799+0000", 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001552882 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.104920+0000", 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120213 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:18:24.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.105094+0000", 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000174592 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.109011+0000", 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039167280000000004 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:18:24.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.109145+0000", 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000133817 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.111037+0000", 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018922629999999999 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.111079+0000", 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2345999999999997e-05 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:23.111112+0000", 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3037999999999998e-05 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.863 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.864 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:18:24.864 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:18:24.864 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:18:24.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:24.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:24.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:24.866 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 22524 22508 140.618 127.562 0.00190247 0.00703641 2023-10-09T04:18:24.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:25.022 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[221067]: 1.c scrub starts 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[221067]: 1.c scrub ok 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[221067]: 1.1a scrub starts 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[221067]: 1.1a scrub ok 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[221067]: pgmap v854: 119 pgs: 119 active+clean; 965 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[213841]: 1.c scrub starts 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[213841]: 1.c scrub ok 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[213841]: 1.1a scrub starts 2023-10-09T04:18:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[213841]: 1.1a scrub ok 2023-10-09T04:18:25.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:24 smithi163 ceph-mon[213841]: pgmap v854: 119 pgs: 119 active+clean; 965 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T04:18:25.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:25.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:25.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:24 smithi181 ceph-mon[196979]: 1.c scrub starts 2023-10-09T04:18:25.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:24 smithi181 ceph-mon[196979]: 1.c scrub ok 2023-10-09T04:18:25.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:24 smithi181 ceph-mon[196979]: 1.1a scrub starts 2023-10-09T04:18:25.126 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:24 smithi181 ceph-mon[196979]: 1.1a scrub ok 2023-10-09T04:18:25.127 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:24 smithi181 ceph-mon[196979]: pgmap v854: 119 pgs: 119 active+clean; 965 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T04:18:25.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:25.282 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:25.385 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:25.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:25.491 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:25.542 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:25.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:25.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:25.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:25.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:25.757 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:25.829 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 24632 24616 139.811 131.75 0.00416959 0.0071183 2023-10-09T04:18:26.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[213841]: 1.2 scrub starts 2023-10-09T04:18:26.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[213841]: 1.2 scrub ok 2023-10-09T04:18:26.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[213841]: 1.1d scrub starts 2023-10-09T04:18:26.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[213841]: 1.1d scrub ok 2023-10-09T04:18:26.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[221067]: 1.2 scrub starts 2023-10-09T04:18:26.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[221067]: 1.2 scrub ok 2023-10-09T04:18:26.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[221067]: 1.1d scrub starts 2023-10-09T04:18:26.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:25 smithi163 ceph-mon[221067]: 1.1d scrub ok 2023-10-09T04:18:26.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:26.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:26.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:26.187 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:18:26.242 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:26.251 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:25 smithi181 ceph-mon[196979]: 1.2 scrub starts 2023-10-09T04:18:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:25 smithi181 ceph-mon[196979]: 1.2 scrub ok 2023-10-09T04:18:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:25 smithi181 ceph-mon[196979]: 1.1d scrub starts 2023-10-09T04:18:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:25 smithi181 ceph-mon[196979]: 1.1d scrub ok 2023-10-09T04:18:26.340 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:26.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:26.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:26.555 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:26.598 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:26.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:26.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:26.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:26.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 26429 26413 137.45 112.312 0.00836658 0.00726751 2023-10-09T04:18:26.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.15 scrub starts 2023-10-09T04:18:26.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.15 scrub ok 2023-10-09T04:18:26.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.11 scrub starts 2023-10-09T04:18:26.852 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.11 scrub ok 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.16 scrub starts 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.16 scrub ok 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: pgmap v855: 119 pgs: 119 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.5 scrub starts 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[221067]: 1.5 scrub ok 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.15 scrub starts 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.15 scrub ok 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.11 scrub starts 2023-10-09T04:18:26.853 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.11 scrub ok 2023-10-09T04:18:26.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.16 scrub starts 2023-10-09T04:18:26.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.16 scrub ok 2023-10-09T04:18:26.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: pgmap v855: 119 pgs: 119 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:18:26.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.5 scrub starts 2023-10-09T04:18:26.854 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:26 smithi163 ceph-mon[213841]: 1.5 scrub ok 2023-10-09T04:18:26.868 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:27.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.15 scrub starts 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.15 scrub ok 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.11 scrub starts 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.11 scrub ok 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.16 scrub starts 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.16 scrub ok 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: pgmap v855: 119 pgs: 119 active+clean; 1.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.5 scrub starts 2023-10-09T04:18:27.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:26 smithi181 ceph-mon[196979]: 1.5 scrub ok 2023-10-09T04:18:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:27 smithi163 conmon[213817]: 2023-10-09T04:18:27.257+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e314 definitely_dead 0 2023-10-09T04:18:27.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 28486 28470 136.766 128.562 0.00797205 0.00729308 2023-10-09T04:18:27.901 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.6. 2023-10-09T04:18:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[221067]: 1.e scrub starts 2023-10-09T04:18:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[221067]: 1.e scrub ok 2023-10-09T04:18:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1246741744' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:18:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:18:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[221067]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:18:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[213841]: 1.e scrub starts 2023-10-09T04:18:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[213841]: 1.e scrub ok 2023-10-09T04:18:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1246741744' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:18:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:18:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:27 smithi163 ceph-mon[213841]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:18:28.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:28.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:28.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:27 smithi181 ceph-mon[196979]: 1.e scrub starts 2023-10-09T04:18:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:27 smithi181 ceph-mon[196979]: 1.e scrub ok 2023-10-09T04:18:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1246741744' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:18:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:18:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:27 smithi181 ceph-mon[196979]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:18:28.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:28.337 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:28.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:28.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:28.486 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:28.512 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:28.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:28.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:28.660 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:28.689 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:28.835 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 28562 28546 127.342 4.75 0.00159002 0.00738702 2023-10-09T04:18:28.972 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: 1.6 scrub starts 2023-10-09T04:18:28.972 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: 1.6 scrub ok 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: pgmap v856: 119 pgs: 119 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: failed to encode map e315 with expected crc 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[213841]: failed to encode map e315 with expected crc 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: 1.6 scrub starts 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: 1.6 scrub ok 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: pgmap v856: 119 pgs: 119 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-10-09T04:18:28.973 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: failed to encode map e315 with expected crc 2023-10-09T04:18:28.974 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:18:28.974 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T04:18:28.974 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:28 smithi163 ceph-mon[221067]: failed to encode map e315 with expected crc 2023-10-09T04:18:29.222 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:29.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: 1.6 scrub starts 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: 1.6 scrub ok 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: pgmap v856: 119 pgs: 119 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: failed to encode map e315 with expected crc 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T04:18:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:28 smithi181 ceph-mon[196979]: failed to encode map e315 with expected crc 2023-10-09T04:18:29.302 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:29.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:29.355 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:29.463 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:29.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:29.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:29.599 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:29.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:29.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:29.711 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:29.731 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:29.751 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:29.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:29.772 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:29.779 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:29.791 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:29.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:29.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 28562 28546 118.812 0 - 0.00738702 2023-10-09T04:18:29.843 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:29.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:29.871 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:29.910 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:18:29.920 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.921 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:18:29.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:30.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: osdmap e316: 8 total, 8 up, 7 in 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2631492575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[213841]: osdmap e317: 8 total, 8 up, 7 in 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: osdmap e316: 8 total, 8 up, 7 in 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.004 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2631492575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:30.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:30.005 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:29 smithi163 ceph-mon[221067]: osdmap e317: 8 total, 8 up, 7 in 2023-10-09T04:18:30.018 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:30.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: osdmap e316: 8 total, 8 up, 7 in 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: failed to encode map e316 with expected crc 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2631492575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:30.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:30.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:29 smithi181 ceph-mon[196979]: osdmap e317: 8 total, 8 up, 7 in 2023-10-09T04:18:30.837 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:18:30.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 29726 29710 115.936 36.375 0.0165012 0.00861521 2023-10-09T04:18:31.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: pgmap v859: 119 pgs: 1 remapped+peering, 5 peering, 113 active+clean; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:18:31.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3556264815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3556264815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[213841]: osdmap e318: 8 total, 8 up, 7 in 2023-10-09T04:18:31.065 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:18:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:18:30] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: pgmap v859: 119 pgs: 1 remapped+peering, 5 peering, 113 active+clean; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3556264815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3556264815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:31.066 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:30 smithi163 ceph-mon[221067]: osdmap e318: 8 total, 8 up, 7 in 2023-10-09T04:18:31.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: pgmap v859: 119 pgs: 1 remapped+peering, 5 peering, 113 active+clean; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: failed to encode map e317 with expected crc 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3556264815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3556264815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:30 smithi181 ceph-mon[196979]: osdmap e318: 8 total, 8 up, 7 in 2023-10-09T04:18:31.330 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:31.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:31.436 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:31.463 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:31.516 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:31.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:31.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:31.667 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:31.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:31.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:31.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:31.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 31128 31112 114.271 87.625 0.0160688 0.0087463 2023-10-09T04:18:31.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:31.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[213841]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[213841]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[213841]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[213841]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.914 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[213841]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T04:18:31.914 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[221067]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[221067]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[221067]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[221067]: failed to encode map e318 with expected crc 2023-10-09T04:18:31.915 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:31 smithi163 ceph-mon[221067]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T04:18:31.917 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:32.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:32.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:32.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:32.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:32.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:31 smithi181 ceph-mon[196979]: failed to encode map e318 with expected crc 2023-10-09T04:18:32.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:31 smithi181 ceph-mon[196979]: failed to encode map e318 with expected crc 2023-10-09T04:18:32.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:31 smithi181 ceph-mon[196979]: failed to encode map e318 with expected crc 2023-10-09T04:18:32.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:31 smithi181 ceph-mon[196979]: failed to encode map e318 with expected crc 2023-10-09T04:18:32.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:31 smithi181 ceph-mon[196979]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T04:18:32.308 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:32.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 32304 32288 112.007 73.5 0.00344999 0.00892276 2023-10-09T04:18:32.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:32.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:33.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:33.094 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:33.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:33.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:33.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: pgmap v862: 119 pgs: 2 active+recovering+undersized+remapped, 10 active+undersized+remapped, 1 remapped+peering, 8 peering, 98 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 3233/89253 objects misplaced (3.622%); 3.4 MiB/s, 56 objects/s recovering 2023-10-09T04:18:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.177 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:32 smithi181 ceph-mon[196979]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.243 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:18:33.243 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:18:33.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: pgmap v862: 119 pgs: 2 active+recovering+undersized+remapped, 10 active+undersized+remapped, 1 remapped+peering, 8 peering, 98 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 3233/89253 objects misplaced (3.622%); 3.4 MiB/s, 56 objects/s recovering 2023-10-09T04:18:33.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[213841]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: pgmap v862: 119 pgs: 2 active+recovering+undersized+remapped, 10 active+undersized+remapped, 1 remapped+peering, 8 peering, 98 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 3233/89253 objects misplaced (3.622%); 3.4 MiB/s, 56 objects/s recovering 2023-10-09T04:18:33.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:32 smithi163 ceph-mon[221067]: failed to encode map e319 with expected crc 2023-10-09T04:18:33.276 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:33.348 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:33.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:33.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:33.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:33.671 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:33.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:33.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:33.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 33891 33875 111.333 99.1875 0.0039416 0.0089738 2023-10-09T04:18:33.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:33.909 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:33.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:33.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:34.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:33 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 1746/91539 objects degraded (1.907%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:34.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:18:34.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:34.081 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:34.140 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:34.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:34.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:34.245 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:34.298 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:34.333 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:18:34.333 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:18:34.333 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:18:34.333 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:18:34.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:33 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 1746/91539 objects degraded (1.907%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:34.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:33 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 1746/91539 objects degraded (1.907%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:34.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:34.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:34.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:18:34.487 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:34.535 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:34.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:34.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:34.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:34.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:34.816 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:34.816 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:34.841 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:18:34.841900+0000 min lat: 0.00114347 max lat: 2.09794 avg lat: 0.00917474 2023-10-09T04:18:34.842 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:18:34.842 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 34895 34879 108.905 62.75 0.0195734 0.00917474 2023-10-09T04:18:34.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:34.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:34.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:18:35.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:35.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:34 smithi163 ceph-mon[221067]: pgmap v864: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 10 active+undersized+remapped, 1 remapped+peering, 3 peering, 102 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 1746/91539 objects degraded (1.907%); 3233/91539 objects misplaced (3.532%); 2.8 MiB/s, 46 objects/s recovering 2023-10-09T04:18:35.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:35.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:35.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:34 smithi163 ceph-mon[213841]: pgmap v864: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 10 active+undersized+remapped, 1 remapped+peering, 3 peering, 102 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 1746/91539 objects degraded (1.907%); 3233/91539 objects misplaced (3.532%); 2.8 MiB/s, 46 objects/s recovering 2023-10-09T04:18:35.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:35.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:35.092 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:35.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:35.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:35.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:34 smithi181 ceph-mon[196979]: pgmap v864: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 10 active+undersized+remapped, 1 remapped+peering, 3 peering, 102 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 1746/91539 objects degraded (1.907%); 3233/91539 objects misplaced (3.532%); 2.8 MiB/s, 46 objects/s recovering 2023-10-09T04:18:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:35.370 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:35.390 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:35.582 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:35.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:35.697 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:35.739 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:35.828 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:35.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:35.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 36279 36263 107.836 86.5 0.0054574 0.00926744 2023-10-09T04:18:35.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:35.989 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:36.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:36.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:36.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:36.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:36.227 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:36.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:36.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:36.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:36.425 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:36.500 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:36.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:36.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:36.674 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:36.843 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 37627 37611 106.758 84.25 0.00382442 0.00935807 2023-10-09T04:18:36.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:36.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:36.909 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:36.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:37.049 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:37.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:36 smithi163 ceph-mon[213841]: pgmap v865: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 6 active+undersized+remapped, 2 peering, 108 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 1746/97830 objects degraded (1.785%); 2535/97830 objects misplaced (2.591%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:18:37.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:36 smithi163 ceph-mon[221067]: pgmap v865: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 6 active+undersized+remapped, 2 peering, 108 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 1746/97830 objects degraded (1.785%); 2535/97830 objects misplaced (2.591%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:18:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:36 smithi181 ceph-mon[196979]: pgmap v865: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 6 active+undersized+remapped, 2 peering, 108 active+clean; 2.0 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 1746/97830 objects degraded (1.785%); 2535/97830 objects misplaced (2.591%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:18:37.503 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:37.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:37.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:37.556 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:37.594 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:37.609 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:37.628 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:37.662 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:37.679 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:37.689 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:37.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:37.731 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:37.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:37.769 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:37.805 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:18:37.809 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.809 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:18:37.838 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:37.843 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 38946 38930 105.701 82.4375 0.00225116 0.00943608 2023-10-09T04:18:37.864 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:37.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:37.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:38.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:38.084 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:38.181 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:38.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:38.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:38.352 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:38.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:38.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:38.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:38.562 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:38.654 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:38.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:38.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:38.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3629654545' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:38 smithi181 ceph-mon[196979]: pgmap v866: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 1746/107529 objects degraded (1.624%); 2115/107529 objects misplaced (1.967%); 10 MiB/s, 168 objects/s recovering 2023-10-09T04:18:38.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3629654545' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:38.816 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:38 smithi163 ceph-mon[221067]: pgmap v866: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 1746/107529 objects degraded (1.624%); 2115/107529 objects misplaced (1.967%); 10 MiB/s, 168 objects/s recovering 2023-10-09T04:18:38.816 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3629654545' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:38.816 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:38 smithi163 ceph-mon[213841]: pgmap v866: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 1746/107529 objects degraded (1.624%); 2115/107529 objects misplaced (1.967%); 10 MiB/s, 168 objects/s recovering 2023-10-09T04:18:38.821 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:38.843 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 40371 40355 105.008 89.0625 0.00725868 0.0095181 2023-10-09T04:18:38.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:38.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:39.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:39.102 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:39.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 41467 41451 103.548 68.5 0.00490317 0.00965125 2023-10-09T04:18:39.896 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:39.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:40.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:40.021 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:40.044 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:40.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:40.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:40.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:40.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:39 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1746/111237 objects degraded (1.570%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:40.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:39 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1746/111237 objects degraded (1.570%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:40.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:39 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1746/111237 objects degraded (1.570%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:40.310 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:40.798 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:18:40.807 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:18:40.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:18:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:18:40] "GET /metrics HTTP/1.1" 200 36612 "" "Prometheus/2.43.0" 2023-10-09T04:18:40.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 42798 42782 102.765 83.1875 0.0121222 0.0097252 2023-10-09T04:18:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:40 smithi181 ceph-mon[196979]: pgmap v867: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 1746/111237 objects degraded (1.570%); 2115/111237 objects misplaced (1.901%); 8.6 MiB/s, 137 objects/s recovering 2023-10-09T04:18:41.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:40 smithi163 ceph-mon[213841]: pgmap v867: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 1746/111237 objects degraded (1.570%); 2115/111237 objects misplaced (1.901%); 8.6 MiB/s, 137 objects/s recovering 2023-10-09T04:18:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:40 smithi163 ceph-mon[221067]: pgmap v867: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 2.3 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 1746/111237 objects degraded (1.570%); 2115/111237 objects misplaced (1.901%); 8.6 MiB/s, 137 objects/s recovering 2023-10-09T04:18:41.428 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:41.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:41.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:41.559 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:41.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:41.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:41.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:41.733 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:41.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:41.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:41.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:41.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 43127 43111 99.7225 20.5625 0.00263196 0.00979144 2023-10-09T04:18:41.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:41.913 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:42.016 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:18:42.127 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:42.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:42.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[213841]: from='client.55537 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:18:42.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[213841]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[213841]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[213841]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[213841]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[221067]: from='client.55537 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[221067]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[221067]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.151 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[221067]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T04:18:42.152 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:41 smithi163 ceph-mon[221067]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.171 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":867,"stamp":"2023-10-09T04:18:39.781836+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2429006706,"num_objects":37079,"num_object_clones":0,"num_object_copies":111237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1746,"num_objects_misplaced":2115,"num_objects_unfound":0,"num_objects_dirty":37079,"num_whiteouts":0,"num_read":197,"num_read_kb":368,"num_write":37213,"num_write_kb":2372520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1398,"num_bytes_recovered":88803172,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42101,"ondisk_log_size":42101,"up":357,"acting":354,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9812120,"kb_used_data":6823712,"kb_used_omap":126,"kb_used_meta":2988225,"kb_avail":646403944,"statfs":{"total":671965249536,"available":661917638656,"internally_reserved":0,"allocated":6987481088,"data_stored":6979217768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3059943035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":557304727,"num_objects":8504,"num_object_clones":0,"num_object_copies":25512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1746,"num_objects_misplaced":556,"num_objects_unfound":0,"num_objects_dirty":8504,"num_whiteouts":0,"num_read":-11,"num_read_kb":-20,"num_write":8499,"num_write_kb":544201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1229,"num_bytes_recovered":80503492,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.957906"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":433,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843149+0000","last_change":"2023-10-09T04:18:04.685106+0000","last_active":"2023-10-09T04:18:30.843149+0000","last_peered":"2023-10-09T04:18:30.843149+0000","last_clean":"2023-10-09T04:18:30.843149+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:18:30.843149+0000","last_undegraded":"2023-10-09T04:18:30.843149+0000","last_fullsized":"2023-10-09T04:18:30.843149+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:00.124399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:00.124399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":63,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834695+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:18:30.834695+0000","last_peered":"2023-10-09T04:18:30.834695+0000","last_clean":"2023-10-09T04:18:30.834695+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:18:30.834695+0000","last_undegraded":"2023-10-09T04:18:30.834695+0000","last_fullsized":"2023-10-09T04:18:30.834695+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":293,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831298+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:18:30.831298+0000","last_peered":"2023-10-09T04:18:30.831298+0000","last_clean":"2023-10-09T04:18:30.831298+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:18:30.831298+0000","last_undegraded":"2023-10-09T04:18:30.831298+0000","last_fullsized":"2023-10-09T04:18:30.831298+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":433,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843552+0000","last_change":"2023-10-09T04:18:04.679798+0000","last_active":"2023-10-09T04:18:30.843552+0000","last_peered":"2023-10-09T04:18:30.843552+0000","last_clean":"2023-10-09T04:18:30.843552+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:18:30.843552+0000","last_undegraded":"2023-10-09T04:18:30.843552+0000","last_fullsized":"2023-10-09T04:18:30.843552+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830913+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:18:30.830913+0000","last_peered":"2023-10-09T04:18:30.830913+0000","last_clean":"2023-10-09T04:18:30.830913+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:18:30.830913+0000","last_undegraded":"2023-10-09T04:18:30.830913+0000","last_fullsized":"2023-10-09T04:18:30.830913+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":287,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830747+0000","last_change":"2023-10-09T04:16:24.469036+0000","last_active":"2023-10-09T04:18:30.830747+0000","last_peered":"2023-10-09T04:18:30.830747+0000","last_clean":"2023-10-09T04:18:30.830747+0000","last_became_active":"2023-10-09T04:16:24.468649+0000","last_became_peered":"2023-10-09T04:16:24.468649+0000","last_unstale":"2023-10-09T04:18:30.830747+0000","last_undegraded":"2023-10-09T04:18:30.830747+0000","last_fullsized":"2023-10-09T04:18:30.830747+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:34:53.505550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":443,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834555+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:18:30.834555+0000","last_peered":"2023-10-09T04:18:30.834555+0000","last_clean":"2023-10-09T04:18:30.834555+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:18:30.834555+0000","last_undegraded":"2023-10-09T04:18:30.834555+0000","last_fullsized":"2023-10-09T04:18:30.834555+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":293,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.829610+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:18:30.829610+0000","last_peered":"2023-10-09T04:18:30.829610+0000","last_clean":"2023-10-09T04:18:30.829610+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:18:30.829610+0000","last_undegraded":"2023-10-09T04:18:30.829610+0000","last_fullsized":"2023-10-09T04:18:30.829610+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":262,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.848882+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:18:30.848882+0000","last_peered":"2023-10-09T04:18:30.848882+0000","last_clean":"2023-10-09T04:18:30.848882+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:18:30.848882+0000","last_undegraded":"2023-10-09T04:18:30.848882+0000","last_fullsized":"2023-10-09T04:18:30.848882+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":718,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837376+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:18:30.837376+0000","last_peered":"2023-10-09T04:18:30.837376+0000","last_clean":"2023-10-09T04:18:30.837376+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:18:30.837376+0000","last_undegraded":"2023-10-09T04:18:30.837376+0000","last_fullsized":"2023-10-09T04:18:30.837376+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":294,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831732+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:18:30.831732+0000","last_peered":"2023-10-09T04:18:30.831732+0000","last_clean":"2023-10-09T04:18:30.831732+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:18:30.831732+0000","last_undegraded":"2023-10-09T04:18:30.831732+0000","last_fullsized":"2023-10-09T04:18:30.831732+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":28,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.842174+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:18:30.842174+0000","last_peered":"2023-10-09T04:18:30.842174+0000","last_clean":"2023-10-09T04:18:30.842174+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:18:30.842174+0000","last_undegraded":"2023-10-09T04:18:30.842174+0000","last_fullsized":"2023-10-09T04:18:30.842174+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":444,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.829441+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:18:30.829441+0000","last_peered":"2023-10-09T04:18:30.829441+0000","last_clean":"2023-10-09T04:18:30.829441+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:18:30.829441+0000","last_undegraded":"2023-10-09T04:18:30.829441+0000","last_fullsized":"2023-10-09T04:18:30.829441+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":278,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845478+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:18:30.845478+0000","last_peered":"2023-10-09T04:18:30.845478+0000","last_clean":"2023-10-09T04:18:30.845478+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:18:30.845478+0000","last_undegraded":"2023-10-09T04:18:30.845478+0000","last_fullsized":"2023-10-09T04:18:30.845478+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843209+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:18:30.843209+0000","last_peered":"2023-10-09T04:18:30.843209+0000","last_clean":"2023-10-09T04:18:30.843209+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:18:30.843209+0000","last_undegraded":"2023-10-09T04:18:30.843209+0000","last_fullsized":"2023-10-09T04:18:30.843209+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":453,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.851743+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:18:31.851743+0000","last_peered":"2023-10-09T04:18:31.851743+0000","last_clean":"2023-10-09T04:18:31.851743+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:18:31.851743+0000","last_undegraded":"2023-10-09T04:18:31.851743+0000","last_fullsized":"2023-10-09T04:18:31.851743+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":345,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837782+0000","last_change":"2023-10-09T04:18:15.756165+0000","last_active":"2023-10-09T04:18:30.837782+0000","last_peered":"2023-10-09T04:18:30.837782+0000","last_clean":"2023-10-09T04:18:30.837782+0000","last_became_active":"2023-10-09T04:18:15.741853+0000","last_became_peered":"2023-10-09T04:18:15.741853+0000","last_unstale":"2023-10-09T04:18:30.837782+0000","last_undegraded":"2023-10-09T04:18:30.837782+0000","last_fullsized":"2023-10-09T04:18:30.837782+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:06.303779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830723+0000","last_change":"2023-10-09T04:18:28.826612+0000","last_active":"2023-10-09T04:18:30.830723+0000","last_peered":"2023-10-09T04:18:30.830723+0000","last_clean":"2023-10-09T04:18:30.830723+0000","last_became_active":"2023-10-09T04:18:28.826243+0000","last_became_peered":"2023-10-09T04:18:28.826243+0000","last_unstale":"2023-10-09T04:18:30.830723+0000","last_undegraded":"2023-10-09T04:18:30.830723+0000","last_fullsized":"2023-10-09T04:18:30.830723+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:16.838841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":460,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.832276+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:18:30.832276+0000","last_peered":"2023-10-09T04:18:30.832276+0000","last_clean":"2023-10-09T04:18:30.832276+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:18:30.832276+0000","last_undegraded":"2023-10-09T04:18:30.832276+0000","last_fullsized":"2023-10-09T04:18:30.832276+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":93,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.832758+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:18:30.832758+0000","last_peered":"2023-10-09T04:18:30.832758+0000","last_clean":"2023-10-09T04:18:30.832758+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:18:30.832758+0000","last_undegraded":"2023-10-09T04:18:30.832758+0000","last_fullsized":"2023-10-09T04:18:30.832758+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834173+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:18:30.834173+0000","last_peered":"2023-10-09T04:18:30.834173+0000","last_clean":"2023-10-09T04:18:30.834173+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:18:30.834173+0000","last_undegraded":"2023-10-09T04:18:30.834173+0000","last_fullsized":"2023-10-09T04:18:30.834173+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":468,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.832023+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:18:30.832023+0000","last_peered":"2023-10-09T04:18:30.832023+0000","last_clean":"2023-10-09T04:18:30.832023+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:18:30.832023+0000","last_undegraded":"2023-10-09T04:18:30.832023+0000","last_fullsized":"2023-10-09T04:18:30.832023+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":312,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837932+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:18:30.837932+0000","last_peered":"2023-10-09T04:18:30.837932+0000","last_clean":"2023-10-09T04:18:30.837932+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:18:30.837932+0000","last_undegraded":"2023-10-09T04:18:30.837932+0000","last_fullsized":"2023-10-09T04:18:30.837932+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":129,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834188+0000","last_change":"2023-10-09T04:18:15.742754+0000","last_active":"2023-10-09T04:18:30.834188+0000","last_peered":"2023-10-09T04:18:30.834188+0000","last_clean":"2023-10-09T04:18:30.834188+0000","last_became_active":"2023-10-09T04:18:15.742276+0000","last_became_peered":"2023-10-09T04:18:15.742276+0000","last_unstale":"2023-10-09T04:18:30.834188+0000","last_undegraded":"2023-10-09T04:18:30.834188+0000","last_fullsized":"2023-10-09T04:18:30.834188+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:48:08.030994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":453,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.848384+0000","last_change":"2023-10-09T04:18:31.848384+0000","last_active":"2023-10-09T04:18:31.848384+0000","last_peered":"2023-10-09T04:18:31.848384+0000","last_clean":"2023-10-09T04:18:31.848384+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:18:31.848384+0000","last_undegraded":"2023-10-09T04:18:31.848384+0000","last_fullsized":"2023-10-09T04:18:31.848384+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":295,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.828904+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:18:30.828904+0000","last_peered":"2023-10-09T04:18:30.828904+0000","last_clean":"2023-10-09T04:18:30.828904+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:18:30.828904+0000","last_undegraded":"2023-10-09T04:18:30.828904+0000","last_fullsized":"2023-10-09T04:18:30.828904+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":297,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845400+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:18:30.845400+0000","last_peered":"2023-10-09T04:18:30.845400+0000","last_clean":"2023-10-09T04:18:30.845400+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:18:30.845400+0000","last_undegraded":"2023-10-09T04:18:30.845400+0000","last_fullsized":"2023-10-09T04:18:30.845400+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":443,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831801+0000","last_change":"2023-10-09T04:18:19.682177+0000","last_active":"2023-10-09T04:18:30.831801+0000","last_peered":"2023-10-09T04:18:30.831801+0000","last_clean":"2023-10-09T04:18:30.831801+0000","last_became_active":"2023-10-09T04:18:04.675910+0000","last_became_peered":"2023-10-09T04:18:04.675910+0000","last_unstale":"2023-10-09T04:18:30.831801+0000","last_undegraded":"2023-10-09T04:18:30.831801+0000","last_fullsized":"2023-10-09T04:18:30.831801+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:35.056855+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834478+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:18:30.834478+0000","last_peered":"2023-10-09T04:18:30.834478+0000","last_clean":"2023-10-09T04:18:30.834478+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:18:30.834478+0000","last_undegraded":"2023-10-09T04:18:30.834478+0000","last_fullsized":"2023-10-09T04:18:30.834478+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":294,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830857+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:18:30.830857+0000","last_peered":"2023-10-09T04:18:30.830857+0000","last_clean":"2023-10-09T04:18:30.830857+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:18:30.830857+0000","last_undegraded":"2023-10-09T04:18:30.830857+0000","last_fullsized":"2023-10-09T04:18:30.830857+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":453,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.847549+0000","last_change":"2023-10-09T04:18:31.847549+0000","last_active":"2023-10-09T04:18:31.847549+0000","last_peered":"2023-10-09T04:18:31.847549+0000","last_clean":"2023-10-09T04:18:31.847549+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:18:31.847549+0000","last_undegraded":"2023-10-09T04:18:31.847549+0000","last_fullsized":"2023-10-09T04:18:31.847549+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":289,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.840544+0000","last_change":"2023-10-09T04:18:28.828405+0000","last_active":"2023-10-09T04:18:31.840544+0000","last_peered":"2023-10-09T04:18:31.840544+0000","last_clean":"2023-10-09T04:18:31.840544+0000","last_became_active":"2023-10-09T04:18:28.818294+0000","last_became_peered":"2023-10-09T04:18:28.818294+0000","last_unstale":"2023-10-09T04:18:31.840544+0000","last_undegraded":"2023-10-09T04:18:31.840544+0000","last_fullsized":"2023-10-09T04:18:31.840544+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:33.633471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":106,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831456+0000","last_change":"2023-10-09T04:16:24.474334+0000","last_active":"2023-10-09T04:18:30.831456+0000","last_peered":"2023-10-09T04:18:30.831456+0000","last_clean":"2023-10-09T04:18:30.831456+0000","last_became_active":"2023-10-09T04:16:24.474003+0000","last_became_peered":"2023-10-09T04:16:24.474003+0000","last_unstale":"2023-10-09T04:18:30.831456+0000","last_undegraded":"2023-10-09T04:18:30.831456+0000","last_fullsized":"2023-10-09T04:18:30.831456+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:38.410877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"319'2313","reported_seq":2335,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:36.662762+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:18:36.662762+0000","last_peered":"2023-10-09T04:18:36.662762+0000","last_clean":"2023-10-09T04:18:36.662762+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:18:36.662762+0000","last_undegraded":"2023-10-09T04:18:36.662762+0000","last_fullsized":"2023-10-09T04:18:36.662762+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2313,"log_dups_size":0,"ondisk_log_size":2313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584768,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2313,"num_write_kb":148032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":708,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837507+0000","last_change":"2023-10-09T04:18:04.686976+0000","last_active":"2023-10-09T04:18:30.837507+0000","last_peered":"2023-10-09T04:18:30.837507+0000","last_clean":"2023-10-09T04:18:30.837507+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:18:30.837507+0000","last_undegraded":"2023-10-09T04:18:30.837507+0000","last_fullsized":"2023-10-09T04:18:30.837507+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":307,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830828+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:18:30.830828+0000","last_peered":"2023-10-09T04:18:30.830828+0000","last_clean":"2023-10-09T04:18:30.830828+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:18:30.830828+0000","last_undegraded":"2023-10-09T04:18:30.830828+0000","last_fullsized":"2023-10-09T04:18:30.830828+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":270,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843252+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:18:30.843252+0000","last_peered":"2023-10-09T04:18:30.843252+0000","last_clean":"2023-10-09T04:18:30.843252+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:18:30.843252+0000","last_undegraded":"2023-10-09T04:18:30.843252+0000","last_fullsized":"2023-10-09T04:18:30.843252+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"319'2461","reported_seq":2492,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:38.179087+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:18:38.179087+0000","last_peered":"2023-10-09T04:18:38.179087+0000","last_clean":"2023-10-09T04:18:38.179087+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:18:38.179087+0000","last_undegraded":"2023-10-09T04:18:38.179087+0000","last_fullsized":"2023-10-09T04:18:38.179087+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2461,"log_dups_size":0,"ondisk_log_size":2461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161284096,"num_objects":2461,"num_object_clones":0,"num_object_copies":7383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2461,"num_write_kb":157504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":485,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.846705+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:18:31.846705+0000","last_peered":"2023-10-09T04:18:31.846705+0000","last_clean":"2023-10-09T04:18:31.846705+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:18:31.846705+0000","last_undegraded":"2023-10-09T04:18:31.846705+0000","last_fullsized":"2023-10-09T04:18:31.846705+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":311,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.836134+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:18:30.836134+0000","last_peered":"2023-10-09T04:18:30.836134+0000","last_clean":"2023-10-09T04:18:30.836134+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:18:30.836134+0000","last_undegraded":"2023-10-09T04:18:30.836134+0000","last_fullsized":"2023-10-09T04:18:30.836134+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834645+0000","last_change":"2023-10-09T04:18:28.820265+0000","last_active":"2023-10-09T04:18:30.834645+0000","last_peered":"2023-10-09T04:18:30.834645+0000","last_clean":"2023-10-09T04:18:30.834645+0000","last_became_active":"2023-10-09T04:18:28.819944+0000","last_became_peered":"2023-10-09T04:18:28.819944+0000","last_unstale":"2023-10-09T04:18:30.834645+0000","last_undegraded":"2023-10-09T04:18:30.834645+0000","last_fullsized":"2023-10-09T04:18:30.834645+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:58.236930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"319'2338","reported_seq":2359,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.792663+0000","last_change":"2023-10-09T04:18:07.758447+0000","last_active":"2023-10-09T04:18:35.792663+0000","last_peered":"2023-10-09T04:18:35.792663+0000","last_clean":"2023-10-09T04:18:35.792663+0000","last_became_active":"2023-10-09T04:18:07.758159+0000","last_became_peered":"2023-10-09T04:18:07.758159+0000","last_unstale":"2023-10-09T04:18:35.792663+0000","last_undegraded":"2023-10-09T04:18:35.792663+0000","last_fullsized":"2023-10-09T04:18:35.792663+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2338,"log_dups_size":0,"ondisk_log_size":2338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:29.487444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153223168,"num_objects":2338,"num_object_clones":0,"num_object_copies":7014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2338,"num_write_kb":149632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":453,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.849056+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:18:31.849056+0000","last_peered":"2023-10-09T04:18:31.849056+0000","last_clean":"2023-10-09T04:18:31.849056+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:18:31.849056+0000","last_undegraded":"2023-10-09T04:18:31.849056+0000","last_fullsized":"2023-10-09T04:18:31.849056+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":276,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837650+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:18:30.837650+0000","last_peered":"2023-10-09T04:18:30.837650+0000","last_clean":"2023-10-09T04:18:30.837650+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:18:30.837650+0000","last_undegraded":"2023-10-09T04:18:30.837650+0000","last_fullsized":"2023-10-09T04:18:30.837650+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":250,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831772+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:18:30.831772+0000","last_peered":"2023-10-09T04:18:30.831772+0000","last_clean":"2023-10-09T04:18:30.831772+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:18:30.831772+0000","last_undegraded":"2023-10-09T04:18:30.831772+0000","last_fullsized":"2023-10-09T04:18:30.831772+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"319'2278","reported_seq":2299,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.794068+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:18:35.794068+0000","last_peered":"2023-10-09T04:18:35.794068+0000","last_clean":"2023-10-09T04:18:35.794068+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:18:35.794068+0000","last_undegraded":"2023-10-09T04:18:35.794068+0000","last_fullsized":"2023-10-09T04:18:35.794068+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2278,"log_dups_size":0,"ondisk_log_size":2278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149291008,"num_objects":2278,"num_object_clones":0,"num_object_copies":6834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2278,"num_write_kb":145792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":463,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.836049+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:18:30.836049+0000","last_peered":"2023-10-09T04:18:30.836049+0000","last_clean":"2023-10-09T04:18:30.836049+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:18:30.836049+0000","last_undegraded":"2023-10-09T04:18:30.836049+0000","last_fullsized":"2023-10-09T04:18:30.836049+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843311+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:18:30.843311+0000","last_peered":"2023-10-09T04:18:30.843311+0000","last_clean":"2023-10-09T04:18:30.843311+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:18:30.843311+0000","last_undegraded":"2023-10-09T04:18:30.843311+0000","last_fullsized":"2023-10-09T04:18:30.843311+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":295,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830184+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:18:30.830184+0000","last_peered":"2023-10-09T04:18:30.830184+0000","last_clean":"2023-10-09T04:18:30.830184+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:18:30.830184+0000","last_undegraded":"2023-10-09T04:18:30.830184+0000","last_fullsized":"2023-10-09T04:18:30.830184+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"319'2148","reported_seq":3808,"reported_epoch":319,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:18:35.755883+0000","last_change":"2023-10-09T04:18:29.913515+0000","last_active":"2023-10-09T04:18:35.755883+0000","last_peered":"2023-10-09T04:18:35.755883+0000","last_clean":"2023-10-09T04:18:27.807101+0000","last_became_active":"2023-10-09T04:18:29.882352+0000","last_became_peered":"2023-10-09T04:18:29.882352+0000","last_unstale":"2023-10-09T04:18:35.755883+0000","last_undegraded":"2023-10-09T04:18:35.755883+0000","last_fullsized":"2023-10-09T04:18:29.811853+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2148,"log_dups_size":0,"ondisk_log_size":2148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:42.037489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140771328,"num_objects":2148,"num_object_clones":0,"num_object_copies":6444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":861,"num_objects_unfound":0,"num_objects_dirty":2148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2148,"num_write_kb":137472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":809,"num_bytes_recovered":53018624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":861},{"shards":"3,7","objects":1287}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":732,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837244+0000","last_change":"2023-10-09T04:18:18.486757+0000","last_active":"2023-10-09T04:18:30.837244+0000","last_peered":"2023-10-09T04:18:30.837244+0000","last_clean":"2023-10-09T04:18:30.837244+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:18:30.837244+0000","last_undegraded":"2023-10-09T04:18:30.837244+0000","last_fullsized":"2023-10-09T04:18:30.837244+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:01.099879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:01.099879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081520749999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":278,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830274+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:18:30.830274+0000","last_peered":"2023-10-09T04:18:30.830274+0000","last_clean":"2023-10-09T04:18:30.830274+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:18:30.830274+0000","last_undegraded":"2023-10-09T04:18:30.830274+0000","last_fullsized":"2023-10-09T04:18:30.830274+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":93,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.832846+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:18:30.832846+0000","last_peered":"2023-10-09T04:18:30.832846+0000","last_clean":"2023-10-09T04:18:30.832846+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:18:30.832846+0000","last_undegraded":"2023-10-09T04:18:30.832846+0000","last_fullsized":"2023-10-09T04:18:30.832846+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"319'2418","reported_seq":2448,"reported_epoch":319,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:18:38.142215+0000","last_change":"2023-10-09T04:18:29.883888+0000","last_active":"2023-10-09T04:18:38.142215+0000","last_peered":"2023-10-09T04:18:38.142215+0000","last_clean":"2023-10-09T04:18:27.791997+0000","last_became_active":"2023-10-09T04:18:29.878258+0000","last_became_peered":"2023-10-09T04:18:29.878258+0000","last_unstale":"2023-10-09T04:18:38.142215+0000","last_undegraded":"2023-10-09T04:18:29.827214+0000","last_fullsized":"2023-10-09T04:18:29.814170+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2418,"log_dups_size":0,"ondisk_log_size":2418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:43.339530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158466048,"num_objects":2418,"num_object_clones":0,"num_object_copies":7254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1746,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2418,"num_write_kb":154752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2","objects":672},{"shards":"1,2,6","objects":1746}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":478,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.847523+0000","last_change":"2023-10-09T04:18:31.847523+0000","last_active":"2023-10-09T04:18:31.847523+0000","last_peered":"2023-10-09T04:18:31.847523+0000","last_clean":"2023-10-09T04:18:31.847523+0000","last_became_active":"2023-10-09T04:18:31.847197+0000","last_became_peered":"2023-10-09T04:18:31.847197+0000","last_unstale":"2023-10-09T04:18:31.847523+0000","last_undegraded":"2023-10-09T04:18:31.847523+0000","last_fullsized":"2023-10-09T04:18:31.847523+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:27.984557+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834592+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:18:30.834592+0000","last_peered":"2023-10-09T04:18:30.834592+0000","last_clean":"2023-10-09T04:18:30.834592+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:18:30.834592+0000","last_undegraded":"2023-10-09T04:18:30.834592+0000","last_fullsized":"2023-10-09T04:18:30.834592+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":63,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834929+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:18:30.834929+0000","last_peered":"2023-10-09T04:18:30.834929+0000","last_clean":"2023-10-09T04:18:30.834929+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:18:30.834929+0000","last_undegraded":"2023-10-09T04:18:30.834929+0000","last_fullsized":"2023-10-09T04:18:30.834929+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"319'2288","reported_seq":2319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.740118+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:18:35.740118+0000","last_peered":"2023-10-09T04:18:35.740118+0000","last_clean":"2023-10-09T04:18:35.740118+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:18:35.740118+0000","last_undegraded":"2023-10-09T04:18:35.740118+0000","last_fullsized":"2023-10-09T04:18:35.740118+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2288,"log_dups_size":0,"ondisk_log_size":2288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149946368,"num_objects":2288,"num_object_clones":0,"num_object_copies":6864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2288,"num_write_kb":146432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":443,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831691+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:18:30.831691+0000","last_peered":"2023-10-09T04:18:30.831691+0000","last_clean":"2023-10-09T04:18:30.831691+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:18:30.831691+0000","last_undegraded":"2023-10-09T04:18:30.831691+0000","last_fullsized":"2023-10-09T04:18:30.831691+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":63,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834361+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:18:30.834361+0000","last_peered":"2023-10-09T04:18:30.834361+0000","last_clean":"2023-10-09T04:18:30.834361+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:18:30.834361+0000","last_undegraded":"2023-10-09T04:18:30.834361+0000","last_fullsized":"2023-10-09T04:18:30.834361+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":103,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.836201+0000","last_change":"2023-10-09T04:18:28.823874+0000","last_active":"2023-10-09T04:18:30.836201+0000","last_peered":"2023-10-09T04:18:30.836201+0000","last_clean":"2023-10-09T04:18:30.836201+0000","last_became_active":"2023-10-09T04:18:28.823517+0000","last_became_peered":"2023-10-09T04:18:28.823517+0000","last_unstale":"2023-10-09T04:18:30.836201+0000","last_undegraded":"2023-10-09T04:18:30.836201+0000","last_fullsized":"2023-10-09T04:18:30.836201+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:21.354274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"319'2450","reported_seq":2472,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:38.173014+0000","last_change":"2023-10-09T04:18:07.756988+0000","last_active":"2023-10-09T04:18:38.173014+0000","last_peered":"2023-10-09T04:18:38.173014+0000","last_clean":"2023-10-09T04:18:38.173014+0000","last_became_active":"2023-10-09T04:18:07.756688+0000","last_became_peered":"2023-10-09T04:18:07.756688+0000","last_unstale":"2023-10-09T04:18:38.173014+0000","last_undegraded":"2023-10-09T04:18:38.173014+0000","last_fullsized":"2023-10-09T04:18:38.173014+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2450,"log_dups_size":0,"ondisk_log_size":2450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:43.014457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160563200,"num_objects":2450,"num_object_clones":0,"num_object_copies":7350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2450,"num_write_kb":156800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":453,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.850140+0000","last_change":"2023-10-09T04:18:31.850140+0000","last_active":"2023-10-09T04:18:31.850140+0000","last_peered":"2023-10-09T04:18:31.850140+0000","last_clean":"2023-10-09T04:18:31.850140+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:18:31.850140+0000","last_undegraded":"2023-10-09T04:18:31.850140+0000","last_fullsized":"2023-10-09T04:18:31.850140+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:02.097241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:02.097241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":108,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.838029+0000","last_change":"2023-10-09T04:16:24.471755+0000","last_active":"2023-10-09T04:18:31.838029+0000","last_peered":"2023-10-09T04:18:31.838029+0000","last_clean":"2023-10-09T04:18:31.838029+0000","last_became_active":"2023-10-09T04:16:24.471509+0000","last_became_peered":"2023-10-09T04:16:24.471509+0000","last_unstale":"2023-10-09T04:18:31.838029+0000","last_undegraded":"2023-10-09T04:18:31.838029+0000","last_fullsized":"2023-10-09T04:18:31.838029+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:57.228210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":310,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831562+0000","last_change":"2023-10-09T04:18:15.729879+0000","last_active":"2023-10-09T04:18:30.831562+0000","last_peered":"2023-10-09T04:18:30.831562+0000","last_clean":"2023-10-09T04:18:30.831562+0000","last_became_active":"2023-10-09T04:18:15.729550+0000","last_became_peered":"2023-10-09T04:18:15.729550+0000","last_unstale":"2023-10-09T04:18:30.831562+0000","last_undegraded":"2023-10-09T04:18:30.831562+0000","last_fullsized":"2023-10-09T04:18:30.831562+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:09:08.495101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"319'2260","reported_seq":2291,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.790050+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:18:35.790050+0000","last_peered":"2023-10-09T04:18:35.790050+0000","last_clean":"2023-10-09T04:18:35.790050+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:18:35.790050+0000","last_undegraded":"2023-10-09T04:18:35.790050+0000","last_fullsized":"2023-10-09T04:18:35.790050+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2260,"log_dups_size":0,"ondisk_log_size":2260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148111360,"num_objects":2260,"num_object_clones":0,"num_object_copies":6780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2260,"num_write_kb":144640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":463,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.849253+0000","last_change":"2023-10-09T04:18:31.849253+0000","last_active":"2023-10-09T04:18:31.849253+0000","last_peered":"2023-10-09T04:18:31.849253+0000","last_clean":"2023-10-09T04:18:31.849253+0000","last_became_active":"2023-10-09T04:18:31.848599+0000","last_became_peered":"2023-10-09T04:18:31.848599+0000","last_unstale":"2023-10-09T04:18:31.849253+0000","last_undegraded":"2023-10-09T04:18:31.849253+0000","last_fullsized":"2023-10-09T04:18:31.849253+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:47.206426+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":297,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834057+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:18:30.834057+0000","last_peered":"2023-10-09T04:18:30.834057+0000","last_clean":"2023-10-09T04:18:30.834057+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:18:30.834057+0000","last_undegraded":"2023-10-09T04:18:30.834057+0000","last_fullsized":"2023-10-09T04:18:30.834057+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845228+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:18:30.845228+0000","last_peered":"2023-10-09T04:18:30.845228+0000","last_clean":"2023-10-09T04:18:30.845228+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:18:30.845228+0000","last_undegraded":"2023-10-09T04:18:30.845228+0000","last_fullsized":"2023-10-09T04:18:30.845228+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":269,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845842+0000","last_change":"2023-10-09T04:16:24.473165+0000","last_active":"2023-10-09T04:18:30.845842+0000","last_peered":"2023-10-09T04:18:30.845842+0000","last_clean":"2023-10-09T04:18:30.845842+0000","last_became_active":"2023-10-09T04:16:24.472923+0000","last_became_peered":"2023-10-09T04:16:24.472923+0000","last_unstale":"2023-10-09T04:18:30.845842+0000","last_undegraded":"2023-10-09T04:18:30.845842+0000","last_fullsized":"2023-10-09T04:18:30.845842+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:01.846352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"319'2350","reported_seq":2370,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:36.681500+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:18:36.681500+0000","last_peered":"2023-10-09T04:18:36.681500+0000","last_clean":"2023-10-09T04:18:36.681500+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:18:36.681500+0000","last_undegraded":"2023-10-09T04:18:36.681500+0000","last_fullsized":"2023-10-09T04:18:36.681500+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2350,"log_dups_size":0,"ondisk_log_size":2350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154009600,"num_objects":2350,"num_object_clones":0,"num_object_copies":7050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2350,"num_write_kb":150400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":433,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843567+0000","last_change":"2023-10-09T04:18:04.684770+0000","last_active":"2023-10-09T04:18:30.843567+0000","last_peered":"2023-10-09T04:18:30.843567+0000","last_clean":"2023-10-09T04:18:30.843567+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:18:30.843567+0000","last_undegraded":"2023-10-09T04:18:30.843567+0000","last_fullsized":"2023-10-09T04:18:30.843567+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":292,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845708+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:18:30.845708+0000","last_peered":"2023-10-09T04:18:30.845708+0000","last_clean":"2023-10-09T04:18:30.845708+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:18:30.845708+0000","last_undegraded":"2023-10-09T04:18:30.845708+0000","last_fullsized":"2023-10-09T04:18:30.845708+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834092+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:18:30.834092+0000","last_peered":"2023-10-09T04:18:30.834092+0000","last_clean":"2023-10-09T04:18:30.834092+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:18:30.834092+0000","last_undegraded":"2023-10-09T04:18:30.834092+0000","last_fullsized":"2023-10-09T04:18:30.834092+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":325,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.833966+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:18:30.833966+0000","last_peered":"2023-10-09T04:18:30.833966+0000","last_clean":"2023-10-09T04:18:30.833966+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:18:30.833966+0000","last_undegraded":"2023-10-09T04:18:30.833966+0000","last_fullsized":"2023-10-09T04:18:30.833966+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"319'2287","reported_seq":2308,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.783799+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:18:35.783799+0000","last_peered":"2023-10-09T04:18:35.783799+0000","last_clean":"2023-10-09T04:18:35.783799+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:18:35.783799+0000","last_undegraded":"2023-10-09T04:18:35.783799+0000","last_fullsized":"2023-10-09T04:18:35.783799+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2287,"log_dups_size":0,"ondisk_log_size":2287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149880832,"num_objects":2287,"num_object_clones":0,"num_object_copies":6861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2287,"num_write_kb":146368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":451,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.828985+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:18:30.828985+0000","last_peered":"2023-10-09T04:18:30.828985+0000","last_clean":"2023-10-09T04:18:30.828985+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:18:30.828985+0000","last_undegraded":"2023-10-09T04:18:30.828985+0000","last_fullsized":"2023-10-09T04:18:30.828985+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":153,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.829133+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:18:30.829133+0000","last_peered":"2023-10-09T04:18:30.829133+0000","last_clean":"2023-10-09T04:18:30.829133+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:18:30.829133+0000","last_undegraded":"2023-10-09T04:18:30.829133+0000","last_fullsized":"2023-10-09T04:18:30.829133+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":301,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.837945+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:18:31.837945+0000","last_peered":"2023-10-09T04:18:31.837945+0000","last_clean":"2023-10-09T04:18:31.837945+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:18:31.837945+0000","last_undegraded":"2023-10-09T04:18:31.837945+0000","last_fullsized":"2023-10-09T04:18:31.837945+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.849096+0000","last_change":"2023-10-09T04:16:24.470074+0000","last_active":"2023-10-09T04:18:30.849096+0000","last_peered":"2023-10-09T04:18:30.849096+0000","last_clean":"2023-10-09T04:18:30.849096+0000","last_became_active":"2023-10-09T04:16:24.469767+0000","last_became_peered":"2023-10-09T04:16:24.469767+0000","last_unstale":"2023-10-09T04:18:30.849096+0000","last_undegraded":"2023-10-09T04:18:30.849096+0000","last_fullsized":"2023-10-09T04:18:30.849096+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:51.823815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"319'2303","reported_seq":2324,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:36.694261+0000","last_change":"2023-10-09T04:18:07.678089+0000","last_active":"2023-10-09T04:18:36.694261+0000","last_peered":"2023-10-09T04:18:36.694261+0000","last_clean":"2023-10-09T04:18:36.694261+0000","last_became_active":"2023-10-09T04:18:07.677798+0000","last_became_peered":"2023-10-09T04:18:07.677798+0000","last_unstale":"2023-10-09T04:18:36.694261+0000","last_undegraded":"2023-10-09T04:18:36.694261+0000","last_fullsized":"2023-10-09T04:18:36.694261+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2303,"log_dups_size":0,"ondisk_log_size":2303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:27.677750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150929408,"num_objects":2303,"num_object_clones":0,"num_object_copies":6909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2303,"num_write_kb":147392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":302,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843436+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:18:30.843436+0000","last_peered":"2023-10-09T04:18:30.843436+0000","last_clean":"2023-10-09T04:18:30.843436+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:18:30.843436+0000","last_undegraded":"2023-10-09T04:18:30.843436+0000","last_fullsized":"2023-10-09T04:18:30.843436+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":713,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.857328+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:18:31.857328+0000","last_peered":"2023-10-09T04:18:31.857328+0000","last_clean":"2023-10-09T04:18:31.857328+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:18:31.857328+0000","last_undegraded":"2023-10-09T04:18:31.857328+0000","last_fullsized":"2023-10-09T04:18:31.857328+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845777+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:18:30.845777+0000","last_peered":"2023-10-09T04:18:30.845777+0000","last_clean":"2023-10-09T04:18:30.845777+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:18:30.845777+0000","last_undegraded":"2023-10-09T04:18:30.845777+0000","last_fullsized":"2023-10-09T04:18:30.845777+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834985+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:18:30.834985+0000","last_peered":"2023-10-09T04:18:30.834985+0000","last_clean":"2023-10-09T04:18:30.834985+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:18:30.834985+0000","last_undegraded":"2023-10-09T04:18:30.834985+0000","last_fullsized":"2023-10-09T04:18:30.834985+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"319'2339","reported_seq":3422,"reported_epoch":319,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:18:36.695553+0000","last_change":"2023-10-09T04:18:29.900468+0000","last_active":"2023-10-09T04:18:36.695553+0000","last_peered":"2023-10-09T04:18:36.695553+0000","last_clean":"2023-10-09T04:18:27.792857+0000","last_became_active":"2023-10-09T04:18:29.874489+0000","last_became_peered":"2023-10-09T04:18:29.874489+0000","last_unstale":"2023-10-09T04:18:36.695553+0000","last_undegraded":"2023-10-09T04:18:36.695553+0000","last_fullsized":"2023-10-09T04:18:29.811044+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2339,"log_dups_size":0,"ondisk_log_size":2339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:56.003248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153288704,"num_objects":2339,"num_object_clones":0,"num_object_copies":7017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1254,"num_objects_unfound":0,"num_objects_dirty":2339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2339,"num_write_kb":149696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":526,"num_bytes_recovered":34471936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":1085},{"shards":"0,5,6","objects":1254}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":461,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.848389+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:18:31.848389+0000","last_peered":"2023-10-09T04:18:31.848389+0000","last_clean":"2023-10-09T04:18:31.848389+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:18:31.848389+0000","last_undegraded":"2023-10-09T04:18:31.848389+0000","last_fullsized":"2023-10-09T04:18:31.848389+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":299,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.832919+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:18:30.832919+0000","last_peered":"2023-10-09T04:18:30.832919+0000","last_clean":"2023-10-09T04:18:30.832919+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:18:30.832919+0000","last_undegraded":"2023-10-09T04:18:30.832919+0000","last_fullsized":"2023-10-09T04:18:30.832919+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":305,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.694644+0000","last_change":"2023-10-09T04:16:24.470732+0000","last_active":"2023-10-09T04:18:31.694644+0000","last_peered":"2023-10-09T04:18:31.694644+0000","last_clean":"2023-10-09T04:18:31.694644+0000","last_became_active":"2023-10-09T04:16:24.470433+0000","last_became_peered":"2023-10-09T04:16:24.470433+0000","last_unstale":"2023-10-09T04:18:31.694644+0000","last_undegraded":"2023-10-09T04:18:31.694644+0000","last_fullsized":"2023-10-09T04:18:31.694644+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:45:54.719857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":310,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830578+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:18:30.830578+0000","last_peered":"2023-10-09T04:18:30.830578+0000","last_clean":"2023-10-09T04:18:30.830578+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:18:30.830578+0000","last_undegraded":"2023-10-09T04:18:30.830578+0000","last_fullsized":"2023-10-09T04:18:30.830578+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"319'2233","reported_seq":2254,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.732757+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:18:35.732757+0000","last_peered":"2023-10-09T04:18:35.732757+0000","last_clean":"2023-10-09T04:18:35.732757+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:18:35.732757+0000","last_undegraded":"2023-10-09T04:18:35.732757+0000","last_fullsized":"2023-10-09T04:18:35.732757+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2233,"log_dups_size":0,"ondisk_log_size":2233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146341888,"num_objects":2233,"num_object_clones":0,"num_object_copies":6699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2233,"num_write_kb":142912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":485,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.848724+0000","last_change":"2023-10-09T04:18:31.848724+0000","last_active":"2023-10-09T04:18:31.848724+0000","last_peered":"2023-10-09T04:18:31.848724+0000","last_clean":"2023-10-09T04:18:31.848724+0000","last_became_active":"2023-10-09T04:18:31.848412+0000","last_became_peered":"2023-10-09T04:18:31.848412+0000","last_unstale":"2023-10-09T04:18:31.848724+0000","last_undegraded":"2023-10-09T04:18:31.848724+0000","last_fullsized":"2023-10-09T04:18:31.848724+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:47.859915+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":302,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.837712+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:18:30.837712+0000","last_peered":"2023-10-09T04:18:30.837712+0000","last_clean":"2023-10-09T04:18:30.837712+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:18:30.837712+0000","last_undegraded":"2023-10-09T04:18:30.837712+0000","last_fullsized":"2023-10-09T04:18:30.837712+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":291,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834149+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:18:30.834149+0000","last_peered":"2023-10-09T04:18:30.834149+0000","last_clean":"2023-10-09T04:18:30.834149+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:18:30.834149+0000","last_undegraded":"2023-10-09T04:18:30.834149+0000","last_fullsized":"2023-10-09T04:18:30.834149+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":262,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830443+0000","last_change":"2023-10-09T04:02:03.589262+0000","last_active":"2023-10-09T04:18:30.830443+0000","last_peered":"2023-10-09T04:18:30.830443+0000","last_clean":"2023-10-09T04:18:30.830443+0000","last_became_active":"2023-10-09T04:02:03.588986+0000","last_became_peered":"2023-10-09T04:02:03.588986+0000","last_unstale":"2023-10-09T04:18:30.830443+0000","last_undegraded":"2023-10-09T04:18:30.830443+0000","last_fullsized":"2023-10-09T04:18:30.830443+0000","mapping_epoch":84,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":85,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:43.214335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"319'2301","reported_seq":2322,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.747755+0000","last_change":"2023-10-09T04:18:07.680633+0000","last_active":"2023-10-09T04:18:35.747755+0000","last_peered":"2023-10-09T04:18:35.747755+0000","last_clean":"2023-10-09T04:18:35.747755+0000","last_became_active":"2023-10-09T04:18:07.680317+0000","last_became_peered":"2023-10-09T04:18:07.680317+0000","last_unstale":"2023-10-09T04:18:35.747755+0000","last_undegraded":"2023-10-09T04:18:35.747755+0000","last_fullsized":"2023-10-09T04:18:35.747755+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2301,"log_dups_size":0,"ondisk_log_size":2301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:32.107095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150798336,"num_objects":2301,"num_object_clones":0,"num_object_copies":6903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2301,"num_write_kb":147264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":463,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.850771+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:18:31.850771+0000","last_peered":"2023-10-09T04:18:31.850771+0000","last_clean":"2023-10-09T04:18:31.850771+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:18:31.850771+0000","last_undegraded":"2023-10-09T04:18:31.850771+0000","last_fullsized":"2023-10-09T04:18:31.850771+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":251,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.833405+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:18:30.833405+0000","last_peered":"2023-10-09T04:18:30.833405+0000","last_clean":"2023-10-09T04:18:30.833405+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:18:30.833405+0000","last_undegraded":"2023-10-09T04:18:30.833405+0000","last_fullsized":"2023-10-09T04:18:30.833405+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":285,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.832410+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:18:30.832410+0000","last_peered":"2023-10-09T04:18:30.832410+0000","last_clean":"2023-10-09T04:18:30.832410+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:18:30.832410+0000","last_undegraded":"2023-10-09T04:18:30.832410+0000","last_fullsized":"2023-10-09T04:18:30.832410+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.835495+0000","last_change":"2023-10-09T04:16:24.469186+0000","last_active":"2023-10-09T04:18:30.835495+0000","last_peered":"2023-10-09T04:18:30.835495+0000","last_clean":"2023-10-09T04:18:30.835495+0000","last_became_active":"2023-10-09T04:16:24.468836+0000","last_became_peered":"2023-10-09T04:16:24.468836+0000","last_unstale":"2023-10-09T04:18:30.835495+0000","last_undegraded":"2023-10-09T04:18:30.835495+0000","last_fullsized":"2023-10-09T04:18:30.835495+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:42.902638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"319'2288","reported_seq":2319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:35.373406+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:18:35.373406+0000","last_peered":"2023-10-09T04:18:35.373406+0000","last_clean":"2023-10-09T04:18:35.373406+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:18:35.373406+0000","last_undegraded":"2023-10-09T04:18:35.373406+0000","last_fullsized":"2023-10-09T04:18:35.373406+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":2288,"log_dups_size":0,"ondisk_log_size":2288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149946368,"num_objects":2288,"num_object_clones":0,"num_object_copies":6864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2288,"num_write_kb":146432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":456,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.833035+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:18:30.833035+0000","last_peered":"2023-10-09T04:18:30.833035+0000","last_clean":"2023-10-09T04:18:30.833035+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:18:30.833035+0000","last_undegraded":"2023-10-09T04:18:30.833035+0000","last_fullsized":"2023-10-09T04:18:30.833035+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":150,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831551+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:18:30.831551+0000","last_peered":"2023-10-09T04:18:30.831551+0000","last_clean":"2023-10-09T04:18:30.831551+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:18:30.831551+0000","last_undegraded":"2023-10-09T04:18:30.831551+0000","last_fullsized":"2023-10-09T04:18:30.831551+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":310,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830668+0000","last_change":"2023-10-09T04:16:20.109259+0000","last_active":"2023-10-09T04:18:30.830668+0000","last_peered":"2023-10-09T04:18:30.830668+0000","last_clean":"2023-10-09T04:18:30.830668+0000","last_became_active":"2023-10-09T04:15:25.496763+0000","last_became_peered":"2023-10-09T04:15:25.496763+0000","last_unstale":"2023-10-09T04:18:30.830668+0000","last_undegraded":"2023-10-09T04:18:30.830668+0000","last_fullsized":"2023-10-09T04:18:30.830668+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:15.807543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834557+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:18:30.834557+0000","last_peered":"2023-10-09T04:18:30.834557+0000","last_clean":"2023-10-09T04:18:30.834557+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:18:30.834557+0000","last_undegraded":"2023-10-09T04:18:30.834557+0000","last_fullsized":"2023-10-09T04:18:30.834557+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":465,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.843203+0000","last_change":"2023-10-09T04:18:31.843203+0000","last_active":"2023-10-09T04:18:31.843203+0000","last_peered":"2023-10-09T04:18:31.843203+0000","last_clean":"2023-10-09T04:18:31.843203+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:18:31.843203+0000","last_undegraded":"2023-10-09T04:18:31.843203+0000","last_fullsized":"2023-10-09T04:18:31.843203+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:01.099879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:01.099879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":283,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.830367+0000","last_change":"2023-10-09T04:15:25.414363+0000","last_active":"2023-10-09T04:18:30.830367+0000","last_peered":"2023-10-09T04:18:30.830367+0000","last_clean":"2023-10-09T04:18:30.830367+0000","last_became_active":"2023-10-09T04:15:25.413987+0000","last_became_peered":"2023-10-09T04:15:25.413987+0000","last_unstale":"2023-10-09T04:18:30.830367+0000","last_undegraded":"2023-10-09T04:18:30.830367+0000","last_fullsized":"2023-10-09T04:18:30.830367+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:45.394691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.829615+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:18:30.829615+0000","last_peered":"2023-10-09T04:18:30.829615+0000","last_clean":"2023-10-09T04:18:30.829615+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:18:30.829615+0000","last_undegraded":"2023-10-09T04:18:30.829615+0000","last_fullsized":"2023-10-09T04:18:30.829615+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":696,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.854487+0000","last_change":"2023-10-09T04:18:31.854487+0000","last_active":"2023-10-09T04:18:31.854487+0000","last_peered":"2023-10-09T04:18:31.854487+0000","last_clean":"2023-10-09T04:18:31.854487+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:18:31.854487+0000","last_undegraded":"2023-10-09T04:18:31.854487+0000","last_fullsized":"2023-10-09T04:18:31.854487+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:18.899158+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:18.899158+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":308,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.843398+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:18:30.843398+0000","last_peered":"2023-10-09T04:18:30.843398+0000","last_clean":"2023-10-09T04:18:30.843398+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:18:30.843398+0000","last_undegraded":"2023-10-09T04:18:30.843398+0000","last_fullsized":"2023-10-09T04:18:30.843398+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834249+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:18:30.834249+0000","last_peered":"2023-10-09T04:18:30.834249+0000","last_clean":"2023-10-09T04:18:30.834249+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:18:30.834249+0000","last_undegraded":"2023-10-09T04:18:30.834249+0000","last_fullsized":"2023-10-09T04:18:30.834249+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":305,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.845539+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:18:30.845539+0000","last_peered":"2023-10-09T04:18:30.845539+0000","last_clean":"2023-10-09T04:18:30.845539+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:18:30.845539+0000","last_undegraded":"2023-10-09T04:18:30.845539+0000","last_fullsized":"2023-10-09T04:18:30.845539+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.834646+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:18:30.834646+0000","last_peered":"2023-10-09T04:18:30.834646+0000","last_clean":"2023-10-09T04:18:30.834646+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:18:30.834646+0000","last_undegraded":"2023-10-09T04:18:30.834646+0000","last_fullsized":"2023-10-09T04:18:30.834646+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":741,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.847724+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:18:31.847724+0000","last_peered":"2023-10-09T04:18:31.847724+0000","last_clean":"2023-10-09T04:18:31.847724+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:18:31.847724+0000","last_undegraded":"2023-10-09T04:18:31.847724+0000","last_fullsized":"2023-10-09T04:18:31.847724+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":162,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.829954+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:18:30.829954+0000","last_peered":"2023-10-09T04:18:30.829954+0000","last_clean":"2023-10-09T04:18:30.829954+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:18:30.829954+0000","last_undegraded":"2023-10-09T04:18:30.829954+0000","last_fullsized":"2023-10-09T04:18:30.829954+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.829202+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:18:30.829202+0000","last_peered":"2023-10-09T04:18:30.829202+0000","last_clean":"2023-10-09T04:18:30.829202+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:18:30.829202+0000","last_undegraded":"2023-10-09T04:18:30.829202+0000","last_fullsized":"2023-10-09T04:18:30.829202+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":443,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.831126+0000","last_change":"2023-10-09T04:18:24.731948+0000","last_active":"2023-10-09T04:18:30.831126+0000","last_peered":"2023-10-09T04:18:30.831126+0000","last_clean":"2023-10-09T04:18:30.831126+0000","last_became_active":"2023-10-09T04:18:04.670874+0000","last_became_peered":"2023-10-09T04:18:04.670874+0000","last_unstale":"2023-10-09T04:18:30.831126+0000","last_undegraded":"2023-10-09T04:18:30.831126+0000","last_fullsized":"2023-10-09T04:18:30.831126+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:31.506985+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":312,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:31.696023+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:18:31.696023+0000","last_peered":"2023-10-09T04:18:31.696023+0000","last_clean":"2023-10-09T04:18:31.696023+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:18:31.696023+0000","last_undegraded":"2023-10-09T04:18:31.696023+0000","last_fullsized":"2023-10-09T04:18:31.696023+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":73,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T04:18:30.833550+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:18:30.833550+0000","last_peered":"2023-10-09T04:18:30.833550+0000","last_clean":"2023-10-09T04:18:30.833550+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:18:30.833550+0000","last_undegraded":"2023-10-09T04:18:30.833550+0000","last_fullsized":"2023-10-09T04:18:30.833550+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":11118,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":564722,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":197,"num_read_kb":368,"num_write":97,"num_write_kb":971,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1301494,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2428436480,"num_objects":37055,"num_object_clones":0,"num_object_copies":111165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1746,"num_objects_misplaced":2115,"num_objects_unfound":0,"num_objects_dirty":37055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":37055,"num_write_kb":2371520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1335,"num_bytes_recovered":87490560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7425032192,"data_stored":7425032192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37055,"ondisk_log_size":37055,"up":48,"acting":45,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201593,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1902088,"kb_used_data":1604800,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":91843064,"statfs":{"total":95995035648,"available":94047297536,"internally_reserved":0,"allocated":1643315200,"data_stored":1642380504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":3.1549999999999998,"15min":2.7429999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.20799999999999999},"max":{"1min":8.9100000000000001,"5min":88.144999999999996,"15min":126.752},"last":1.105},{"interface":"front","average":{"1min":2.331,"5min":2.9199999999999999,"15min":2.6779999999999999},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.4829999999999997,"5min":89.284999999999997,"15min":128.88399999999999},"last":2.4769999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":3.839,"15min":3.8959999999999999},"min":{"1min":0.28399999999999997,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":8.8119999999999994,"5min":103.324,"15min":245.28700000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.77,"5min":3.718,"15min":3.7850000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.047000000000001,"5min":104.248,"15min":244.626},"last":2.2599999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2309999999999999,"5min":2.4969999999999999,"15min":1.9690000000000001},"min":{"1min":0.29799999999999999,"5min":0.27800000000000002,"15min":0.27400000000000002},"max":{"1min":14.076000000000001,"5min":76.774000000000001,"15min":76.774000000000001},"last":0.95899999999999996},{"interface":"front","average":{"1min":4.1580000000000004,"5min":3.0779999999999998,"15min":2.5939999999999999},"min":{"1min":0.47499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":14.061999999999999,"5min":76.715999999999994,"15min":99.372},"last":2.4140000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1739999999999999,"5min":2.9350000000000001,"15min":2.5219999999999998},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":8.8550000000000004,"5min":85.575000000000003,"15min":99.134},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.1019999999999999,"5min":3.1619999999999999,"15min":2.6880000000000002},"min":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.0489999999999995,"5min":85.528000000000006,"15min":99.117999999999995},"last":2.2370000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":0.72499999999999998,"15min":0.77900000000000003},"min":{"1min":0.27000000000000002,"5min":0.252,"15min":0.23400000000000001},"max":{"1min":3.5219999999999998,"5min":4.7110000000000003,"15min":16.195},"last":0.377},{"interface":"front","average":{"1min":1.756,"5min":1.2529999999999999,"15min":0.96899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.9500000000000002,"5min":9.282,"15min":12.442},"last":2.214}]},{"osd":5,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":2.302,"15min":1.2190000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":6.6390000000000002,"5min":120.066,"15min":120.066},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.585,"5min":2.1299999999999999,"15min":1.2110000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.1369999999999996,"5min":120.09699999999999,"15min":120.09699999999999},"last":0.55200000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":1.248,"15min":0.93799999999999994},"min":{"1min":0.31900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.6820000000000004,"5min":8.452,"15min":8.452},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.3500000000000001,"5min":0.89100000000000001,"15min":0.86499999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.101,"5min":7.101,"15min":7.101},"last":0.66200000000000003}]}]},{"osd":6,"up_from":0,"seq":201863463230,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089539,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1838100,"kb_used_data":1397452,"kb_used_omap":9,"kb_used_meta":440630,"kb_avail":91907052,"statfs":{"total":95995035648,"available":94112821248,"internally_reserved":0,"allocated":1430990848,"data_stored":1429637485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":451206121},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":9.5830000000000002,"5min":4.726,"15min":3.8660000000000001},"min":{"1min":0.23300000000000001,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":98.257999999999996,"5min":98.257999999999996,"15min":98.257999999999996},"last":0.63200000000000001},{"interface":"front","average":{"1min":20.742999999999999,"5min":7.1580000000000004,"15min":4.9850000000000003},"min":{"1min":0.312,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":154.654,"5min":154.654,"15min":154.654},"last":0.215}]},{"osd":2,"last update":"Mon Oct 9 04:18:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.8169999999999999,"15min":2.4489999999999998},"min":{"1min":0.27400000000000002,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":4.077,"5min":14.115,"15min":121.54600000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.0980000000000001,"5min":2.3639999999999999,"15min":2.8149999999999999},"min":{"1min":0.31900000000000001,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":4.5170000000000003,"5min":16.114999999999998,"15min":99.899000000000001},"last":0.372}]},{"osd":3,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":3.8010000000000002,"5min":2.923,"15min":3.577},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":15.259,"5min":68.054000000000002,"15min":138.87299999999999},"last":2.286},{"interface":"front","average":{"1min":4.7000000000000002,"5min":3.9209999999999998,"15min":4.4470000000000001},"min":{"1min":0.34499999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":20.975000000000001,"5min":68.563999999999993,"15min":138.49799999999999},"last":0.251}]},{"osd":4,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":2.7269999999999999,"5min":2.173,"15min":2.1819999999999999},"min":{"1min":0.375,"5min":0.32200000000000001,"15min":0.254},"max":{"1min":16.27,"5min":16.27,"15min":99.989999999999995},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.966,"5min":1.917,"15min":2.0619999999999998},"min":{"1min":0.32900000000000001,"5min":0.318,"15min":0.25900000000000001},"max":{"1min":5.367,"5min":13.428000000000001,"15min":100.249},"last":0.28499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":5.5970000000000004,"5min":2.8599999999999999,"15min":2.375},"min":{"1min":0.434,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":38.848999999999997,"5min":38.848999999999997,"15min":99.933999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":5.492,"5min":2.859,"15min":2.48},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":39.593000000000004,"5min":39.593000000000004,"15min":100.34099999999999},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1429999999999998,"5min":2.1920000000000002,"15min":2.3530000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":5.4379999999999997,"5min":13.377000000000001,"15min":100.309},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.853,"15min":2.048},"min":{"1min":0.41099999999999998,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":14.141999999999999,"5min":14.141999999999999,"15min":100.143},"last":0.69199999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":3.8719999999999999,"5min":2.5529999999999999,"15min":2.4500000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":17.460000000000001,"5min":17.460000000000001,"15min":100.214},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.101,"5min":2.2290000000000001,"15min":2.1850000000000001},"min":{"1min":0.45000000000000001,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":14.167,"5min":14.167,"15min":100.113},"last":0.66100000000000003}]}]},{"osd":0,"up_from":84,"seq":360777253065,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097880,"kb_used_data":612048,"kb_used_omap":28,"kb_used_meta":485795,"kb_avail":92647272,"statfs":{"total":95995035648,"available":94870806528,"internally_reserved":0,"allocated":626737152,"data_stored":625319604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":497454928},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":9.2669999999999995,"5min":6.1100000000000003,"15min":4.2169999999999996},"min":{"1min":0.23799999999999999,"5min":0.187,"15min":0.17000000000000001},"max":{"1min":92.066999999999993,"5min":120.511,"15min":120.511},"last":0.26000000000000001},{"interface":"front","average":{"1min":8.5030000000000001,"5min":6.4329999999999998,"15min":4.5880000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.214},"max":{"1min":94.894000000000005,"5min":120.934,"15min":120.934},"last":0.248}]},{"osd":2,"last update":"Mon Oct 9 04:17:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5659999999999998,"5min":2.4969999999999999,"15min":2.9420000000000002},"min":{"1min":0.26600000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":17.466999999999999,"5min":17.466999999999999,"15min":118.54000000000001},"last":0.309},{"interface":"front","average":{"1min":5.2290000000000001,"5min":3.4830000000000001,"15min":3.3889999999999998},"min":{"1min":0.217,"5min":0.21099999999999999,"15min":0.20300000000000001},"max":{"1min":20.262,"5min":20.262,"15min":118.727},"last":0.22800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5230000000000001,"5min":4.0730000000000004,"15min":5.109},"min":{"1min":0.26600000000000001,"5min":0.219,"15min":0.20999999999999999},"max":{"1min":10.276,"5min":99.709000000000003,"15min":156.488},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.4089999999999998,"5min":4.2089999999999996,"15min":5.1790000000000003},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.22500000000000001},"max":{"1min":9.1300000000000008,"5min":99.560000000000002,"15min":156.50700000000001},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:17:42 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.7989999999999999,"15min":2.2120000000000002},"min":{"1min":0.35299999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":9.0519999999999996,"5min":9.0519999999999996,"15min":118.633},"last":0.55800000000000005},{"interface":"front","average":{"1min":4.2590000000000003,"5min":2.6840000000000002,"15min":2.73},"min":{"1min":0.40699999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":19.527000000000001,"5min":19.527000000000001,"15min":118.913},"last":3.754}]},{"osd":5,"last update":"Mon Oct 9 04:17:42 2023","interfaces":[{"interface":"back","average":{"1min":4.2279999999999998,"5min":4.843,"15min":3.9590000000000001},"min":{"1min":0.371,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.9770000000000003,"5min":156.96100000000001,"15min":182.68899999999999},"last":0.91700000000000004},{"interface":"front","average":{"1min":3.6190000000000002,"5min":3.8479999999999999,"15min":3.4569999999999999},"min":{"1min":0.47499999999999998,"5min":0.28399999999999997,"15min":0.28100000000000003},"max":{"1min":10.249000000000001,"5min":156.97300000000001,"15min":182.66300000000001},"last":1.5900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:17:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":1.804,"15min":2.6819999999999999},"min":{"1min":0.42699999999999999,"5min":0.33400000000000002,"15min":0.30599999999999999},"max":{"1min":10.119999999999999,"5min":10.797000000000001,"15min":137.36099999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.016,"5min":1.96,"15min":2.7050000000000001},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":9.1050000000000004,"5min":9.1050000000000004,"15min":136.88999999999999},"last":2.9740000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:17:42 2023","interfaces":[{"interface":"back","average":{"1min":3.8860000000000001,"5min":2.0169999999999999,"15min":3.3730000000000002},"min":{"1min":0.44900000000000001,"5min":0.39200000000000002,"15min":0.32100000000000001},"max":{"1min":10.156000000000001,"5min":10.77,"15min":185.024},"last":1.8879999999999999},{"interface":"front","average":{"1min":3.1480000000000001,"5min":2.0939999999999999,"15min":2.9830000000000001},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":9.1959999999999997,"5min":9.1959999999999997,"15min":184.98599999999999},"last":3.0139999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926013,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1241508,"kb_used_data":737500,"kb_used_omap":81,"kb_used_meta":503918,"kb_avail":92503644,"statfs":{"total":95995035648,"available":94723731456,"internally_reserved":0,"allocated":755200000,"data_stored":753835124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":516012250},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":8.3719999999999999,"5min":6.4080000000000004,"15min":3.8199999999999998},"min":{"1min":0.25900000000000001,"5min":0.215,"15min":0.17399999999999999},"max":{"1min":57.241,"5min":81.385999999999996,"15min":81.385999999999996},"last":1.194},{"interface":"front","average":{"1min":7.6299999999999999,"5min":5.6079999999999997,"15min":3.8050000000000002},"min":{"1min":0.25,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":61.363,"5min":79.453999999999994,"15min":79.453999999999994},"last":1.0349999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":7.452,"5min":4.9619999999999997,"15min":3.641},"min":{"1min":0.23100000000000001,"5min":0.191,"15min":0.191},"max":{"1min":58.994,"5min":75.067999999999998,"15min":75.067999999999998},"last":1.8360000000000001},{"interface":"front","average":{"1min":8.2059999999999995,"5min":5.585,"15min":3.9279999999999999},"min":{"1min":0.29699999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":58.881,"5min":80.168000000000006,"15min":80.168000000000006},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:17:53 2023","interfaces":[{"interface":"back","average":{"1min":6.657,"5min":3.27,"15min":2.5329999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":58.997,"5min":58.997,"15min":58.997},"last":1.3979999999999999},{"interface":"front","average":{"1min":5.8230000000000004,"5min":2.8420000000000001,"15min":2.6840000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":66.135000000000005,"5min":66.135000000000005,"15min":66.135000000000005},"last":0.65700000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":2.5129999999999999,"15min":2.0960000000000001},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":58.798000000000002,"5min":58.798000000000002,"15min":58.798000000000002},"last":1.4590000000000001},{"interface":"front","average":{"1min":5.6059999999999999,"5min":2.7090000000000001,"15min":1.9890000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":56.616,"5min":56.616,"15min":56.616},"last":1.429}]},{"osd":5,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":4.7539999999999996,"5min":2.6680000000000001,"15min":2.2189999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":56.524000000000001,"5min":56.524000000000001,"15min":56.524000000000001},"last":2.4609999999999999},{"interface":"front","average":{"1min":4.2759999999999998,"5min":2.5859999999999999,"15min":2.2570000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":58.854999999999997,"5min":58.854999999999997,"15min":58.854999999999997},"last":1.988}]},{"osd":6,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":5.6139999999999999,"5min":2.8820000000000001,"15min":2.4689999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":56.561,"5min":56.561,"15min":114.14},"last":3.1829999999999998},{"interface":"front","average":{"1min":4.0960000000000001,"5min":2.4420000000000002,"15min":2.5019999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":56.460999999999999,"5min":56.460999999999999,"15min":114.123},"last":0.69099999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":2.988,"15min":2.6800000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":56.591000000000001,"5min":56.591000000000001,"15min":117.32599999999999},"last":1.5189999999999999},{"interface":"front","average":{"1min":4.7590000000000003,"5min":2.6749999999999998,"15min":2.6949999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":56.497999999999998,"5min":56.497999999999998,"15min":117.28700000000001},"last":0.83699999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762486,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1341140,"kb_used_data":691468,"kb_used_omap":7,"kb_used_meta":649656,"kb_avail":92404012,"statfs":{"total":95995035648,"available":94621708288,"internally_reserved":0,"allocated":708063232,"data_stored":706708451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":665248360},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6789999999999998,"5min":5.5250000000000004,"15min":5.2469999999999999},"min":{"1min":0.253,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":14.101000000000001,"5min":125.765,"15min":125.765},"last":39.718000000000004},{"interface":"front","average":{"1min":4.6100000000000003,"5min":5.875,"15min":5.5209999999999999},"min":{"1min":0.307,"5min":0.193,"15min":0.193},"max":{"1min":11.314,"5min":125.46899999999999,"15min":125.46899999999999},"last":38.786000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":8.5589999999999993,"5min":7.3010000000000002,"15min":4.8079999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":73.665000000000006,"5min":186.21299999999999,"15min":186.21299999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":7.5609999999999999,"5min":6.8559999999999999,"15min":4.5019999999999998},"min":{"1min":0.28299999999999997,"5min":0.23000000000000001,"15min":0.193},"max":{"1min":73.817999999999998,"5min":191.78,"15min":191.78},"last":0.253}]},{"osd":2,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":9.4510000000000005,"5min":4.3739999999999997,"15min":3.702},"min":{"1min":0.32600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":127.599,"5min":127.599,"15min":127.599},"last":0.28799999999999998},{"interface":"front","average":{"1min":11.304,"5min":5.9969999999999999,"15min":4.6609999999999996},"min":{"1min":0.28100000000000003,"5min":0.214,"15min":0.214},"max":{"1min":129.98599999999999,"5min":129.98599999999999,"15min":129.98599999999999},"last":0.30299999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5430000000000001,"5min":1.9850000000000001,"15min":2.556},"min":{"1min":0.33300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.6609999999999996,"5min":27.539999999999999,"15min":106.85899999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.673,"5min":2.4870000000000001,"15min":2.1019999999999999},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":9.7170000000000005,"5min":26.663,"15min":37.442},"last":0.36699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5470000000000002,"5min":2.7669999999999999,"15min":2.105},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":12.144,"5min":17.901,"15min":17.901},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.9730000000000001,"15min":1.913},"min":{"1min":0.41399999999999998,"5min":0.248,"15min":0.248},"max":{"1min":4.7560000000000002,"5min":18.068000000000001,"15min":18.068000000000001},"last":0.45400000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.9630000000000001,"15min":2.2749999999999999},"min":{"1min":0.438,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6989999999999998,"5min":14.976000000000001,"15min":106.036},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.8580000000000001,"5min":2.169,"15min":2.4780000000000002},"min":{"1min":0.39700000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.7290000000000001,"5min":14.529,"15min":106.741},"last":0.379}]},{"osd":7,"last update":"Mon Oct 9 04:18:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":2.089,"15min":2.4900000000000002},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":10.789,"5min":14.291,"15min":106.82599999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.9500000000000002,"5min":2.6379999999999999,"15min":2.5379999999999998},"min":{"1min":0.40899999999999997,"5min":0.38900000000000001,"15min":0.35999999999999999},"max":{"1min":9.7479999999999993,"5min":20.757999999999999,"15min":106.084},"last":0.504}]}]},{"osd":4,"up_from":31,"seq":133143986502,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":941948,"kb_used_data":632372,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92803204,"statfs":{"total":95995035648,"available":95030480896,"internally_reserved":0,"allocated":647548928,"data_stored":646625641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":10.032999999999999,"5min":8.5579999999999998,"15min":4.8029999999999999},"min":{"1min":0.30099999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":95.796000000000006,"5min":154.62100000000001,"15min":154.62100000000001},"last":61.409999999999997},{"interface":"front","average":{"1min":9.4600000000000009,"5min":8.7370000000000001,"15min":4.8159999999999998},"min":{"1min":0.316,"5min":0.25,"15min":0.23400000000000001},"max":{"1min":95.655000000000001,"5min":154.37299999999999,"15min":154.37299999999999},"last":61.170999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":6.7060000000000004,"15min":3.5990000000000002},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.23499999999999999},"max":{"1min":8.7590000000000003,"5min":117.81,"15min":117.81},"last":9.0199999999999996},{"interface":"front","average":{"1min":2.456,"5min":6.29,"15min":3.508},"min":{"1min":0.254,"5min":0.251,"15min":0.251},"max":{"1min":9.5999999999999996,"5min":117.848,"15min":117.848},"last":8.9039999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:17:38 2023","interfaces":[{"interface":"back","average":{"1min":3.077,"5min":3.0499999999999998,"15min":2.0019999999999998},"min":{"1min":0.39200000000000002,"5min":0.26500000000000001,"15min":0.245},"max":{"1min":8.468,"5min":123.598,"15min":123.598},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.863,"5min":2.899,"15min":1.909},"min":{"1min":0.308,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":8.4800000000000004,"5min":123.619,"15min":123.619},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":2.0910000000000002,"15min":2.1280000000000001},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.27100000000000002},"max":{"1min":7.4930000000000003,"5min":28.056000000000001,"15min":125.90300000000001},"last":0.438},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.137,"15min":2.286},"min":{"1min":0.41999999999999998,"5min":0.32600000000000001,"15min":0.29099999999999998},"max":{"1min":8.9169999999999998,"5min":26.82,"15min":127.90000000000001},"last":0.41699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:17:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":3.21,"15min":1.4890000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.0860000000000003,"5min":111.809,"15min":111.809},"last":0.375},{"interface":"front","average":{"1min":0.79600000000000004,"5min":3.3319999999999999,"15min":1.6519999999999999},"min":{"1min":0.28299999999999997,"5min":0.255,"15min":0.25},"max":{"1min":2.9980000000000002,"5min":112.002,"15min":112.002},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:17:59 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":2.1110000000000002,"15min":1.268},"min":{"1min":0.33000000000000002,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":3.3919999999999999,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.501},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.909,"15min":1.133},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.6769999999999996,"5min":78.421999999999997,"15min":78.421999999999997},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:18:25 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.1859999999999999,"15min":0.91200000000000003},"min":{"1min":0.32900000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.117,"5min":34.039000000000001,"15min":34.039000000000001},"last":0.62},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.2450000000000001,"15min":0.98299999999999998},"min":{"1min":0.34899999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.1930000000000001,"5min":33.433999999999997,"15min":33.433999999999997},"last":0.64700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724866,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1449456,"kb_used_data":1148072,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92295696,"statfs":{"total":95995035648,"available":94510792704,"internally_reserved":0,"allocated":1175625728,"data_stored":1174710959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:17:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0659999999999998,"5min":5.9699999999999998,"15min":4.9619999999999997},"min":{"1min":0.21299999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.44,"5min":118.98399999999999,"15min":151.23599999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.7679999999999998,"5min":6.4240000000000004,"15min":5.2880000000000003},"min":{"1min":0.26300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":11.016999999999999,"5min":118.846,"15min":151.566},"last":0.34799999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4900000000000002,"5min":5.0149999999999997,"15min":3.6680000000000001},"min":{"1min":0.29599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":19.472000000000001,"5min":111.194,"15min":111.194},"last":32.073999999999998},{"interface":"front","average":{"1min":3.2999999999999998,"5min":5.4370000000000003,"15min":3.9950000000000001},"min":{"1min":0.35099999999999998,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":20.004000000000001,"5min":112.226,"15min":112.226},"last":32.432000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:17:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5430000000000001,"5min":3.9809999999999999,"15min":3.0059999999999998},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.23499999999999999},"max":{"1min":11.324,"5min":106.508,"15min":106.508},"last":1.087},{"interface":"front","average":{"1min":2.2349999999999999,"5min":3.3410000000000002,"15min":2.8180000000000001},"min":{"1min":0.29699999999999999,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":6.7889999999999997,"5min":106.462,"15min":106.462},"last":0.41799999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.27,"15min":2.4460000000000002},"min":{"1min":0.32500000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.4019999999999992,"5min":27.925000000000001,"15min":91.834999999999994},"last":0.503},{"interface":"front","average":{"1min":2.5699999999999998,"5min":2.4529999999999998,"15min":2.5430000000000001},"min":{"1min":0.316,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.102,"5min":27.939,"15min":91.802000000000007},"last":0.48899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":1.2290000000000001,"15min":0.90600000000000003},"min":{"1min":0.254,"5min":0.246,"15min":0.246},"max":{"1min":3.04,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.90200000000000002,"5min":1.0549999999999999,"15min":0.89200000000000002},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.26500000000000001},"max":{"1min":4.5640000000000001,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.0609999999999999,"15min":1.2909999999999999},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":3.488,"5min":12.068,"15min":123.239},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.3520000000000001,"15min":1.399},"min":{"1min":0.307,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":2.6469999999999998,"5min":9.7720000000000002,"15min":123.014},"last":1.135}]},{"osd":7,"last update":"Mon Oct 9 04:17:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.169,"15min":0.97199999999999998},"min":{"1min":0.34899999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.7869999999999999,"5min":12.096,"15min":12.096},"last":0.435},{"interface":"front","average":{"1min":1.169,"5min":1.3740000000000001,"15min":1.0529999999999999},"min":{"1min":0.35299999999999998,"5min":0.33600000000000002,"15min":0.307},"max":{"1min":5.9420000000000002,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":1.157}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":645464064,"data_stored":645464064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1449394176,"data_stored":1449394176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":772407296,"data_stored":772407296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":722141184,"data_stored":722141184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":667222016,"data_stored":667222016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1196621824,"data_stored":1196621824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":340459520,"data_stored":340459520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1631322112,"data_stored":1631322112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:18:42.179 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:18:42.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-10-09T04:18:42.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:42.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:41 smithi181 ceph-mon[196979]: from='client.55537 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:18:42.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:41 smithi181 ceph-mon[196979]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:41 smithi181 ceph-mon[196979]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:41 smithi181 ceph-mon[196979]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T04:18:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:41 smithi181 ceph-mon[196979]: failed to encode map e320 with expected crc 2023-10-09T04:18:42.351 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:42.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:42.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:42.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 44135 44119 98.4115 63 0.00541873 0.0101565 2023-10-09T04:18:42.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[213841]: 1.d scrub starts 2023-10-09T04:18:42.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[213841]: pgmap v869: 119 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 1746/126582 objects degraded (1.379%); 642/126582 objects misplaced (0.507%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:18:42.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[213841]: failed to encode map e321 with expected crc 2023-10-09T04:18:42.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[213841]: failed to encode map e321 with expected crc 2023-10-09T04:18:42.902 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[213841]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[213841]: failed to encode map e321 with expected crc 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[221067]: 1.d scrub starts 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[221067]: pgmap v869: 119 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 1746/126582 objects degraded (1.379%); 642/126582 objects misplaced (0.507%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[221067]: failed to encode map e321 with expected crc 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[221067]: failed to encode map e321 with expected crc 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[221067]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T04:18:42.903 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:42 smithi163 ceph-mon[221067]: failed to encode map e321 with expected crc 2023-10-09T04:18:42.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:42.962 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:42 smithi181 ceph-mon[196979]: 1.d scrub starts 2023-10-09T04:18:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:42 smithi181 ceph-mon[196979]: pgmap v869: 119 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 1746/126582 objects degraded (1.379%); 642/126582 objects misplaced (0.507%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:18:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:42 smithi181 ceph-mon[196979]: failed to encode map e321 with expected crc 2023-10-09T04:18:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:42 smithi181 ceph-mon[196979]: failed to encode map e321 with expected crc 2023-10-09T04:18:43.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:42 smithi181 ceph-mon[196979]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T04:18:43.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:42 smithi181 ceph-mon[196979]: failed to encode map e321 with expected crc 2023-10-09T04:18:43.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:43.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:43.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:43.724 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:43.793 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:43.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:43.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:43.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 45276 45260 97.4759 71.3125 0.00552702 0.010249 2023-10-09T04:18:43.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:18:43.913 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:18:43.913 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:18:43.913 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:18:43.913 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:18:43.913 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:18:43.988 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:44.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:18:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:44.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:18:44.754 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:44.789 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:44.844 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 46511 46495 96.7997 77.1875 0.0096135 0.010325 2023-10-09T04:18:44.864 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:44.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:44.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:44.912 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:44 smithi163 ceph-mon[213841]: pgmap v871: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 1746/129348 objects degraded (1.350%); 642/129348 objects misplaced (0.496%); 15 MiB/s, 236 objects/s recovering 2023-10-09T04:18:44.912 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3825053529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:18:44.912 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:18:44.912 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:44 smithi163 ceph-mon[221067]: pgmap v871: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 1746/129348 objects degraded (1.350%); 642/129348 objects misplaced (0.496%); 15 MiB/s, 236 objects/s recovering 2023-10-09T04:18:44.912 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3825053529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:18:44.913 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:18:44.947 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:18:44.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:45.026 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:45.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:45.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:44 smithi181 ceph-mon[196979]: pgmap v871: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 2.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 1746/129348 objects degraded (1.350%); 642/129348 objects misplaced (0.496%); 15 MiB/s, 236 objects/s recovering 2023-10-09T04:18:45.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3825053529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:18:45.112 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:18:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:45.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:45.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:45.288 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:45.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:18:45.715 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:45.718 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:45.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 47612 47596 95.8919 68.8125 0.0104079 0.0104227 2023-10-09T04:18:45.890 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:45.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:45.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:46.038 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:46.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:46.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:46.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:46.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:45 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:46.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:45 smithi181 ceph-mon[196979]: failed to encode map e322 with expected crc 2023-10-09T04:18:46.236 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:45 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:18:46.236 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:45 smithi181 ceph-mon[196979]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T04:18:46.236 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:45 smithi181 ceph-mon[196979]: failed to encode map e322 with expected crc 2023-10-09T04:18:46.240 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:46.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:46.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[213841]: failed to encode map e322 with expected crc 2023-10-09T04:18:46.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:18:46.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[213841]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T04:18:46.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[213841]: failed to encode map e322 with expected crc 2023-10-09T04:18:46.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:46.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[221067]: failed to encode map e322 with expected crc 2023-10-09T04:18:46.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:18:46.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[221067]: osdmap e322: 8 total, 8 up, 7 in 2023-10-09T04:18:46.250 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:45 smithi163 ceph-mon[221067]: failed to encode map e322 with expected crc 2023-10-09T04:18:46.321 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:46.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:46.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:46.500 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:46.562 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:46.760 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:46.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:46.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:46.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 48934 48918 95.4774 82.625 0.00957224 0.0104674 2023-10-09T04:18:46.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:46.917 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:46.923 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:46 smithi163 ceph-mon[221067]: pgmap v873: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 1746/133923 objects degraded (1.304%); 642/133923 objects misplaced (0.479%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:18:46.923 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:46 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1746/133923 objects degraded (1.304%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:46.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:46 smithi163 ceph-mon[213841]: pgmap v873: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 1746/133923 objects degraded (1.304%); 642/133923 objects misplaced (0.479%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:18:46.924 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:46 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1746/133923 objects degraded (1.304%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:47.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:47.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:47.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:47.139 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:47.183 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:47.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:46 smithi181 ceph-mon[196979]: pgmap v873: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 1746/133923 objects degraded (1.304%); 642/133923 objects misplaced (0.479%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:18:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:46 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1746/133923 objects degraded (1.304%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:18:47.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:47.313 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:47.358 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:47.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:47.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:47.461 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:47.490 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:47.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:47.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:47.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:47.666 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:47.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 49242 49226 93.1688 19.25 0.00259725 0.0104633 2023-10-09T04:18:48.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:18:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:47 smithi181 ceph-mon[196979]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:47 smithi181 ceph-mon[196979]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T04:18:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:47 smithi181 ceph-mon[196979]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:47 smithi181 ceph-mon[196979]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[213841]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[213841]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T04:18:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[213841]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[213841]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[221067]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[221067]: osdmap e323: 8 total, 8 up, 7 in 2023-10-09T04:18:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[221067]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:47 smithi163 ceph-mon[221067]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:48.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:48.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:48.548 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:48.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:48.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:48.847 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 50297 50281 92.3681 65.9375 0.0139405 0.0108213 2023-10-09T04:18:48.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:48.937 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:48.939 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[221067]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[221067]: pgmap v875: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 1746/141666 objects degraded (1.232%); 7/141666 objects misplaced (0.005%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:18:48.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[221067]: failed to encode map e324 with expected crc 2023-10-09T04:18:48.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[221067]: failed to encode map e324 with expected crc 2023-10-09T04:18:48.940 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[221067]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T04:18:48.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[213841]: failed to encode map e323 with expected crc 2023-10-09T04:18:48.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[213841]: pgmap v875: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 1746/141666 objects degraded (1.232%); 7/141666 objects misplaced (0.005%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:18:48.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[213841]: failed to encode map e324 with expected crc 2023-10-09T04:18:48.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[213841]: failed to encode map e324 with expected crc 2023-10-09T04:18:48.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:48 smithi163 ceph-mon[213841]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T04:18:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:48 smithi181 ceph-mon[196979]: failed to encode map e323 with expected crc 2023-10-09T04:18:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:48 smithi181 ceph-mon[196979]: pgmap v875: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 1746/141666 objects degraded (1.232%); 7/141666 objects misplaced (0.005%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:18:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:48 smithi181 ceph-mon[196979]: failed to encode map e324 with expected crc 2023-10-09T04:18:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:48 smithi181 ceph-mon[196979]: failed to encode map e324 with expected crc 2023-10-09T04:18:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:48 smithi181 ceph-mon[196979]: osdmap e324: 8 total, 8 up, 7 in 2023-10-09T04:18:49.547 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:49.578 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:49.602 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:49.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:49.623 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:49.676 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:49.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:49.733 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:49.766 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:49.818 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:49.821 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:49.846 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 51444 51428 91.7774 71.6875 0.0216174 0.0108903 2023-10-09T04:18:49.863 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:18:49.891 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.892 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.892 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.892 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.892 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.892 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.893 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.893 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:18:49.939 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:49.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[221067]: failed to encode map e324 with expected crc 2023-10-09T04:18:49.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[221067]: failed to encode map e324 with expected crc 2023-10-09T04:18:49.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:49.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:49.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:49.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/184256552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:49.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:50.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-10-09T04:18:50.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:50.104 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:50.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:50.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:49 smithi181 ceph-mon[196979]: failed to encode map e324 with expected crc 2023-10-09T04:18:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:49 smithi181 ceph-mon[196979]: failed to encode map e324 with expected crc 2023-10-09T04:18:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/184256552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:50.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[213841]: failed to encode map e324 with expected crc 2023-10-09T04:18:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[213841]: failed to encode map e324 with expected crc 2023-10-09T04:18:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:50.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/184256552' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:18:50.371 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:50.636 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:18:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:18:50] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.43.0" 2023-10-09T04:18:50.847 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 52658 52642 91.3357 75.875 0.0107814 0.0109432 2023-10-09T04:18:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:50 smithi181 ceph-mon[196979]: pgmap v877: 119 pgs: 2 active+recovering+undersized+remapped, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s; 1539/144615 objects misplaced (1.064%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:18:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3377075690' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T04:18:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:50 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1746/141666 objects degraded (1.232%), 1 pg degraded) 2023-10-09T04:18:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:50 smithi163 ceph-mon[221067]: pgmap v877: 119 pgs: 2 active+recovering+undersized+remapped, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s; 1539/144615 objects misplaced (1.064%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:18:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3377075690' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T04:18:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:50 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1746/141666 objects degraded (1.232%), 1 pg degraded) 2023-10-09T04:18:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:50 smithi163 ceph-mon[213841]: pgmap v877: 119 pgs: 2 active+recovering+undersized+remapped, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s; 1539/144615 objects misplaced (1.064%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:18:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3377075690' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T04:18:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:50 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1746/141666 objects degraded (1.232%), 1 pg degraded) 2023-10-09T04:18:51.847 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 53830 53814 90.847 73.25 0.00751208 0.0110036 2023-10-09T04:18:52.076 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:52.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:18:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:52.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:52.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:51 smithi181 ceph-mon[196979]: pgmap v878: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 1532/156804 objects misplaced (0.977%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:18:52.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:51 smithi163 ceph-mon[221067]: pgmap v878: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 1532/156804 objects misplaced (0.977%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:18:52.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:51 smithi163 ceph-mon[213841]: pgmap v878: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 1532/156804 objects misplaced (0.977%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:18:52.346 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:52.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:52.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:52.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:52.689 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:52.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:52.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:52.848 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 54977 54961 90.341 71.6875 0.0188147 0.0110641 2023-10-09T04:18:52.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:52.904 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:18:52.904 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:18:52.944 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:53.848 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 56219 56203 90.0149 77.625 0.0113715 0.0111042 2023-10-09T04:18:54.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:18:54.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:54.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:54.579 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:18:54.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:18:54.661 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:54.773 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:18:54.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:54.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:54.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:54 smithi163 ceph-mon[221067]: pgmap v879: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 1259/160041 objects misplaced (0.787%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:18:54.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:54.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:54.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:54 smithi163 ceph-mon[213841]: pgmap v879: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 1259/160041 objects misplaced (0.787%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:18:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:54.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:18:54.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:18:54.853393+0000 min lat: 0.00114347 max lat: 2.09794 avg lat: 0.0111724 2023-10-09T04:18:54.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:18:54.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 57307 57291 89.4538 68 0.0193977 0.0111724 2023-10-09T04:18:54.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:54.916 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:54.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:54.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:55.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:55.122 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:55.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:18:55.123 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:54 smithi181 ceph-mon[196979]: pgmap v879: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 1259/160041 objects misplaced (0.787%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:18:55.125 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:55.201 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:55.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:55.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:55.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:18:55.342 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:55.392 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:55.415 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:18:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:55.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:18:55.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:55.559 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:55.620 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:55.807 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:55.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:55.853 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 58334 58318 88.8377 64.1875 0.0189862 0.011252 2023-10-09T04:18:55.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:18:55.997 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:56.853 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 59635 59619 88.6585 81.3125 0.00925109 0.0112723 2023-10-09T04:18:56.980 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:18:57.160 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:18:57.160 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:18:57.160 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:18:57.160 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.976766 2023-10-09T04:18:57.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.9767662909891589 2023-10-09T04:18:57.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:56 smithi163 ceph-mon[213841]: pgmap v880: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 734 op/s; 1259/161157 objects misplaced (0.781%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:18:57.206 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3890450742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:57.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:56 smithi163 ceph-mon[221067]: pgmap v880: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 734 op/s; 1259/161157 objects misplaced (0.781%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:18:57.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3890450742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:56 smithi181 ceph-mon[196979]: pgmap v880: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 46 MiB/s wr, 734 op/s; 1259/161157 objects misplaced (0.781%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:18:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3890450742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:18:57.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:18:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:57.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:18:57.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:18:57.805 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:57.853 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 60675 60659 88.1084 65 0.00826182 0.0113442 2023-10-09T04:18:57.879 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:18:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:57.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:18:57.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:18:58.035 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:58.112 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:18:58.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:57 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:58.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:57 smithi181 ceph-mon[196979]: failed to encode map e325 with expected crc 2023-10-09T04:18:58.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3890450742' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:58.115 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:57 smithi181 ceph-mon[196979]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T04:18:58.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:57 smithi181 ceph-mon[196979]: failed to encode map e325 with expected crc 2023-10-09T04:18:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:58.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:18:58.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:58.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:58.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[213841]: failed to encode map e325 with expected crc 2023-10-09T04:18:58.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3890450742' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:58.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[213841]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T04:18:58.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[213841]: failed to encode map e325 with expected crc 2023-10-09T04:18:58.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:18:58.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[221067]: failed to encode map e325 with expected crc 2023-10-09T04:18:58.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3890450742' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:18:58.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[221067]: osdmap e325: 8 total, 8 up, 7 in 2023-10-09T04:18:58.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:57 smithi163 ceph-mon[221067]: failed to encode map e325 with expected crc 2023-10-09T04:18:58.270 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:58.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:58.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:58.853 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 61832 61816 87.7491 72.3125 0.0147898 0.0113907 2023-10-09T04:18:58.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:58.929 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:58.974 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:58 smithi163 ceph-mon[213841]: pgmap v882: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 1259/174048 objects misplaced (0.723%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:18:58.974 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:58 smithi163 ceph-mon[221067]: pgmap v882: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 1259/174048 objects misplaced (0.723%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:18:58.982 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:59.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:59.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:59.266 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:18:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:58 smithi181 ceph-mon[196979]: pgmap v882: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 1259/174048 objects misplaced (0.723%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:18:59.355 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:59.613 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:18:59.613 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:18:59.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:18:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:59.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:18:59.718 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:18:59.767 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:18:59.797 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:18:59.809 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:18:59.846 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:18:59.848 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:18:59.854 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 62896 62880 87.2763 66.5 0.0127202 0.0114525 2023-10-09T04:18:59.882 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:18:59.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:18:59.917 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:18:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:18:59.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:18:59.952 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:18:59.985 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.000 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:19:00.025 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:18:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/628708271' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:00.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:18:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/628708271' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:00.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:00.082 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:00.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:00.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:18:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/628708271' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:00.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:00.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:00.341 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:00.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:19:00.854 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 63828 63812 86.6455 58.25 0.0202526 0.0115347 2023-10-09T04:19:00.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:00 smithi163 ceph-mon[213841]: pgmap v883: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 938/177285 objects misplaced (0.529%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:19:00.918 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:19:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:19:00] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.43.0" 2023-10-09T04:19:00.918 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:00 smithi163 ceph-mon[221067]: pgmap v883: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 938/177285 objects misplaced (0.529%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:19:01.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:00 smithi181 ceph-mon[196979]: pgmap v883: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 938/177285 objects misplaced (0.529%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:19:01.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:19:01.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:01.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:01.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:01.854 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 64900 64884 86.2276 67 0.0107112 0.0115927 2023-10-09T04:19:01.878 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:02.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:02.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:02.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:02.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:02.854 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 65951 65935 85.7998 65.6875 0.0101755 0.0116512 2023-10-09T04:19:02.878 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:02.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:03.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:03.001 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:02 smithi163 ceph-mon[213841]: pgmap v884: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 938/190626 objects misplaced (0.492%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:19:03.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:02 smithi163 ceph-mon[221067]: pgmap v884: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 938/190626 objects misplaced (0.492%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:19:03.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:03.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:03.150 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:03.164 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:03.187 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:03.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:03.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:03.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:03.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:02 smithi181 ceph-mon[196979]: pgmap v884: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 938/190626 objects misplaced (0.492%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:19:03.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:03.367 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:03.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:03.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:03.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:03.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:03.568 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:03.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:03.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:03.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:03.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:19:03.750 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:03.854 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:03.854 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 66630 66614 84.9152 42.4375 0.0274089 0.0117687 2023-10-09T04:19:03.975 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:03.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:04.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:04.134 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:04.198 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:04.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:04.303 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:04.350 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:04.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:04.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:04.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:04.612 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:04.854 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 67649 67633 84.4907 63.6875 0.00618394 0.0118196 2023-10-09T04:19:04.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:04 smithi163 ceph-mon[213841]: pgmap v885: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 641/193512 objects misplaced (0.331%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:19:04.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:04.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:04 smithi181 ceph-mon[196979]: pgmap v885: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 641/193512 objects misplaced (0.331%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:19:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:05.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:04 smithi163 ceph-mon[221067]: pgmap v885: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 641/193512 objects misplaced (0.331%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:19:05.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:05.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:05.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:05.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:05.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 68757 68741 84.1883 69.25 0.00990114 0.0118732 2023-10-09T04:19:05.877 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:05.911 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2836546668' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]: dispatch 2023-10-09T04:19:05.911 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]: dispatch 2023-10-09T04:19:05.949 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:05.958 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.0 primary-affinity to 0.976766 (8640132) 2023-10-09T04:19:06.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:06.225 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2836546668' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]: dispatch 2023-10-09T04:19:06.225 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]: dispatch 2023-10-09T04:19:06.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:06.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2836546668' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]: dispatch 2023-10-09T04:19:06.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]: dispatch 2023-10-09T04:19:06.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:06.424 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:06.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 69710 69694 83.7148 59.5625 0.00290335 0.0119208 2023-10-09T04:19:07.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: pgmap v886: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 641/194427 objects misplaced (0.330%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:19:07.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]': finished 2023-10-09T04:19:07.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T04:19:07.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:06 smithi181 ceph-mon[196979]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:07.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: pgmap v886: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 641/194427 objects misplaced (0.330%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:19:07.329 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.329 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]': finished 2023-10-09T04:19:07.329 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T04:19:07.329 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.329 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.329 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[213841]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:07.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:07.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: pgmap v886: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 641/194427 objects misplaced (0.330%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:19:07.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.9767662909891589}]': finished 2023-10-09T04:19:07.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: osdmap e326: 8 total, 8 up, 7 in 2023-10-09T04:19:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:06 smithi163 ceph-mon[221067]: failed to encode map e326 with expected crc 2023-10-09T04:19:07.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:07.497 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:07.681 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:07.733 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:07.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:07.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:07.857 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 15 70692 70677 83.2945 61.4375 0.0110018 0.0119998 2023-10-09T04:19:07.869 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:19:07.869 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:19:07.869 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:19:07.869 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:19:07.869 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:19:07.887 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:07.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:07.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:08.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:08.113 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:08.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:08.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:08.188 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:08.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T04:19:08.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:08.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:08.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: osdmap e327: 8 total, 8 up, 7 in 2023-10-09T04:19:08.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: failed to encode map e327 with expected crc 2023-10-09T04:19:08.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:08.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:08.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:08.373 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:08.581 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:08.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:08.692 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:08.745 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:08.838 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:08.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:08.859 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 71748 71732 82.9695 65.9375 0.014555 0.0120476 2023-10-09T04:19:08.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:08 smithi181 ceph-mon[196979]: pgmap v889: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 641/205491 objects misplaced (0.312%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:19:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1509976782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:09.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:09.024 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:09.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[221067]: pgmap v889: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 641/205491 objects misplaced (0.312%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:19:09.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:09.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:09.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:09.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1509976782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:09.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:09.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[213841]: pgmap v889: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 641/205491 objects misplaced (0.312%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:19:09.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:09.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:09.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:09.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1509976782' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:09.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:09.271 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:19:09.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:09.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:09.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:09.518 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:19:09.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:09.579 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:09.621 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:09.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:09.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:09.761 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:09.860 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 72609 72593 82.4394 53.8125 0.010583 0.0121255 2023-10-09T04:19:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:09 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:19:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:09 smithi181 ceph-mon[196979]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T04:19:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:09 smithi181 ceph-mon[196979]: failed to encode map e328 with expected crc 2023-10-09T04:19:10.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:19:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[221067]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T04:19:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[221067]: failed to encode map e328 with expected crc 2023-10-09T04:19:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:19:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[213841]: osdmap e328: 8 total, 8 up, 7 in 2023-10-09T04:19:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:09 smithi163 ceph-mon[213841]: failed to encode map e328 with expected crc 2023-10-09T04:19:10.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:10.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:10.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:10.458 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:10.526 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:10.641 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:19:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:19:10] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.43.0" 2023-10-09T04:19:10.794 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:10.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:10.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:10.862 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 73737 73721 82.2223 70.5 0.0133983 0.0121574 2023-10-09T04:19:10.914 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:10.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:10.956 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:11.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:11.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:11.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:11.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:10 smithi163 ceph-mon[213841]: failed to encode map e328 with expected crc 2023-10-09T04:19:11.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:10 smithi163 ceph-mon[213841]: pgmap v891: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 263/208221 objects misplaced (0.126%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:19:11.223 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:10 smithi163 ceph-mon[213841]: 1.1b scrub starts 2023-10-09T04:19:11.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:10 smithi163 ceph-mon[221067]: failed to encode map e328 with expected crc 2023-10-09T04:19:11.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:10 smithi163 ceph-mon[221067]: pgmap v891: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 263/208221 objects misplaced (0.126%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:19:11.223 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:10 smithi163 ceph-mon[221067]: 1.1b scrub starts 2023-10-09T04:19:11.232 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:11.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:10 smithi181 ceph-mon[196979]: failed to encode map e328 with expected crc 2023-10-09T04:19:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:10 smithi181 ceph-mon[196979]: pgmap v891: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 263/208221 objects misplaced (0.126%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:19:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:10 smithi181 ceph-mon[196979]: 1.1b scrub starts 2023-10-09T04:19:11.347 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:19:11.386 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:19:11.425 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:19:11.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:11.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:11.466 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:19:11.496 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:19:11.531 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:19:11.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:11.566 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:19:11.609 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:11.615 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:11.630 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.631 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:19:11.862 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 74790 74774 81.9345 65.8125 0.0114333 0.0121995 2023-10-09T04:19:12.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3550217322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:11 smithi181 ceph-mon[196979]: 1.1 scrub starts 2023-10-09T04:19:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3550217322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:12.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:11 smithi163 ceph-mon[221067]: 1.1 scrub starts 2023-10-09T04:19:12.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3550217322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:12.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:11 smithi163 ceph-mon[213841]: 1.1 scrub starts 2023-10-09T04:19:12.638 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:19:12.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:19:12.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:12.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:12.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:12.795 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:19:12.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:12.847 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:12.862 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 75192 75176 80.9555 25.125 0.00295267 0.0121958 2023-10-09T04:19:12.888 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:12.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:12.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[213841]: pgmap v892: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 263/220104 objects misplaced (0.119%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:19:12.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[213841]: failed to encode map e329 with expected crc 2023-10-09T04:19:12.975 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[213841]: failed to encode map e329 with expected crc 2023-10-09T04:19:12.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[213841]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T04:19:12.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[221067]: pgmap v892: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 263/220104 objects misplaced (0.119%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:19:12.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[221067]: failed to encode map e329 with expected crc 2023-10-09T04:19:12.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[221067]: failed to encode map e329 with expected crc 2023-10-09T04:19:12.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:12 smithi163 ceph-mon[221067]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T04:19:12.995 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:13.025 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:13.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:13.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:13.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:13.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:12 smithi181 ceph-mon[196979]: pgmap v892: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 263/220104 objects misplaced (0.119%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:19:13.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:12 smithi181 ceph-mon[196979]: failed to encode map e329 with expected crc 2023-10-09T04:19:13.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:12 smithi181 ceph-mon[196979]: failed to encode map e329 with expected crc 2023-10-09T04:19:13.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:12 smithi181 ceph-mon[196979]: osdmap e329: 8 total, 8 up, 7 in 2023-10-09T04:19:13.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:13.296 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:13.398 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:13.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:13.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:13.554 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:13.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:13.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:13.656 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:13.687 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:19:13.687 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:19:13.707 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:13.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:13.765 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:13.844 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:13.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:13.862 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 76070 76054 80.5136 54.875 0.0147715 0.0124155 2023-10-09T04:19:13.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: failed to encode map e330 with expected crc 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: failed to encode map e330 with expected crc 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: failed to encode map e330 with expected crc 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: 1.18 scrub starts 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: 1.18 scrub ok 2023-10-09T04:19:13.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[213841]: pgmap v895: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:19:14.030 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:14.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: failed to encode map e330 with expected crc 2023-10-09T04:19:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T04:19:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: failed to encode map e330 with expected crc 2023-10-09T04:19:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: failed to encode map e330 with expected crc 2023-10-09T04:19:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: 1.18 scrub starts 2023-10-09T04:19:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: 1.18 scrub ok 2023-10-09T04:19:14.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:13 smithi181 ceph-mon[196979]: pgmap v895: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: failed to encode map e330 with expected crc 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: osdmap e330: 8 total, 8 up, 7 in 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: failed to encode map e330 with expected crc 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: failed to encode map e330 with expected crc 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: 1.18 scrub starts 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: 1.18 scrub ok 2023-10-09T04:19:14.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:13 smithi163 ceph-mon[221067]: pgmap v895: 119 pgs: 119 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:19:14.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:14.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:14.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:14.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:19:14.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:14.820 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:14.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:19:14.863322+0000 min lat: 0.00114347 max lat: 2.09794 avg lat: 0.0124377 2023-10-09T04:19:14.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:19:14.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 77219 77203 80.3685 71.8125 0.0121701 0.0124377 2023-10-09T04:19:15.212 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:19:15.485 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:15.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:15 smithi163 ceph-mon[221067]: 1.d scrub starts 2023-10-09T04:19:15.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:15 smithi163 ceph-mon[221067]: 1.d scrub ok 2023-10-09T04:19:15.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:15 smithi163 ceph-mon[213841]: 1.d scrub starts 2023-10-09T04:19:15.488 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:15 smithi163 ceph-mon[213841]: 1.d scrub ok 2023-10-09T04:19:15.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:15.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:15.590 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:15.649 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:15 smithi181 ceph-mon[196979]: 1.d scrub starts 2023-10-09T04:19:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:15 smithi181 ceph-mon[196979]: 1.d scrub ok 2023-10-09T04:19:15.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 78116 78100 79.9702 56.0625 0.0235846 0.0124997 2023-10-09T04:19:16.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:16.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:16.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:16.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:16.576 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:16.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:16.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:16 smithi181 ceph-mon[196979]: 1.9 scrub starts 2023-10-09T04:19:16.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:16 smithi181 ceph-mon[196979]: 1.9 scrub ok 2023-10-09T04:19:16.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:16 smithi181 ceph-mon[196979]: pgmap v896: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 984 op/s; 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:19:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:16.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:16.769 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:16 smithi163 ceph-mon[221067]: 1.9 scrub starts 2023-10-09T04:19:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:16 smithi163 ceph-mon[221067]: 1.9 scrub ok 2023-10-09T04:19:16.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:16 smithi163 ceph-mon[221067]: pgmap v896: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 984 op/s; 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:19:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:16 smithi163 ceph-mon[213841]: 1.9 scrub starts 2023-10-09T04:19:16.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:16 smithi163 ceph-mon[213841]: 1.9 scrub ok 2023-10-09T04:19:16.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:16 smithi163 ceph-mon[213841]: pgmap v896: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 984 op/s; 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:19:16.861 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:16.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 79165 79149 79.7378 65.5625 0.0161902 0.0125367 2023-10-09T04:19:17.081 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:17.087 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:17.115 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:17.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:17.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:17.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:17.276 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:17.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:17.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:17.406 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[221067]: 1.10 scrub starts 2023-10-09T04:19:17.406 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[221067]: 1.10 scrub ok 2023-10-09T04:19:17.406 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[221067]: 1.1e scrub starts 2023-10-09T04:19:17.406 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[221067]: 1.1e scrub ok 2023-10-09T04:19:17.456 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:17.486 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:17.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:17.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:17.551 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:17 smithi181 ceph-mon[196979]: 1.10 scrub starts 2023-10-09T04:19:17.551 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:17 smithi181 ceph-mon[196979]: 1.10 scrub ok 2023-10-09T04:19:17.551 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:17 smithi181 ceph-mon[196979]: 1.1e scrub starts 2023-10-09T04:19:17.551 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:17 smithi181 ceph-mon[196979]: 1.1e scrub ok 2023-10-09T04:19:17.652 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:17.685 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:17.738 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[213841]: 1.10 scrub starts 2023-10-09T04:19:17.738 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[213841]: 1.10 scrub ok 2023-10-09T04:19:17.738 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[213841]: 1.1e scrub starts 2023-10-09T04:19:17.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:17 smithi163 ceph-mon[213841]: 1.1e scrub ok 2023-10-09T04:19:17.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:17.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:17.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:17.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 80113 80097 79.4127 59.25 0.0185017 0.0125872 2023-10-09T04:19:17.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:17.902 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:17.983 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:17.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:18.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:18.139 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:18.217 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:18.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:18.325 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:18.372 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:18.419 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:18.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:18.511 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:19:18.512 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:19:18.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:18.580 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:18.676 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:18.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:18.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3531442684' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:18.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:18.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3690858447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:19:18.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:18 smithi181 ceph-mon[196979]: pgmap v897: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:19:18.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3531442684' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3690858447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:19:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[221067]: pgmap v897: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:19:18.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3531442684' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:18.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:18.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3690858447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:19:18.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:18 smithi163 ceph-mon[213841]: pgmap v897: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:19:18.840 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:18.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 81095 81079 79.1309 61.375 0.00827217 0.0126325 2023-10-09T04:19:18.941 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:18.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:18.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:19.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:19.089 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:19.159 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:19.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:19.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:19.340 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:19:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:19:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: failed to encode map e331 with expected crc 2023-10-09T04:19:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: failed to encode map e331 with expected crc 2023-10-09T04:19:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:19.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3690858447' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: 1.19 scrub starts 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: 1.19 scrub ok 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: failed to encode map e331 with expected crc 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: failed to encode map e331 with expected crc 2023-10-09T04:19:19.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3690858447' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:19:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T04:19:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: 1.19 scrub starts 2023-10-09T04:19:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: 1.19 scrub ok 2023-10-09T04:19:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:19.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:19.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:19:19.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:19:19.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: failed to encode map e331 with expected crc 2023-10-09T04:19:19.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: failed to encode map e331 with expected crc 2023-10-09T04:19:19.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3690858447' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: osdmap e331: 8 total, 8 up, 7 in 2023-10-09T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: 1.19 scrub starts 2023-10-09T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: 1.19 scrub ok 2023-10-09T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:19.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 82047 82031 78.829 59.5 0.0148698 0.0126808 2023-10-09T04:19:20.401 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:20.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:20.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:20.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:19:20.566 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:20.815 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:19:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:19:20] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-10-09T04:19:20.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 83074 83058 78.6071 64.1875 0.0319091 0.0127155 2023-10-09T04:19:20.869 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:20.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:20.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:20.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:21.047 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[213841]: 1.12 scrub starts 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[213841]: 1.12 scrub ok 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[213841]: pgmap v899: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[221067]: 1.12 scrub starts 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[221067]: 1.12 scrub ok 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[221067]: pgmap v899: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:19:21.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:21.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:21.154 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:20 smithi181 ceph-mon[196979]: 1.12 scrub starts 2023-10-09T04:19:21.154 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:20 smithi181 ceph-mon[196979]: 1.12 scrub ok 2023-10-09T04:19:21.154 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:20 smithi181 ceph-mon[196979]: pgmap v899: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:19:21.154 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:21.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:21.170 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:19:21.170 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:19:21.238 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:19:21.256 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:21.275 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:19:21.307 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:21.315 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:19:21.356 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:19:21.391 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:21.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:21.405 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:19:21.445 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:19:21.486 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:19:21.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:21.527 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.540 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:19:21.565 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:21.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:21.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:21.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:21.822 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:21.863 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 84046 84030 78.3407 60.75 0.0151774 0.0127598 2023-10-09T04:19:22.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1378327507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:22.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1378327507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:22.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1378327507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:22.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:19:22.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:19:22.864 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 84950 84934 78.0195 56.5 0.0163704 0.0128125 2023-10-09T04:19:22.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:22.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:22.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:23.007 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:22 smithi163 ceph-mon[221067]: pgmap v900: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:19:23.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:22 smithi163 ceph-mon[213841]: pgmap v900: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:19:23.013 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:23.034 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:23.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:23.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:23.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:23.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:22 smithi181 ceph-mon[196979]: pgmap v900: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T04:19:23.313 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:23.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:23.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:23.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:23.597 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:23.671 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:23.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:23.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:23.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 85887 85871 77.7376 58.5625 0.00406645 0.0128583 2023-10-09T04:19:23.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:23.899 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:23.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:23.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:24.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:24.090 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:24.486 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:24.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:24.501 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:24.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:24.700 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:24.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:24.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:24.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 86589 86573 77.2533 43.875 0.0197515 0.012939 2023-10-09T04:19:24.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:25.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:19:25.021 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:25.059 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:24 smithi163 ceph-mon[221067]: pgmap v901: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T04:19:25.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:24 smithi163 ceph-mon[213841]: pgmap v901: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T04:19:25.117 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:25.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:25.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:25.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:24 smithi181 ceph-mon[196979]: pgmap v901: 119 pgs: 119 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-09T04:19:25.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:25.313 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:25.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 87372 87356 76.8546 48.9375 0.0118007 0.013006 2023-10-09T04:19:25.993 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:19:26.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2225906070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:19:26.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:26.179 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2225906070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:19:26.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2225906070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:19:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:26.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:26.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:26.310 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:26.655 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:26.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:26.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:26.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:26.789 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:26.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:26.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:26.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:26.864 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 87628 87612 76.0098 16 0.00415222 0.013004 2023-10-09T04:19:26.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:26.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: 1.a deep-scrub starts 2023-10-09T04:19:26.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: 1.a deep-scrub ok 2023-10-09T04:19:26.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: pgmap v902: 119 pgs: 119 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 933 op/s 2023-10-09T04:19:26.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2225906070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: osdmap e332: 8 total, 8 up, 7 in 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[213841]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: 1.a deep-scrub starts 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: 1.a deep-scrub ok 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: pgmap v902: 119 pgs: 119 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 933 op/s 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2225906070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: osdmap e332: 8 total, 8 up, 7 in 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.957 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.958 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:26 smithi163 ceph-mon[221067]: failed to encode map e332 with expected crc 2023-10-09T04:19:26.970 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:27.020 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:27.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:27.125 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:27.150 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: 1.a deep-scrub starts 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: 1.a deep-scrub ok 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: pgmap v902: 119 pgs: 119 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 933 op/s 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2225906070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: osdmap e332: 8 total, 8 up, 7 in 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.151 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.152 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:26 smithi181 ceph-mon[196979]: failed to encode map e332 with expected crc 2023-10-09T04:19:27.154 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:27.206 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:27.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:27.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:27.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:19:27.349 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:27.430 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:27.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:27.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:27.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:27.575 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:27.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:27.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:27.725 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:27.752 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:27.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:27.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:27.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 87628 87612 74.969 0 - 0.013004 2023-10-09T04:19:27.876 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:19:27.876 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:19:27.876 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:19:27.876 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:19:27.876 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:19:27.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:27.905 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:19:27.929 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:27.972 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:27.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:28.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:28.114 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:28.236 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[221067]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.236 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[221067]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[221067]: osdmap e333: 8 total, 8 up, 7 in 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[221067]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[221067]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2339526979' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[213841]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[213841]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[213841]: osdmap e333: 8 total, 8 up, 7 in 2023-10-09T04:19:28.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[213841]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[213841]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.238 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2339526979' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:19:28.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:27 smithi181 ceph-mon[196979]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:27 smithi181 ceph-mon[196979]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:27 smithi181 ceph-mon[196979]: osdmap e333: 8 total, 8 up, 7 in 2023-10-09T04:19:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:27 smithi181 ceph-mon[196979]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:27 smithi181 ceph-mon[196979]: failed to encode map e333 with expected crc 2023-10-09T04:19:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2339526979' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:19:28.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:19:28.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 87647 87631 73.9724 0.59375 2.82435 0.0132636 2023-10-09T04:19:28.976 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:28.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: pgmap v905: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: failed to encode map e334 with expected crc 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2339526979' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: osdmap e334: 8 total, 8 up, 7 in 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: failed to encode map e334 with expected crc 2023-10-09T04:19:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[221067]: 9.d starting backfill to osd.1 from (0'0,0'0] MAX to 331'5474 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: pgmap v905: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: failed to encode map e334 with expected crc 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2339526979' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: osdmap e334: 8 total, 8 up, 7 in 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: failed to encode map e334 with expected crc 2023-10-09T04:19:29.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:28 smithi163 ceph-mon[213841]: 9.d starting backfill to osd.1 from (0'0,0'0] MAX to 331'5474 2023-10-09T04:19:29.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:29.126 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: pgmap v905: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: failed to encode map e334 with expected crc 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2339526979' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: osdmap e334: 8 total, 8 up, 7 in 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: failed to encode map e334 with expected crc 2023-10-09T04:19:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:28 smithi181 ceph-mon[196979]: 9.d starting backfill to osd.1 from (0'0,0'0] MAX to 331'5474 2023-10-09T04:19:29.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:29.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:29.675 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:29.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:29.769 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:29.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 88500 88484 73.697 53.3125 0.00690567 0.0135631 2023-10-09T04:19:30.208 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:30.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:30.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:30.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:30.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:30.427 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:30.474 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:19:30.504 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:19:30.553 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:19:30.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:30.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:30.583 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:19:30.612 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:19:30.648 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:19:30.662 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:30.677 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:19:30.699 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:30.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.700 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.701 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.701 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.701 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.701 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.701 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:19:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:30.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 89265 89249 73.3565 47.8125 0.0047573 0.0135948 2023-10-09T04:19:30.926 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:19:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:19:30] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-10-09T04:19:30.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:30 smithi163 ceph-mon[213841]: pgmap v907: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s 2023-10-09T04:19:30.926 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2058920926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:31.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:31.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:30 smithi163 ceph-mon[221067]: pgmap v907: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s 2023-10-09T04:19:31.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2058920926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:31.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:31.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:30 smithi181 ceph-mon[196979]: pgmap v907: 119 pgs: 119 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s 2023-10-09T04:19:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2058920926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:31.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:31.356 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:31.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:19:31.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 90115 90099 73.0938 53.125 0.0107229 0.0136762 2023-10-09T04:19:32.018 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:32.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:32.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:32.188 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:32.280 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:32.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:32.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:32.442 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:32.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:32.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:32.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:32.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:32.720 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:32.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:32.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:32.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:32.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 90936 90920 72.8146 51.3125 0.106762 0.0137147 2023-10-09T04:19:32.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:32.969 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:33.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:32 smithi181 ceph-mon[196979]: pgmap v908: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 690 op/s; 5414/267456 objects misplaced (2.024%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:19:33.290 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:32 smithi163 ceph-mon[221067]: pgmap v908: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 690 op/s; 5414/267456 objects misplaced (2.024%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:19:33.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:32 smithi163 ceph-mon[213841]: pgmap v908: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 690 op/s; 5414/267456 objects misplaced (2.024%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:19:33.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:33.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:33.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:33.469 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:33.475 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:33.572 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:33.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:33.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:33.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:33.760 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:33.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:19:33.865 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 91824 91808 72.5954 55.5 0.0163234 0.0137697 2023-10-09T04:19:33.928 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:33.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:33.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:34.007 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:19:34.040 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:34.060 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:19:34.061 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:19:34.092 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:34.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:34.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:34.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:34.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1764514098' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:19:34.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:33 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:19:34.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:34.300 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1764514098' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:19:34.300 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:33 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:19:34.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1764514098' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:19:34.301 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:33 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:19:34.304 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:34.867 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:19:34.867697+0000 min lat: 0.00114347 max lat: 2.93391 avg lat: 0.0138435 2023-10-09T04:19:34.867 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:19:34.867 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 92489 92473 72.206 41.5625 0.0105515 0.0138435 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: pgmap v909: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 5414/269568 objects misplaced (2.008%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: failed to encode map e335 with expected crc 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: failed to encode map e335 with expected crc 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: osdmap e335: 8 total, 8 up, 7 in 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:34.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: pgmap v909: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 5414/269568 objects misplaced (2.008%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: failed to encode map e335 with expected crc 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: failed to encode map e335 with expected crc 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: osdmap e335: 8 total, 8 up, 7 in 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:34.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:35.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:35.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:35.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: pgmap v909: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 5414/269568 objects misplaced (2.008%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:19:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: failed to encode map e335 with expected crc 2023-10-09T04:19:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: failed to encode map e335 with expected crc 2023-10-09T04:19:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:19:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: osdmap e335: 8 total, 8 up, 7 in 2023-10-09T04:19:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:19:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:35.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:35.350 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:35.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:19:35.867 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 93266 93250 71.9142 48.5625 0.0220503 0.0139001 2023-10-09T04:19:36.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:36.121 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:36 smithi163 ceph-mon[213841]: pgmap v911: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 5081/273180 objects misplaced (1.860%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T04:19:36.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:36 smithi163 ceph-mon[221067]: pgmap v911: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 5081/273180 objects misplaced (1.860%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T04:19:36.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:36.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:36.164 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:19:36.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:36 smithi181 ceph-mon[196979]: pgmap v911: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 5081/273180 objects misplaced (1.860%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T04:19:36.289 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:36.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:36.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:36.495 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:19:36.501 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:36.552 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:36.631 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:36.807 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:36.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:36.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:36.868 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 94076 94060 71.6543 50.625 0.0262674 0.0139509 2023-10-09T04:19:36.921 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:36.983 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:37.118 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:19:37.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/483701408' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:37.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/483701408' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:37.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/483701408' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:37.868 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 94871 94855 71.3897 49.6875 0.0141871 0.0140032 2023-10-09T04:19:38.050 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:38.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:38.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:38.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:38.222 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:38.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:38.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/483701408' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:38.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: osdmap e336: 8 total, 8 up, 7 in 2023-10-09T04:19:38.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[213841]: pgmap v913: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 693 op/s; 5081/279072 objects misplaced (1.821%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/483701408' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: osdmap e336: 8 total, 8 up, 7 in 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.340 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.341 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:38 smithi163 ceph-mon[221067]: pgmap v913: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 693 op/s; 5081/279072 objects misplaced (1.821%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/483701408' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: osdmap e336: 8 total, 8 up, 7 in 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.515 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.516 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.516 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: failed to encode map e336 with expected crc 2023-10-09T04:19:38.516 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:38 smithi181 ceph-mon[196979]: pgmap v913: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 43 MiB/s wr, 693 op/s; 5081/279072 objects misplaced (1.821%); 5.6 MiB/s, 88 objects/s recovering 2023-10-09T04:19:38.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:38.568 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:38.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:38.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:38.726 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:38.868 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 95595 95579 71.0786 45.25 0.00966776 0.0140636 2023-10-09T04:19:39.050 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:39.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:39.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:39.089 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:19:39.129 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:19:39.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:39.169 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:19:39.195 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:19:39.224 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:19:39.226 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:19:39.226 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:19:39.241 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:39.263 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:19:39.298 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:19:39.335 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:39.338 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.339 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:19:39.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:39.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:39.589 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:39.635 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2204873707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:39.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2204873707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:39.642 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:39.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2204873707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:39.868 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 96373 96357 70.8144 48.625 0.0139066 0.0141131 2023-10-09T04:19:40.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:19:40.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:40 smithi163 ceph-mon[213841]: pgmap v914: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 5081/281322 objects misplaced (1.806%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T04:19:40.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:19:40.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:40.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:40.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:40 smithi181 ceph-mon[196979]: pgmap v914: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 5081/281322 objects misplaced (1.806%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T04:19:40.805 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:19:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:19:40] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-10-09T04:19:40.806 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:40 smithi163 ceph-mon[221067]: pgmap v914: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 5081/281322 objects misplaced (1.806%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T04:19:40.856 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:40.869 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 97119 97103 70.5327 46.625 0.0161132 0.0141728 2023-10-09T04:19:40.887 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:19:40.934 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:41.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:41.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:41.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:41.239 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:41.319 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:41.346 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:41.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:41.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:41.509 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:41.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:41.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:41.747 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:19:41.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3970545279' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:19:41.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3970545279' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:19:41.786 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:41.835 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3970545279' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:19:41.843 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:41.869 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 97935 97919 70.3082 51 0.00804231 0.0142079 2023-10-09T04:19:41.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:41.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:42.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:42.045 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:42.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:42.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:42.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:42.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:42.225 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:42.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:42.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:42.604 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:42.648 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:42.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:19:42.869 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 97985 97969 69.5451 3.125 0.0052245 0.0142048 2023-10-09T04:19:42.966 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:19:42.966 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:19:42.966 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:19:42.966 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:19:42.966 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3970545279' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: osdmap e337: 8 total, 8 up, 7 in 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: pgmap v916: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 4782/290568 objects misplaced (1.646%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T04:19:43.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:42 smithi181 ceph-mon[196979]: osdmap e338: 8 total, 8 up, 7 in 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3970545279' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: osdmap e337: 8 total, 8 up, 7 in 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.020 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: pgmap v916: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 4782/290568 objects misplaced (1.646%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[221067]: osdmap e338: 8 total, 8 up, 7 in 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3970545279' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: osdmap e337: 8 total, 8 up, 7 in 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: failed to encode map e337 with expected crc 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: pgmap v916: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 4782/290568 objects misplaced (1.646%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T04:19:43.021 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:42 smithi163 ceph-mon[213841]: osdmap e338: 8 total, 8 up, 7 in 2023-10-09T04:19:43.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:19:43.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:43.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:43.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:43.392 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:43.399 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:43.430 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:43.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:43.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:43.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:43.608 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:43.710 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:43.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.799 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[213841]: osdmap e339: 8 total, 8 up, 7 in 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.800 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: failed to encode map e338 with expected crc 2023-10-09T04:19:43.801 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:43 smithi163 ceph-mon[221067]: osdmap e339: 8 total, 8 up, 7 in 2023-10-09T04:19:43.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 98086 98070 68.8289 6.3125 0.0245654 0.0145236 2023-10-09T04:19:43.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:43.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:43.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:19:44.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:44.097 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: failed to encode map e338 with expected crc 2023-10-09T04:19:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: failed to encode map e338 with expected crc 2023-10-09T04:19:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: failed to encode map e338 with expected crc 2023-10-09T04:19:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: failed to encode map e338 with expected crc 2023-10-09T04:19:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: failed to encode map e338 with expected crc 2023-10-09T04:19:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: failed to encode map e338 with expected crc 2023-10-09T04:19:44.098 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:43 smithi181 ceph-mon[196979]: osdmap e339: 8 total, 8 up, 7 in 2023-10-09T04:19:44.143 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:44.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:44.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:44.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:44.487 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:44.538 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:19:44.620 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:19:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: failed to encode map e339 with expected crc 2023-10-09T04:19:44.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: failed to encode map e339 with expected crc 2023-10-09T04:19:44.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.011 2023-10-09T04:19:44.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 98723 2023-10-09T04:19:44.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T04:19:44.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T04:19:44.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 68.5492 2023-10-09T04:19:44.877 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 39.8978 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 231.25 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1096 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 638.374 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3700 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0145839 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0563398 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 2.93391 2023-10-09T04:19:44.878 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00114347 2023-10-09T04:19:44.883 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:19:45.067 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: 9.8 starting backfill to osd.5 from (0'0,0'0] MAX to 336'6130 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: pgmap v919: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 4782/292338 objects misplaced (1.636%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3107041406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:19:45.068 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:44 smithi181 ceph-mon[196979]: osdmap e340: 8 total, 8 up, 7 in 2023-10-09T04:19:45.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.077 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: 9.8 starting backfill to osd.5 from (0'0,0'0] MAX to 336'6130 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: pgmap v919: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 4782/292338 objects misplaced (1.636%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3107041406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:19:45.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[221067]: osdmap e340: 8 total, 8 up, 7 in 2023-10-09T04:19:45.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: 9.8 starting backfill to osd.5 from (0'0,0'0] MAX to 336'6130 2023-10-09T04:19:45.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: failed to encode map e339 with expected crc 2023-10-09T04:19:45.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: pgmap v919: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 4782/292338 objects misplaced (1.636%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:19:45.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3107041406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:45.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:19:45.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:19:45.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:19:45.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:44 smithi163 ceph-mon[213841]: osdmap e340: 8 total, 8 up, 7 in 2023-10-09T04:19:45.765 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:45.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:45.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:45.898 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:45 smithi163 ceph-mon[213841]: failed to encode map e340 with expected crc 2023-10-09T04:19:45.899 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:45 smithi163 ceph-mon[213841]: osdmap e341: 8 total, 8 up, 7 in 2023-10-09T04:19:45.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:45 smithi163 ceph-mon[221067]: failed to encode map e340 with expected crc 2023-10-09T04:19:45.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:45 smithi163 ceph-mon[221067]: osdmap e341: 8 total, 8 up, 7 in 2023-10-09T04:19:45.980 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:46.248 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:45 smithi181 ceph-mon[196979]: failed to encode map e340 with expected crc 2023-10-09T04:19:46.248 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:45 smithi181 ceph-mon[196979]: osdmap e341: 8 total, 8 up, 7 in 2023-10-09T04:19:46.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:46.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:46.388 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:46.442 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:46.651 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:46.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:46.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:46.758 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:46.854 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:46.937 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:19:46.967 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:19:47.001 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:19:47.001 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36610 7.1 7:86d4b555:::benchmark_data_smithi163_346082_object36609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.296917+0000", 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 598.69853925500001, 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97434309600000002, 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:19:47.002 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36610 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296917+0000", 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296917+0000", 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296910+0000", 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999933 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.296970+0000", 2023-10-09T04:19:47.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0127999999999998e-05 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297009+0000", 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8614e-05 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297022+0000", 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3477e-05 2023-10-09T04:19:47.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306475+0000", 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0094528070000000006 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306491+0000", 2023-10-09T04:19:47.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5642e-05 2023-10-09T04:19:47.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.265950+0000", 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95945866800000001 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.268174+0000", 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002224536 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271214+0000", 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003039618 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271260+0000", 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6233000000000003e-05 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:36613 7.1 7:85783f21:::benchmark_data_smithi163_346082_object36612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:09:48.297031+0000", 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 598.69842550400006, 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97415562300000003, 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 36613 2023-10-09T04:19:47.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297031+0000", 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297031+0000", 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297030+0000", 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297063+0000", 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2888000000000001e-05 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297070+0000", 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.418e-06 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.297075+0000", 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7690000000000004e-06 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306589+0000", 2023-10-09T04:19:47.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0095134810000000007 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:48.306596+0000", 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2200000000000003e-06 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.268225+0000", 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96162872300000002 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.269076+0000", 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00085153799999999999 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271114+0000", 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020374289999999999 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:09:49.271186+0000", 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2655000000000001e-05 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52729 7.13 7:cc295b9f:::benchmark_data_smithi163_346082_object52728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.293100+0000", 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.70235590000004, 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98155146500000001, 2023-10-09T04:19:47.018 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52729 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293100+0000", 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293100+0000", 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293099+0000", 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293174+0000", 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5077e-05 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293189+0000", 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4742e-05 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293195+0000", 2023-10-09T04:19:47.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1120000000000001e-06 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293248+0000", 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2986999999999999e-05 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.293258+0000", 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0259e-05 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.270342+0000", 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97708439199999997 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.270441+0000", 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8899000000000003e-05 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.270565+0000", 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123616 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273472+0000", 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029067540000000001 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273525+0000", 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.342e-05 2023-10-09T04:19:47.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274570+0000", 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010451099999999999 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274615+0000", 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4753999999999999e-05 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274652+0000", 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6816000000000001e-05 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52747 7.13 7:cda012a2:::benchmark_data_smithi163_346082_object52746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.302565+0000", 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.6928911, 2023-10-09T04:19:47.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97614182299999996, 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52747 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302565+0000", 2023-10-09T04:19:47.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302565+0000", 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302564+0000", 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302622+0000", 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8745000000000001e-05 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302645+0000", 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.317e-05 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302653+0000", 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.289e-06 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302734+0000", 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0788e-05 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.302746+0000", 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.29e-05 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.271839+0000", 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96909244900000002 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.271972+0000", 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132862 2023-10-09T04:19:47.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272106+0000", 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013400200000000001 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273559+0000", 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001452977 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274496+0000", 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00093728699999999997 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278619+0000", 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041224909999999998 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278667+0000", 2023-10-09T04:19:47.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8822000000000001e-05 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278707+0000", 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9437000000000002e-05 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.034 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52798 7.13 7:cc44a9ee:::benchmark_data_smithi163_346082_object52797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.336634+0000", 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.658821923, 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94227905499999998, 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52798 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336634+0000", 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336634+0000", 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336633+0000", 2023-10-09T04:19:47.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.037 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:47.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336683+0000", 2023-10-09T04:19:47.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0476000000000003e-05 2023-10-09T04:19:47.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336716+0000", 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2693e-05 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336732+0000", 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6399999999999999e-05 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336797+0000", 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4950999999999995e-05 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.336807+0000", 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8889999999999997e-06 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272736+0000", 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93592933 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272820+0000", 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3343999999999994e-05 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.272913+0000", 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3618999999999999e-05 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274531+0000", 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001617531 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274537+0000", 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7270000000000004e-06 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278834+0000", 2023-10-09T04:19:47.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042977900000000001 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278879+0000", 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4243000000000003e-05 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278913+0000", 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4530000000000003e-05 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.043 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52815 7.13 7:cd1aa158:::benchmark_data_smithi163_346082_object52814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.351193+0000", 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.64426303799996, 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92784179700000002, 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.044 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52815 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351193+0000", 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351193+0000", 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351191+0000", 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351246+0000", 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4660999999999997e-05 2023-10-09T04:19:47.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351259+0000", 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3348e-05 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351266+0000", 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3040000000000001e-06 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351324+0000", 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8427999999999997e-05 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.351336+0000", 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1749000000000001e-05 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273572+0000", 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92223578900000003 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273650+0000", 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8481999999999998e-05 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.273775+0000", 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012533299999999999 2023-10-09T04:19:47.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275290+0000", 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001514149 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275329+0000", 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9737000000000003e-05 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278961+0000", 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036314860000000002 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279004+0000", 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3208999999999999e-05 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279035+0000", 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0839000000000001e-05 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52830 7.13 7:cb4153a8:::benchmark_data_smithi163_346082_object52829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.365239+0000", 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.63021671499996, 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91388604799999995, 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.052 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52830 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365239+0000", 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365239+0000", 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365238+0000", 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365296+0000", 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8714000000000001e-05 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365314+0000", 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7504999999999999e-05 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365321+0000", 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2579999999999998e-06 2023-10-09T04:19:47.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365387+0000", 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5335999999999999e-05 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.365396+0000", 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2860000000000005e-06 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274673+0000", 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90927686100000005 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274745+0000", 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2463000000000001e-05 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.274831+0000", 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5567000000000003e-05 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276891+0000", 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020600050000000002 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276937+0000", 2023-10-09T04:19:47.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6011999999999999e-05 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279076+0000", 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021394700000000001 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279105+0000", 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8549000000000001e-05 2023-10-09T04:19:47.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279125+0000", 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0636999999999999e-05 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52848 7.13 7:cd543811:::benchmark_data_smithi163_346082_object52847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.379437+0000", 2023-10-09T04:19:47.060 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.61601877600003, 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89977305799999996, 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52848 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379437+0000", 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379437+0000", 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379436+0000", 2023-10-09T04:19:47.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379497+0000", 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1274999999999999e-05 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379508+0000", 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0465999999999999e-05 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379512+0000", 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000004e-06 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379570+0000", 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8325000000000001e-05 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.379581+0000", 2023-10-09T04:19:47.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0358e-05 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275369+0000", 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89578837700000002 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275474+0000", 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010519299999999999 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.275577+0000", 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010239299999999999 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276928+0000", 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001351093 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276945+0000", 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7564e-05 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279156+0000", 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022106909999999999 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279181+0000", 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5131999999999999e-05 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279210+0000", 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9074999999999999e-05 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52857 7.13 7:cc4020e3:::benchmark_data_smithi163_346082_object52856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.385532+0000", 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.60992372600003, 2023-10-09T04:19:47.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89567147499999999, 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52857 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385532+0000", 2023-10-09T04:19:47.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385532+0000", 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385531+0000", 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385594+0000", 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2288999999999999e-05 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385608+0000", 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4569000000000001e-05 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385615+0000", 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1899999999999998e-06 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385670+0000", 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4166000000000003e-05 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.385679+0000", 2023-10-09T04:19:47.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7739999999999999e-06 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276088+0000", 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89040851399999998 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276210+0000", 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012211599999999999 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276353+0000", 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014332700000000001 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278512+0000", 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002158751 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278565+0000", 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3368999999999997e-05 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281109+0000", 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025435779999999999 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281164+0000", 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4601000000000001e-05 2023-10-09T04:19:47.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281204+0000", 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0277000000000001e-05 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52865 7.13 7:cefc765f:::benchmark_data_smithi163_346082_object52864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.390532+0000", 2023-10-09T04:19:47.076 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.60492428099997, 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89079931999999995, 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52865 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390532+0000", 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390532+0000", 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390531+0000", 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:47.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390597+0000", 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6179000000000005e-05 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390609+0000", 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2157999999999999e-05 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390615+0000", 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9349999999999999e-06 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390678+0000", 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2751999999999998e-05 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.390687+0000", 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3130000000000004e-06 2023-10-09T04:19:47.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.276971+0000", 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88628442699999999 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277087+0000", 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0001154 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277191+0000", 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104571 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278716+0000", 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015244970000000001 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278774+0000", 2023-10-09T04:19:47.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8325999999999997e-05 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281248+0000", 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024733749999999999 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281292+0000", 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4144000000000003e-05 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281331+0000", 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9363000000000002e-05 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52871 7.13 7:c955e08a:::benchmark_data_smithi163_346082_object52870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.395047+0000", 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.60040945100002, 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88652477799999996, 2023-10-09T04:19:47.084 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52871 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395047+0000", 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395047+0000", 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395045+0000", 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395097+0000", 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1892999999999999e-05 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395114+0000", 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7306e-05 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395123+0000", 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4970000000000001e-06 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395185+0000", 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2212999999999995e-05 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.395194+0000", 2023-10-09T04:19:47.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2210000000000003e-06 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277731+0000", 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88253674599999998 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277845+0000", 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011417300000000001 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.277994+0000", 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014894999999999999 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.278925+0000", 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00093115099999999996 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279241+0000", 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031537999999999998 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281476+0000", 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022349290000000001 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281535+0000", 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9324999999999999e-05 2023-10-09T04:19:47.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281571+0000", 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6480999999999999e-05 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52913 7.13 7:ce6a4229:::benchmark_data_smithi163_346082_object52912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:19:47.092 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.56621377199997, 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85417199600000004, 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52913 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429242+0000", 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:47.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429266+0000", 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4411999999999999e-05 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429273+0000", 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5470000000000002e-06 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429276+0000", 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.898e-06 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429314+0000", 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8690999999999999e-05 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.429321+0000", 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5400000000000001e-06 2023-10-09T04:19:47.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279261+0000", 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84994013899999998 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279332+0000", 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1373999999999998e-05 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.279470+0000", 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013728500000000001 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281216+0000", 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001746446 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281445+0000", 2023-10-09T04:19:47.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022927900000000001 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283362+0000", 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019170459999999999 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283393+0000", 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0997000000000002e-05 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283414+0000", 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0826e-05 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52963 7.13 7:c8f4aee2:::benchmark_data_smithi163_346082_object52962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.472521+0000", 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.52293516500004, 2023-10-09T04:19:47.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.81096846499999997, 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52963 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472521+0000", 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472521+0000", 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472520+0000", 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472555+0000", 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4545000000000002e-05 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472568+0000", 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3464e-05 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472574+0000", 2023-10-09T04:19:47.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9070000000000004e-06 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472636+0000", 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1921999999999994e-05 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.472664+0000", 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7872999999999999e-05 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.280145+0000", 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80748120000000001 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.280241+0000", 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5638999999999997e-05 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.280364+0000", 2023-10-09T04:19:47.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123497 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281414+0000", 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010499019999999999 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281613+0000", 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019900399999999999 2023-10-09T04:19:47.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283440+0000", 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018269479999999999 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283464+0000", 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3978999999999999e-05 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283489+0000", 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5471000000000001e-05 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:52997 7.13 7:cdf64a9a:::benchmark_data_smithi163_346082_object52996:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.48457576099997, 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.77769543699999999, 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.108 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 52997 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510880+0000", 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510911+0000", 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0945000000000003e-05 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510916+0000", 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5419999999999996e-06 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510919+0000", 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.27e-06 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510966+0000", 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6329999999999999e-05 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.510973+0000", 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1080000000000004e-06 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281637+0000", 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.77066409199999997 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281739+0000", 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010242300000000001 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.281872+0000", 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013302100000000001 2023-10-09T04:19:47.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.283420+0000", 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001547732 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.285133+0000", 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017123399999999999 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288476+0000", 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033432010000000001 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288536+0000", 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0207999999999997e-05 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288576+0000", 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9786000000000002e-05 2023-10-09T04:19:47.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:53010 7.13 7:cd640b3a:::benchmark_data_smithi163_346082_object53009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:08.531489+0000", 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 578.46396676500001, 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75803050299999997, 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:47.116 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 53010 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531489+0000", 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531489+0000", 2023-10-09T04:19:47.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531480+0000", 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999905 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531540+0000", 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0124999999999998e-05 2023-10-09T04:19:47.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531564+0000", 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3927e-05 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531574+0000", 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9429999999999995e-06 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531699+0000", 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012563799999999999 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:08.531722+0000", 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2609e-05 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.282598+0000", 2023-10-09T04:19:47.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75087631399999999 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.282676+0000", 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7862999999999996e-05 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.282756+0000", 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0043000000000006e-05 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.285163+0000", 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024068679999999999 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.288608+0000", 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003444754 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.289432+0000", 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000824009 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.289486+0000", 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4159000000000001e-05 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:09.289520+0000", 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3914000000000001e-05 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.123 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35475.0:170449 8.b e290/276 8:dcbe724e:::benchmark_data_smithi163_404719_object79865:head v 290'10663, mlcod=248'2592)", 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:31.712419+0000", 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.28303679300001, 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.097435250000000001, 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.712419+0000", 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.712419+0000", 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.712419+0000", 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.712424+0000", 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8930000000000004e-06 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.792791+0000", 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.080367298000000004 2023-10-09T04:19:47.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.794320+0000", 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001528668 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.796153+0000", 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018333710000000001 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.796176+0000", 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3244e-05 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.809821+0000", 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013644346 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.809855+0000", 2023-10-09T04:19:47.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3958999999999998e-05 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:22518 9.c 9:37620917:::benchmark_data_smithi163_464650_object22517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:24.732638+0000", 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 82.262817935000001, 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.097274786000000002, 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.129 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 22518 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732638+0000", 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732638+0000", 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732637+0000", 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732664+0000", 2023-10-09T04:19:47.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6859000000000002e-05 2023-10-09T04:19:47.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:47.132 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:46 smithi181 ceph-mon[196979]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.132 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:46 smithi181 ceph-mon[196979]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.133 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:46 smithi181 ceph-mon[196979]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.133 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:46 smithi181 ceph-mon[196979]: pgmap v922: 119 pgs: 2 peering, 1 active+undersized+remapped, 1 active+remapped+backfilling, 115 active+clean; 6.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 11 MiB/s wr, 182 op/s; 4782/292653 objects misplaced (1.634%) 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732676+0000", 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1793e-05 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732682+0000", 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0580000000000003e-06 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732728+0000", 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5649000000000003e-05 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732841+0000", 2023-10-09T04:19:47.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011328900000000001 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.732987+0000", 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014564299999999999 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.733852+0000", 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086507800000000003 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.734427+0000", 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057551200000000003 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.829846+0000", 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.095418769000000001 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.829886+0000", 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0431e-05 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:24.829913+0000", 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6466e-05 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.137 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:38929 9.c 9:3774ead0:::benchmark_data_smithi163_464650_object38928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:37.798770+0000", 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 69.196686559, 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10792890500000001, 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38929 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798770+0000", 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798770+0000", 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798768+0000", 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798826+0000", 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8443999999999998e-05 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798842+0000", 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5053e-05 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798848+0000", 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2009999999999996e-06 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798899+0000", 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1659999999999997e-05 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.799042+0000", 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014276499999999999 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.799190+0000", 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014819799999999999 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.800645+0000", 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014543399999999999 2023-10-09T04:19:47.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.801886+0000", 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001241027 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.906605+0000", 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10471944399999999 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.906659+0000", 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3653000000000003e-05 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.906698+0000", 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9570000000000002e-05 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:72489 9.c 9:3680aebc:::benchmark_data_smithi163_464650_object72488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:09.567138+0000", 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 37.428318191000002, 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15767242400000001, 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72489 2023-10-09T04:19:47.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567138+0000", 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567138+0000", 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567136+0000", 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567177+0000", 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1226999999999997e-05 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567193+0000", 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5841999999999998e-05 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567201+0000", 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0819999999999999e-06 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567258+0000", 2023-10-09T04:19:47.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6674e-05 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567424+0000", 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016615099999999999 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567603+0000", 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178467 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.570436+0000", 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00283358 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573203+0000", 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027669740000000002 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.724713+0000", 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.151509703 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.724770+0000", 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7485000000000003e-05 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.724810+0000", 2023-10-09T04:19:47.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0048e-05 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:72498 9.c 9:39f48ce2:::benchmark_data_smithi163_464650_object72497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:09.573382+0000", 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 37.422074164000001, 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15170962700000001, 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:47.152 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72498 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573382+0000", 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573382+0000", 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573381+0000", 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:47.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573430+0000", 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9427e-05 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573440+0000", 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0009e-05 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573445+0000", 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8029999999999996e-06 2023-10-09T04:19:47.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573490+0000", 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5427999999999999e-05 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573634+0000", 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014404500000000001 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573736+0000", 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101561 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.575906+0000", 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021696910000000001 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.576350+0000", 2023-10-09T04:19:47.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044416399999999998 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.725020+0000", 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14867 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.725056+0000", 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6068000000000001e-05 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:47.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.725092+0000", 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5540999999999997e-05 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:47.159 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:19:47.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:47.160 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:19:47.161 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:19:47.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[213841]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[213841]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[213841]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[213841]: pgmap v922: 119 pgs: 2 peering, 1 active+undersized+remapped, 1 active+remapped+backfilling, 115 active+clean; 6.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 11 MiB/s wr, 182 op/s; 4782/292653 objects misplaced (1.634%) 2023-10-09T04:19:47.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[221067]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.161 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[221067]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[221067]: failed to encode map e341 with expected crc 2023-10-09T04:19:47.162 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:46 smithi163 ceph-mon[221067]: pgmap v922: 119 pgs: 2 peering, 1 active+undersized+remapped, 1 active+remapped+backfilling, 115 active+clean; 6.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 11 MiB/s wr, 182 op/s; 4782/292653 objects misplaced (1.634%) 2023-10-09T04:19:47.162 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:19:47.162 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:19:47.180 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:19:47.223 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:47.233 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.234 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:19:47.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:47.330 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:47.455 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:47.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:47.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:47.644 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:47.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:47.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:47.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:19:47.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:47.845 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:48.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/548069036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:48.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1646285455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:19:48.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:19:48.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/548069036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:48.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1646285455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:19:48.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:19:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/548069036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1646285455' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:19:48.094 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:19:48.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:19:48.121 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:48.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:48.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:48.279 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:48.368 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:48.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:48.463 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:48.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:48.537 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:19:48.566 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:48.686 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:48.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:48.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:48.831 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:19:48.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:48 smithi181 ceph-mon[196979]: pgmap v923: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 6.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 192 B/s rd, 21 MiB/s wr, 482 op/s; 9336/293412 objects misplaced (3.182%); 19 MiB/s, 302 objects/s recovering 2023-10-09T04:19:48.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:19:48.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:48 smithi181 ceph-mon[196979]: osdmap e342: 8 total, 8 up, 7 in 2023-10-09T04:19:48.837 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:48 smithi181 ceph-mon[196979]: failed to encode map e342 with expected crc 2023-10-09T04:19:48.859 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:48.978 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:48.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:49.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[213841]: pgmap v923: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 6.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 192 B/s rd, 21 MiB/s wr, 482 op/s; 9336/293412 objects misplaced (3.182%); 19 MiB/s, 302 objects/s recovering 2023-10-09T04:19:49.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:19:49.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[213841]: osdmap e342: 8 total, 8 up, 7 in 2023-10-09T04:19:49.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[213841]: failed to encode map e342 with expected crc 2023-10-09T04:19:49.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[221067]: pgmap v923: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 6.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 192 B/s rd, 21 MiB/s wr, 482 op/s; 9336/293412 objects misplaced (3.182%); 19 MiB/s, 302 objects/s recovering 2023-10-09T04:19:49.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:19:49.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[221067]: osdmap e342: 8 total, 8 up, 7 in 2023-10-09T04:19:49.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:48 smithi163 ceph-mon[221067]: failed to encode map e342 with expected crc 2023-10-09T04:19:49.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:49.120 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:49.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:49.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:49.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:49.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:49.371 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:49.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:19:49.606 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:49.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:49.712 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:49.804 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:49.875 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:49.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:49.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: failed to encode map e342 with expected crc 2023-10-09T04:19:49.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: failed to encode map e342 with expected crc 2023-10-09T04:19:49.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 5]}]: dispatch 2023-10-09T04:19:49.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]: dispatch 2023-10-09T04:19:49.933 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: failed to encode map e342 with expected crc 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: failed to encode map e342 with expected crc 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 5]}]: dispatch 2023-10-09T04:19:49.934 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]: dispatch 2023-10-09T04:19:49.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T04:19:49.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-10-09T04:19:49.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T04:19:49.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:49.978 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:50.006 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: failed to encode map e342 with expected crc 2023-10-09T04:19:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: failed to encode map e342 with expected crc 2023-10-09T04:19:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 5]}]: dispatch 2023-10-09T04:19:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]: dispatch 2023-10-09T04:19:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]: dispatch 2023-10-09T04:19:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-10-09T04:19:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T04:19:50.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:50.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:19:50.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:50.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:50.553 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:50.637 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:19:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:19:50] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-10-09T04:19:50.639 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:50.769 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:50.871 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:50.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:50.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:51.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: pgmap v925: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 14 MiB/s wr, 432 op/s; 9336/292359 objects misplaced (3.193%); 17 MiB/s, 268 objects/s recovering 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 5]}]': finished 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]': finished 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: osdmap e343: 8 total, 8 up, 7 in 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: failed to encode map e343 with expected crc 2023-10-09T04:19:51.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[213841]: failed to encode map e343 with expected crc 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: pgmap v925: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 14 MiB/s wr, 432 op/s; 9336/292359 objects misplaced (3.193%); 17 MiB/s, 268 objects/s recovering 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 5]}]': finished 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]': finished 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: osdmap e343: 8 total, 8 up, 7 in 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: failed to encode map e343 with expected crc 2023-10-09T04:19:51.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:50 smithi163 ceph-mon[221067]: failed to encode map e343 with expected crc 2023-10-09T04:19:51.018 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:51.172 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:19:51.172 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:19:51.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: pgmap v925: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 170 B/s rd, 14 MiB/s wr, 432 op/s; 9336/292359 objects misplaced (3.193%); 17 MiB/s, 268 objects/s recovering 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 5]}]': finished 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [4, 5]}]': finished 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 2]}]': finished 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: osdmap e343: 8 total, 8 up, 7 in 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: failed to encode map e343 with expected crc 2023-10-09T04:19:51.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:50 smithi181 ceph-mon[196979]: failed to encode map e343 with expected crc 2023-10-09T04:19:51.233 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:51.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:51.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:51.360 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:51.398 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:51.464 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:51.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:51.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:19:51.640 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:51.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:19:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:51.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:19:51.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:51.912 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:19:51.939 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:52.017 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:52.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: osdmap e344: 8 total, 8 up, 7 in 2023-10-09T04:19:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3544950983' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:52.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:52.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:52.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:52.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.172 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: osdmap e344: 8 total, 8 up, 7 in 2023-10-09T04:19:52.172 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.172 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.172 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.172 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.173 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.173 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.173 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3544950983' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: osdmap e344: 8 total, 8 up, 7 in 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.174 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: failed to encode map e344 with expected crc 2023-10-09T04:19:52.175 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3544950983' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:19:52.176 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:52.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:19:52.814 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:19:53.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:53.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: pgmap v928: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 170 B/s rd, 12 MiB/s wr, 927 op/s; 2/283035 objects degraded (0.001%); 8031/283035 objects misplaced (2.837%); 26 MiB/s, 421 objects/s recovering 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 2/283035 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3544950983' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: osdmap e345: 8 total, 8 up, 7 in 2023-10-09T04:19:53.139 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:52 smithi181 ceph-mon[196979]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:53.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: pgmap v928: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 170 B/s rd, 12 MiB/s wr, 927 op/s; 2/283035 objects degraded (0.001%); 8031/283035 objects misplaced (2.837%); 26 MiB/s, 421 objects/s recovering 2023-10-09T04:19:53.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 2/283035 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3544950983' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: osdmap e345: 8 total, 8 up, 7 in 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.205 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[213841]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: pgmap v928: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 170 B/s rd, 12 MiB/s wr, 927 op/s; 2/283035 objects degraded (0.001%); 8031/283035 objects misplaced (2.837%); 26 MiB/s, 421 objects/s recovering 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 2/283035 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3544950983' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: osdmap e345: 8 total, 8 up, 7 in 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.206 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.207 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:52 smithi163 ceph-mon[221067]: failed to encode map e345 with expected crc 2023-10-09T04:19:53.209 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:54.015 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:54.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:54.118 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:54.181 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:54.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:54.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:54.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:54.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:54.637 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:54.806 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:54.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:54.914 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:54.934 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:19:54.992 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:55.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:19:55.049 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:19:55.081 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:19:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:54 smithi163 ceph-mon[221067]: pgmap v930: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 2/280965 objects degraded (0.001%); 8031/280965 objects misplaced (2.858%); 11 MiB/s, 173 objects/s recovering 2023-10-09T04:19:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1718288846' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:19:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:19:55.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:54 smithi163 ceph-mon[213841]: pgmap v930: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 2/280965 objects degraded (0.001%); 8031/280965 objects misplaced (2.858%); 11 MiB/s, 173 objects/s recovering 2023-10-09T04:19:55.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1718288846' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:19:55.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:19:55.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:55.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:54 smithi181 ceph-mon[196979]: pgmap v930: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 2/280965 objects degraded (0.001%); 8031/280965 objects misplaced (2.858%); 11 MiB/s, 173 objects/s recovering 2023-10-09T04:19:55.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1718288846' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:19:55.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:19:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:55.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:55.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:19:55.149 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:19:55.186 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:19:55.186 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:19:55.186 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:19:55.186 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:19:55.186 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:19:55.186 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:19:55.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:55.223 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:19:55.272 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:19:55.274 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:55.290 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:19:55.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:55.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:55.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:55.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:19:55.493 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:55.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:55.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:55.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:55.715 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:55.835 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:55.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:55.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:55.980 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:19:55.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: osdmap e346: 8 total, 8 up, 7 in 2023-10-09T04:19:55.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: failed to encode map e346 with expected crc 2023-10-09T04:19:55.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: failed to encode map e346 with expected crc 2023-10-09T04:19:55.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: failed to encode map e346 with expected crc 2023-10-09T04:19:55.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: failed to encode map e346 with expected crc 2023-10-09T04:19:55.981 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3487267051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:55.985 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:56.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:19:56.216 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:56.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: osdmap e346: 8 total, 8 up, 7 in 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3487267051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:19:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: osdmap e346: 8 total, 8 up, 7 in 2023-10-09T04:19:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: failed to encode map e346 with expected crc 2023-10-09T04:19:56.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3487267051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:19:56.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:19:56.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:19:56.442 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:56.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:19:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:56.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:19:56.663 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:56.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:56.782 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:56.916 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:56 smithi181 ceph-mon[196979]: pgmap v932: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 2/280194 objects degraded (0.001%); 8031/280194 objects misplaced (2.866%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:19:56.941 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:19:56.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:56.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:57.023 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:56 smithi163 ceph-mon[213841]: pgmap v932: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 2/280194 objects degraded (0.001%); 8031/280194 objects misplaced (2.866%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:19:57.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:56 smithi163 ceph-mon[221067]: pgmap v932: 119 pgs: 1 activating+degraded, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 2/280194 objects degraded (0.001%); 8031/280194 objects misplaced (2.866%); 9.6 MiB/s, 154 objects/s recovering 2023-10-09T04:19:57.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:19:57.122 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:57.299 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:19:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:57.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:19:57.407 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:57.469 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:57.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:57.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:57.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:19:57.858 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:58.054 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:58.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:19:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:58.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:19:58.188 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:19:58.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2/280194 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:19:58.214 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2/280194 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:19:58.217 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:58.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2/280194 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:19:58.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:19:58.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:58.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:19:58.468 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:19:58.503 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:19:58.533 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:58.692 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:19:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:58.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:19:58.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:19:58.881 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:59.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:58 smithi163 ceph-mon[221067]: pgmap v933: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 6882/271668 objects misplaced (2.533%); 16 MiB/s, 254 objects/s recovering 2023-10-09T04:19:59.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:58 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/280194 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:19:59.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:58 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:19:59.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:58 smithi163 ceph-mon[213841]: pgmap v933: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 6882/271668 objects misplaced (2.533%); 16 MiB/s, 254 objects/s recovering 2023-10-09T04:19:59.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:58 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/280194 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:19:59.245 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:58 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:19:59.257 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:19:59.258 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:19:59.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:19:59.258 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:19:59.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.258 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80494 7.e 7:79ff37ea:::benchmark_data_smithi163_346082_object80493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 561.80296800099995, 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.00624114, 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.259 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80494 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449628+0000", 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449627+0000", 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449685+0000", 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8378000000000002e-05 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449697+0000", 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2427e-05 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449703+0000", 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2379999999999997e-06 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449751+0000", 2023-10-09T04:19:59.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8476999999999997e-05 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.449761+0000", 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0378e-05 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441447+0000", 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99168547399999996 2023-10-09T04:19:59.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441622+0000", 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017500600000000001 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.441827+0000", 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020516999999999999 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451473+0000", 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096461649999999999 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452805+0000", 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013312770000000001 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454609+0000", 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018042959999999999 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.454649+0000", 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0256999999999998e-05 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.455869+0000", 2023-10-09T04:19:59.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001219649 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80509 7.e 7:77548b6a:::benchmark_data_smithi163_346082_object80508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 561.78413003399999, 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0024772179999999, 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.268 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80509 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468466+0000", 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468464+0000", 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468512+0000", 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615000000000001e-05 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468527+0000", 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4778e-05 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468534+0000", 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2049999999999996e-06 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468595+0000", 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0865000000000001e-05 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.468603+0000", 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2439999999999996e-06 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445422+0000", 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97681920499999997 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445635+0000", 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000213382 2023-10-09T04:19:59.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.445836+0000", 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00020043 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451507+0000", 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056710220000000004 2023-10-09T04:19:59.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.452890+0000", 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013838699999999999 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469631+0000", 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01674059 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.469678+0000", 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6551999999999997e-05 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.470943+0000", 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00126515 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.276 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80511 7.e 7:74ae5f43:::benchmark_data_smithi163_346082_object80510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 561.78256773099997, 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002226042, 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:59.277 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80511 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470028+0000", 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470027+0000", 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470082+0000", 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5106999999999999e-05 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470093+0000", 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0738e-05 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470098+0000", 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5820000000000003e-06 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470142+0000", 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4261000000000002e-05 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.470148+0000", 2023-10-09T04:19:59.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0170000000000002e-06 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.447968+0000", 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97782021900000005 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448099+0000", 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013048700000000001 2023-10-09T04:19:59.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.448204+0000", 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105266 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451513+0000", 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033096240000000002 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.456169+0000", 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046552060000000003 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471004+0000", 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014834926999999999 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.471062+0000", 2023-10-09T04:19:59.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8873999999999999e-05 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472254+0000", 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011913939999999999 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.285 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80513 7.e 7:768b3fe4:::benchmark_data_smithi163_346082_object80512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 561.78064202099995, 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0014613109999999, 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80513 2023-10-09T04:19:59.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471954+0000", 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.471952+0000", 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472006+0000", 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3981999999999999e-05 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472018+0000", 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1924e-05 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472023+0000", 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6999999999999999e-06 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472070+0000", 2023-10-09T04:19:59.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7218000000000001e-05 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.472078+0000", 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8830000000000006e-06 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449809+0000", 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97773131000000002 2023-10-09T04:19:59.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.449917+0000", 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107543 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.450027+0000", 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010959399999999999 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.451621+0000", 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015948360000000001 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.466881+0000", 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015260029 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472336+0000", 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054546960000000002 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.472393+0000", 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6829000000000001e-05 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.473415+0000", 2023-10-09T04:19:59.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001021852 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131725 8.3 8:c2b90b78:::benchmark_data_smithi163_404719_object41141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.77805959299999, 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1240774949999999, 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.294 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131725 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:59.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474538+0000", 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4329999999999998e-06 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474549+0000", 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0927000000000001e-05 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474553+0000", 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8800000000000001e-06 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474594+0000", 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1332999999999999e-05 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474602+0000", 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468816+0000", 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99421445799999997 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468827+0000", 2023-10-09T04:19:59.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1093e-05 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.586018+0000", 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.117190543 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.588171+0000", 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021532040000000001 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.588321+0000", 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015008999999999999 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598261+0000", 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099394879999999998 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598299+0000", 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8522000000000001e-05 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598361+0000", 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1239000000000001e-05 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598457+0000", 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6027999999999996e-05 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598531+0000", 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4678999999999994e-05 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598613+0000", 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2201999999999997e-05 2023-10-09T04:19:59.303 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131729 8.3 8:c6b15366:::benchmark_data_smithi163_404719_object41145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.76797456899999, 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1141904230000002, 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.304 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131729 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484620+0000", 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484626+0000", 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0870000000000003e-06 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484639+0000", 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3502e-05 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484647+0000", 2023-10-09T04:19:59.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484692+0000", 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5667999999999997e-05 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484702+0000", 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5850000000000006e-06 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468853+0000", 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98415053600000002 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468860+0000", 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1189999999999999e-06 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591501+0000", 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1226411199999999 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591590+0000", 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9573000000000005e-05 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591667+0000", 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7033000000000006e-05 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598305+0000", 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066379239999999999 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598404+0000", 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9100999999999999e-05 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598650+0000", 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024551400000000001 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598726+0000", 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6175999999999997e-05 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598754+0000", 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7898999999999998e-05 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598811+0000", 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7370999999999997e-05 2023-10-09T04:19:59.313 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131731 8.3 8:c392b46b:::benchmark_data_smithi163_404719_object41147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.767902972, 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1142496469999998, 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131731 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484692+0000", 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484695+0000", 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7630000000000001e-06 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484701+0000", 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1909999999999997e-06 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484705+0000", 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9700000000000001e-06 2023-10-09T04:19:59.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484752+0000", 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6964e-05 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484761+0000", 2023-10-09T04:19:59.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5590000000000006e-06 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468882+0000", 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98412151699999995 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468889+0000", 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8260000000000004e-06 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594688+0000", 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1257991060000001 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594798+0000", 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010983 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594900+0000", 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102451 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598656+0000", 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037554839999999999 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598683+0000", 2023-10-09T04:19:59.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6998000000000001e-05 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598703+0000", 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9925999999999999e-05 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598843+0000", 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014050299999999999 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598872+0000", 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8965000000000001e-05 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598942+0000", 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0003e-05 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.323 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131738 8.3 8:cc1c5c5f:::benchmark_data_smithi163_404719_object41154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.757278571, 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1194174960000001, 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.324 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131738 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:19:59.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495319+0000", 2023-10-09T04:19:59.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9379999999999999e-06 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495324+0000", 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1200000000000001e-06 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495326+0000", 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4820000000000001e-06 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495378+0000", 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1295000000000003e-05 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495384+0000", 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4320000000000004e-06 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468935+0000", 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97355068700000003 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468944+0000", 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.816e-06 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598966+0000", 2023-10-09T04:19:59.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.130022938 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.599049+0000", 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2161000000000002e-05 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.599124+0000", 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4906000000000004e-05 2023-10-09T04:19:59.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614430+0000", 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015306491 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614476+0000", 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6394999999999999e-05 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614536+0000", 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9799000000000001e-05 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614566+0000", 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0008000000000001e-05 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614623+0000", 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6487999999999999e-05 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614735+0000", 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111797 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131739 8.3 8:ccd4c823:::benchmark_data_smithi163_404719_object41155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.736576654, 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.098935633, 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131739 2023-10-09T04:19:59.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516018+0000", 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516024+0000", 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.421e-06 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516038+0000", 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3558e-05 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516044+0000", 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4400000000000002e-06 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516099+0000", 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4361999999999999e-05 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516110+0000", 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1661000000000001e-05 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468972+0000", 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95286181000000003 2023-10-09T04:19:59.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468979+0000", 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6800000000000004e-06 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602170+0000", 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.133191676 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602304+0000", 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013313999999999999 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602429+0000", 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012518499999999999 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614498+0000", 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012068972000000001 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614780+0000", 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000281978 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614813+0000", 2023-10-09T04:19:59.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3337000000000003e-05 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614843+0000", 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0256999999999998e-05 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614872+0000", 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.898e-05 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614955+0000", 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2304000000000003e-05 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131750 8.3 8:caad7466:::benchmark_data_smithi163_404719_object41166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.71977130600001, 2023-10-09T04:19:59.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0823976559999999, 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131750 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:19:59.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532826+0000", 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.779e-06 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532829+0000", 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4290000000000001e-06 2023-10-09T04:19:59.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532832+0000", 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3879999999999998e-06 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532884+0000", 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2642999999999997e-05 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532892+0000", 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6290000000000001e-06 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469000+0000", 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93610853400000005 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469007+0000", 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.605999+0000", 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.136991761 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.606144+0000", 2023-10-09T04:19:59.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014551200000000001 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.606301+0000", 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000156458 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614505+0000", 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082041619999999992 2023-10-09T04:19:59.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614978+0000", 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047337600000000002 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615009+0000", 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0454999999999999e-05 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615040+0000", 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1758000000000003e-05 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615086+0000", 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5494999999999997e-05 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615222+0000", 2023-10-09T04:19:59.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013613699999999999 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131757 8.3 8:cde86114:::benchmark_data_smithi163_404719_object41173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.701488418, 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0643897739999999, 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.353 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131757 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551105+0000", 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551114+0000", 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2609999999999993e-06 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551130+0000", 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5934e-05 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551137+0000", 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1019999999999999e-06 2023-10-09T04:19:59.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551191+0000", 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4138000000000002e-05 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551202+0000", 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.164e-05 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469028+0000", 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91782591899999999 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469035+0000", 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2580000000000001e-06 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610215+0000", 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1411807170000001 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610340+0000", 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012489900000000001 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610434+0000", 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4154999999999996e-05 2023-10-09T04:19:59.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614745+0000", 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043102599999999998 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615262+0000", 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00051697899999999998 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615294+0000", 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2675000000000001e-05 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615328+0000", 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3495999999999999e-05 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615374+0000", 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6514000000000002e-05 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615497+0000", 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122583 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131773 8.3 8:cc396e7f:::benchmark_data_smithi163_404719_object41189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 215.668165098, 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0561476569999999, 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131773 2023-10-09T04:19:59.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584429+0000", 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584435+0000", 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4090000000000002e-06 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584448+0000", 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.361e-05 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584456+0000", 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6389999999999992e-06 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584515+0000", 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8638999999999999e-05 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584525+0000", 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0314e-05 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469134+0000", 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88460910100000001 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469142+0000", 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0639999999999994e-06 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615534+0000", 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1463917850000001 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615640+0000", 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010655800000000001 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615751+0000", 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011056 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627128+0000", 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011377376 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631239+0000", 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041103110000000002 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631277+0000", 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8482000000000002e-05 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640379+0000", 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0091021850000000005 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640436+0000", 2023-10-09T04:19:59.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6545000000000002e-05 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640578+0000", 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142261 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.372 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131846 8.3 8:c021821e:::benchmark_data_smithi163_404719_object41262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.464231461, 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.852533604, 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131846 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.373 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788363+0000", 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788368+0000", 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5560000000000001e-06 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788375+0000", 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1559999999999998e-06 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788379+0000", 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0430000000000001e-06 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788435+0000", 2023-10-09T04:19:59.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5859000000000002e-05 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788441+0000", 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.156e-06 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619638+0000", 2023-10-09T04:19:59.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.831196571 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619758+0000", 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120253 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619848+0000", 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9886e-05 2023-10-09T04:19:59.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631195+0000", 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011347183 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631309+0000", 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113992 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.379 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640695+0000", 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093864970000000006 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640734+0000", 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8285999999999999e-05 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640782+0000", 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615999999999997e-05 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640898+0000", 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011548900000000001 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.381 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131856 8.3 8:c67caa71:::benchmark_data_smithi163_404719_object41272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.433073503, 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82161764000000004, 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.382 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131856 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819524+0000", 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1560000000000002e-06 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819529+0000", 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2159999999999997e-06 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819532+0000", 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.79e-06 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819581+0000", 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8748999999999997e-05 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819589+0000", 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8879999999999997e-06 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623021+0000", 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80343243799999997 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623126+0000", 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105138 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623256+0000", 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129383 2023-10-09T04:19:59.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631246+0000", 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079899900000000006 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631345+0000", 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8989999999999999e-05 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640971+0000", 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096260070000000007 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641005+0000", 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4100000000000002e-05 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.389 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641043+0000", 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8825000000000002e-05 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641140+0000", 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6241000000000002e-05 2023-10-09T04:19:59.390 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.391 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.391 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.391 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131865 8.3 8:c397b806:::benchmark_data_smithi163_404719_object41281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.391 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:19:59.392 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.41116778, 2023-10-09T04:19:59.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79990704499999998, 2023-10-09T04:19:59.392 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.392 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.392 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131865 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:19:59.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841427+0000", 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841432+0000", 2023-10-09T04:19:59.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7679999999999997e-06 2023-10-09T04:19:59.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841444+0000", 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1588e-05 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841449+0000", 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0769999999999998e-06 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841505+0000", 2023-10-09T04:19:59.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6144000000000003e-05 2023-10-09T04:19:59.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841514+0000", 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6100000000000006e-06 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627156+0000", 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78564268000000004 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627290+0000", 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013397600000000001 2023-10-09T04:19:59.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627401+0000", 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011101 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640330+0000", 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012928057 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640641+0000", 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031165099999999997 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641173+0000", 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00053144399999999995 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641202+0000", 2023-10-09T04:19:59.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9847000000000001e-05 2023-10-09T04:19:59.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641241+0000", 2023-10-09T04:19:59.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8713e-05 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641335+0000", 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3648999999999997e-05 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131881 8.3 8:cd41d762:::benchmark_data_smithi163_404719_object41297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.374483594, 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79379251699999998, 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131881 2023-10-09T04:19:59.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.405 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:19:59.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:19:59.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878111+0000", 2023-10-09T04:19:59.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:59.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878118+0000", 2023-10-09T04:19:59.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.956e-06 2023-10-09T04:19:59.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.408 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878131+0000", 2023-10-09T04:19:59.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3141e-05 2023-10-09T04:19:59.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878138+0000", 2023-10-09T04:19:59.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8109999999999997e-06 2023-10-09T04:19:59.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878215+0000", 2023-10-09T04:19:59.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7330000000000007e-05 2023-10-09T04:19:59.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878228+0000", 2023-10-09T04:19:59.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3209e-05 2023-10-09T04:19:59.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631369+0000", 2023-10-09T04:19:59.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75314052899999995 2023-10-09T04:19:59.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631495+0000", 2023-10-09T04:19:59.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125823 2023-10-09T04:19:59.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631615+0000", 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120013 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640655+0000", 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0090400859999999993 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640937+0000", 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028247100000000001 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649876+0000", 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089392050000000004 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671738+0000", 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021861968999999998 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671788+0000", 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9493999999999997e-05 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671905+0000", 2023-10-09T04:19:59.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011675699999999999 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131884 8.3 8:c4f20fc8:::benchmark_data_smithi163_404719_object41300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.36927987199999, 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78877238199999999, 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131884 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883314+0000", 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883321+0000", 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0639999999999996e-06 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883334+0000", 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2595000000000001e-05 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883340+0000", 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3409999999999999e-06 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883406+0000", 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6045999999999998e-05 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883416+0000", 2023-10-09T04:19:59.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0387000000000001e-05 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635386+0000", 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75196957900000005 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635512+0000", 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012591300000000001 2023-10-09T04:19:59.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635633+0000", 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121377 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640702+0000", 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050688790000000001 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641370+0000", 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00066799299999999997 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649915+0000", 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085445550000000006 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671963+0000", 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022048742 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671998+0000", 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4585000000000002e-05 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672088+0000", 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0163999999999999e-05 2023-10-09T04:19:59.425 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131932 8.3 8:c8cbc02f:::benchmark_data_smithi163_404719_object41348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.244463137, 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.66412386599999995, 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.426 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131932 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008131+0000", 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008140+0000", 2023-10-09T04:19:59.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.901e-06 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008151+0000", 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008158+0000", 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3949999999999998e-06 2023-10-09T04:19:59.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008225+0000", 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7541999999999998e-05 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008237+0000", 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1732999999999999e-05 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641390+0000", 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.633153454 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641535+0000", 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014420899999999999 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641667+0000", 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205199999999999 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649884+0000", 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082173850000000007 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649944+0000", 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0229000000000002e-05 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649976+0000", 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1479000000000001e-05 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672141+0000", 2023-10-09T04:19:59.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022165625000000001 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672185+0000", 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3373e-05 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672256+0000", 2023-10-09T04:19:59.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1606000000000005e-05 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131959 8.3 8:c13e43ce:::benchmark_data_smithi163_404719_object41375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 214.17575208599999, 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.59562893100000003, 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.435 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131959 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076848+0000", 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076858+0000", 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0603e-05 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076863+0000", 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4159999999999997e-06 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076923+0000", 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9760999999999999e-05 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076933+0000", 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0849999999999999e-05 2023-10-09T04:19:59.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645730+0000", 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.56879615900000002 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645860+0000", 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013080899999999999 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645984+0000", 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012326399999999999 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658944+0000", 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012960704999999999 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658951+0000", 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0469999999999996e-06 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658992+0000", 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1028000000000001e-05 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672327+0000", 2023-10-09T04:19:59.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013334278999999999 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672372+0000", 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4783000000000001e-05 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672472+0000", 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010089 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.443 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:108770 9.a 9:5e149d9c:::benchmark_data_smithi163_464650_object10044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:57.999631+0000", 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 1.2529648609999999, 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.27594479, 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:19:59.444 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 108770 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:57.999631+0000", 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:19:59.445 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:57.999631+0000", 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:57.999630+0000", 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:57.999633+0000", 2023-10-09T04:19:59.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2699999999999999e-06 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:57.999638+0000", 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6910000000000002e-06 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:57.999642+0000", 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6289999999999999e-06 2023-10-09T04:19:59.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.002462+0000", 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028195989999999999 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.002577+0000", 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115538 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.002655+0000", 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7885000000000004e-05 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.006167+0000", 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035124280000000002 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011810+0000", 2023-10-09T04:19:59.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056430810000000003 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.275490+0000", 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26367953900000002 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.275536+0000", 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6470000000000001e-05 2023-10-09T04:19:59.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.275576+0000", 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9038000000000001e-05 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:19:59.451 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:19:59.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:19:59.452 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:58 smithi181 ceph-mon[196979]: pgmap v933: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 6882/271668 objects misplaced (2.533%); 16 MiB/s, 254 objects/s recovering 2023-10-09T04:19:59.452 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:58 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/280194 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:19:59.452 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:58 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:19:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:59.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:19:59.554 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:59.627 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:59.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:19:59.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:19:59.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:19:59.740 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:59.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:19:59.884 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:19:59.985 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:20:00.130 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2965754941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:00.130 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:19:59 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:00.131 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2965754941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:00.131 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:19:59 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:00.182 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:20:00.242 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2965754941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:00.242 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:19:59 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:00.612 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:00.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:00.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:00.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:00.769 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:00.846 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: pgmap v934: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 6882/269613 objects misplaced (2.553%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: osdmap e347: 8 total, 8 up, 7 in 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: failed to encode map e347 with expected crc 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: failed to encode map e347 with expected crc 2023-10-09T04:20:00.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: failed to encode map e347 with expected crc 2023-10-09T04:20:00.937 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: overall HEALTH_WARN noscrub flag(s) set 2023-10-09T04:20:00.937 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1396146176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:20:00.937 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:20:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:20:00] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-10-09T04:20:00.971 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:20:01.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: pgmap v934: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 6882/269613 objects misplaced (2.553%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:20:01.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:01.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:20:01.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: osdmap e347: 8 total, 8 up, 7 in 2023-10-09T04:20:01.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: failed to encode map e347 with expected crc 2023-10-09T04:20:01.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: failed to encode map e347 with expected crc 2023-10-09T04:20:01.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: failed to encode map e347 with expected crc 2023-10-09T04:20:01.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: overall HEALTH_WARN noscrub flag(s) set 2023-10-09T04:20:01.191 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1396146176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:20:01.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: pgmap v934: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 6882/269613 objects misplaced (2.553%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:20:01.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:01.239 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:20:01.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: osdmap e347: 8 total, 8 up, 7 in 2023-10-09T04:20:01.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: failed to encode map e347 with expected crc 2023-10-09T04:20:01.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: failed to encode map e347 with expected crc 2023-10-09T04:20:01.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: failed to encode map e347 with expected crc 2023-10-09T04:20:01.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: overall HEALTH_WARN noscrub flag(s) set 2023-10-09T04:20:01.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1396146176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:20:01.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:01.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:01.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:01.664 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:01.719 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:01.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:01.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:01.854 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:01.858 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:20:01.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:20:01.952 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:01.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:01.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:02.055 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:02.081 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:02.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:02.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[221067]: failed to encode map e348 with expected crc 2023-10-09T04:20:02.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1396146176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:20:02.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[221067]: osdmap e348: 8 total, 8 up, 7 in 2023-10-09T04:20:02.146 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[221067]: failed to encode map e348 with expected crc 2023-10-09T04:20:02.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[213841]: failed to encode map e348 with expected crc 2023-10-09T04:20:02.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1396146176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:20:02.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[213841]: osdmap e348: 8 total, 8 up, 7 in 2023-10-09T04:20:02.147 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:01 smithi163 ceph-mon[213841]: failed to encode map e348 with expected crc 2023-10-09T04:20:02.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:02.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:02.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:01 smithi181 ceph-mon[196979]: failed to encode map e348 with expected crc 2023-10-09T04:20:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1396146176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:20:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:01 smithi181 ceph-mon[196979]: osdmap e348: 8 total, 8 up, 7 in 2023-10-09T04:20:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:01 smithi181 ceph-mon[196979]: failed to encode map e348 with expected crc 2023-10-09T04:20:02.279 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:02.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:02.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:02.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:02.522 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:02.523 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:02.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:02.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:02.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:02.769 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:02.823 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:02.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:02.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:02.866 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:20:02.867 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:20:02.867 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:20:02.867 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:20:02.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:02.951 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:20:02.964 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:02.978 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:20:03.014 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:20:03.047 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:20:03.077 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:20:03.109 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:20:03.150 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:20:03.154 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:03.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:03.192 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:20:03.202 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.202 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.202 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.202 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.202 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.202 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.203 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.203 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:20:03.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: pgmap v937: 119 pgs: 1 remapped+peering, 2 peering, 1 active+remapped+backfilling, 115 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 946 op/s; 1861/261279 objects misplaced (0.712%); 16 MiB/s, 263 objects/s recovering 2023-10-09T04:20:03.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.231 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: osdmap e349: 8 total, 8 up, 7 in 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: 9.8 starting backfill to osd.3 from (0'0,0'0] MAX to 347'6965 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: 9.8 continuing backfill to osd.5 from (314'900,347'6965] 9:14904db3:::benchmark_data_smithi163_464650_object72627:head to 347'6965 2023-10-09T04:20:03.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2144443918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:03.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: pgmap v937: 119 pgs: 1 remapped+peering, 2 peering, 1 active+remapped+backfilling, 115 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 946 op/s; 1861/261279 objects misplaced (0.712%); 16 MiB/s, 263 objects/s recovering 2023-10-09T04:20:03.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: osdmap e349: 8 total, 8 up, 7 in 2023-10-09T04:20:03.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: 9.8 starting backfill to osd.3 from (0'0,0'0] MAX to 347'6965 2023-10-09T04:20:03.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: 9.8 continuing backfill to osd.5 from (314'900,347'6965] 9:14904db3:::benchmark_data_smithi163_464650_object72627:head to 347'6965 2023-10-09T04:20:03.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2144443918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:03.269 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:03.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: pgmap v937: 119 pgs: 1 remapped+peering, 2 peering, 1 active+remapped+backfilling, 115 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 946 op/s; 1861/261279 objects misplaced (0.712%); 16 MiB/s, 263 objects/s recovering 2023-10-09T04:20:03.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: osdmap e349: 8 total, 8 up, 7 in 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: 9.8 starting backfill to osd.3 from (0'0,0'0] MAX to 347'6965 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: failed to encode map e349 with expected crc 2023-10-09T04:20:03.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: 9.8 continuing backfill to osd.5 from (314'900,347'6965] 9:14904db3:::benchmark_data_smithi163_464650_object72627:head to 347'6965 2023-10-09T04:20:03.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2144443918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:03.317 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:03.396 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:03.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:03.503 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:03.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:20:03.584 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:03.719 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:03.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:03.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:20:03.830 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:03.893 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:03.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:03.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:04.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:04.141 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:03 smithi163 ceph-mon[213841]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:03 smithi163 ceph-mon[213841]: osdmap e350: 8 total, 8 up, 7 in 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:03 smithi163 ceph-mon[213841]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:03 smithi163 ceph-mon[213841]: 9.2 starting backfill to osd.3 from (0'0,0'0] MAX to 347'6939 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:03 smithi163 ceph-mon[213841]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:03 smithi163 ceph-mon[213841]: 9.7 starting backfill to osd.3 from (0'0,0'0] MAX to 347'7072 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:04 smithi163 ceph-mon[221067]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:04 smithi163 ceph-mon[221067]: osdmap e350: 8 total, 8 up, 7 in 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:04 smithi163 ceph-mon[221067]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:04 smithi163 ceph-mon[221067]: 9.2 starting backfill to osd.3 from (0'0,0'0] MAX to 347'6939 2023-10-09T04:20:04.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:04 smithi163 ceph-mon[221067]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:04 smithi163 ceph-mon[221067]: 9.7 starting backfill to osd.3 from (0'0,0'0] MAX to 347'7072 2023-10-09T04:20:04.162 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:04.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:03 smithi181 ceph-mon[196979]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:03 smithi181 ceph-mon[196979]: osdmap e350: 8 total, 8 up, 7 in 2023-10-09T04:20:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:03 smithi181 ceph-mon[196979]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:03 smithi181 ceph-mon[196979]: 9.2 starting backfill to osd.3 from (0'0,0'0] MAX to 347'6939 2023-10-09T04:20:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:03 smithi181 ceph-mon[196979]: failed to encode map e350 with expected crc 2023-10-09T04:20:04.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:03 smithi181 ceph-mon[196979]: 9.7 starting backfill to osd.3 from (0'0,0'0] MAX to 347'7072 2023-10-09T04:20:04.286 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:20:04.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:04.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:04.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:05.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:05.065 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:05.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:05.143 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:05.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:05.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:05.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: pgmap v940: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 112 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 7329/260052 objects misplaced (2.818%); 15 MiB/s, 243 objects/s recovering 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: osdmap e351: 8 total, 8 up, 7 in 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:05.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: pgmap v940: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 112 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 7329/260052 objects misplaced (2.818%); 15 MiB/s, 243 objects/s recovering 2023-10-09T04:20:05.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: osdmap e351: 8 total, 8 up, 7 in 2023-10-09T04:20:05.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.288 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: pgmap v940: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 112 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 7329/260052 objects misplaced (2.818%); 15 MiB/s, 243 objects/s recovering 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: osdmap e351: 8 total, 8 up, 7 in 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: failed to encode map e351 with expected crc 2023-10-09T04:20:05.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:05.293 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:05.469 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:05.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:05.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:05.652 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:05.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:05.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:06.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:06.085 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:06.100 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:20:06.103 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:06.158 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:06.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[221067]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[221067]: osdmap e352: 8 total, 8 up, 7 in 2023-10-09T04:20:06.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[221067]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[221067]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.160 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[221067]: pgmap v943: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 112 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 7329/259602 objects misplaced (2.823%) 2023-10-09T04:20:06.160 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[213841]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[213841]: osdmap e352: 8 total, 8 up, 7 in 2023-10-09T04:20:06.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[213841]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[213841]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.161 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:06 smithi163 ceph-mon[213841]: pgmap v943: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 112 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 7329/259602 objects misplaced (2.823%) 2023-10-09T04:20:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:06.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:06.263 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:06.289 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:06.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:06.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:06.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:06.341 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:20:06.341 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:20:06.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:06 smithi181 ceph-mon[196979]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:06 smithi181 ceph-mon[196979]: osdmap e352: 8 total, 8 up, 7 in 2023-10-09T04:20:06.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:06 smithi181 ceph-mon[196979]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:06 smithi181 ceph-mon[196979]: failed to encode map e352 with expected crc 2023-10-09T04:20:06.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:06 smithi181 ceph-mon[196979]: pgmap v943: 119 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 peering, 2 active+undersized+remapped, 1 active+remapped+backfilling, 112 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 7329/259602 objects misplaced (2.823%) 2023-10-09T04:20:06.434 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:06.500 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:06.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:06.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:06.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:06.759 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:06.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:06.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:06.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:20:06.942 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:06.990 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:07.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:07 smithi163 ceph-mon[213841]: from='client.45854 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:07.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:20:07.297 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:07 smithi181 ceph-mon[196979]: from='client.45854 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:07.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:07 smithi163 ceph-mon[221067]: from='client.45854 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:07.385 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:07.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:07.493 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":943,"stamp":"2023-10-09T04:20:05.828934+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5670125753,"num_objects":86534,"num_object_clones":0,"num_object_copies":259602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7329,"num_objects_unfound":0,"num_objects_dirty":86534,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":111031,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10816,"num_bytes_recovered":704947937,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94963,"ondisk_log_size":94963,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":365,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20620708,"kb_used_data":17039404,"kb_used_omap":126,"kb_used_meta":3580865,"kb_avail":635595356,"statfs":{"total":671965249536,"available":650849644544,"internally_reserved":0,"allocated":17448349696,"data_stored":17439542612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3666806395},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,4,7,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-36634624,"num_objects":-559,"num_object_clones":0,"num_object_copies":-1677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5468,"num_objects_unfound":0,"num_objects_dirty":-559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":559,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.001461"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":471,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.935390+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:19:54.935390+0000","last_peered":"2023-10-09T04:19:54.935390+0000","last_clean":"2023-10-09T04:19:54.935390+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:19:54.935390+0000","last_undegraded":"2023-10-09T04:19:54.935390+0000","last_fullsized":"2023-10-09T04:19:54.935390+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":96,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.014040+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:20:04.014040+0000","last_peered":"2023-10-09T04:20:04.014040+0000","last_clean":"2023-10-09T04:20:04.014040+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:20:04.014040+0000","last_undegraded":"2023-10-09T04:20:04.014040+0000","last_fullsized":"2023-10-09T04:20:04.014040+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":325,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.971381+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:20:01.971381+0000","last_peered":"2023-10-09T04:20:01.971381+0000","last_clean":"2023-10-09T04:20:01.971381+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:20:01.971381+0000","last_undegraded":"2023-10-09T04:20:01.971381+0000","last_fullsized":"2023-10-09T04:20:01.971381+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":464,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.928803+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:19:54.928803+0000","last_peered":"2023-10-09T04:19:54.928803+0000","last_clean":"2023-10-09T04:19:54.928803+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:19:54.928803+0000","last_undegraded":"2023-10-09T04:19:54.928803+0000","last_fullsized":"2023-10-09T04:19:54.928803+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.971384+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:20:01.971384+0000","last_peered":"2023-10-09T04:20:01.971384+0000","last_clean":"2023-10-09T04:20:01.971384+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:20:01.971384+0000","last_undegraded":"2023-10-09T04:20:01.971384+0000","last_fullsized":"2023-10-09T04:20:01.971384+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":317,"reported_epoch":348,"state":"peering","last_fresh":"2023-10-09T04:20:00.982190+0000","last_change":"2023-10-09T04:20:00.982321+0000","last_active":"2023-10-09T04:20:00.982190+0000","last_peered":"2023-10-09T04:19:59.979539+0000","last_clean":"2023-10-09T04:19:59.979539+0000","last_became_active":"2023-10-09T04:16:24.468649+0000","last_became_peered":"2023-10-09T04:16:24.468649+0000","last_unstale":"2023-10-09T04:20:00.982190+0000","last_undegraded":"2023-10-09T04:20:00.982190+0000","last_fullsized":"2023-10-09T04:20:00.982190+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:31.828636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":476,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013750+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:20:04.013750+0000","last_peered":"2023-10-09T04:20:04.013750+0000","last_clean":"2023-10-09T04:20:04.013750+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:20:04.013750+0000","last_undegraded":"2023-10-09T04:20:04.013750+0000","last_fullsized":"2023-10-09T04:20:04.013750+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":323,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979082+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:19:59.979082+0000","last_peered":"2023-10-09T04:19:59.979082+0000","last_clean":"2023-10-09T04:19:59.979082+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:19:59.979082+0000","last_undegraded":"2023-10-09T04:19:59.979082+0000","last_fullsized":"2023-10-09T04:19:59.979082+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":290,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.934646+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:19:54.934646+0000","last_peered":"2023-10-09T04:19:54.934646+0000","last_clean":"2023-10-09T04:19:54.934646+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:19:54.934646+0000","last_undegraded":"2023-10-09T04:19:54.934646+0000","last_fullsized":"2023-10-09T04:19:54.934646+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":746,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939692+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:19:54.939692+0000","last_peered":"2023-10-09T04:19:54.939692+0000","last_clean":"2023-10-09T04:19:54.939692+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:19:54.939692+0000","last_undegraded":"2023-10-09T04:19:54.939692+0000","last_fullsized":"2023-10-09T04:19:54.939692+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":324,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.980535+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:19:59.980535+0000","last_peered":"2023-10-09T04:19:59.980535+0000","last_clean":"2023-10-09T04:19:59.980535+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:19:59.980535+0000","last_undegraded":"2023-10-09T04:19:59.980535+0000","last_fullsized":"2023-10-09T04:19:59.980535+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.016559+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:20:04.016559+0000","last_peered":"2023-10-09T04:20:04.016559+0000","last_clean":"2023-10-09T04:20:04.016559+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:20:04.016559+0000","last_undegraded":"2023-10-09T04:20:04.016559+0000","last_fullsized":"2023-10-09T04:20:04.016559+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":476,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.997240+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:20:02.997240+0000","last_peered":"2023-10-09T04:20:02.997240+0000","last_clean":"2023-10-09T04:20:02.997240+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:20:02.997240+0000","last_undegraded":"2023-10-09T04:20:02.997240+0000","last_fullsized":"2023-10-09T04:20:02.997240+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.928517+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:19:54.928517+0000","last_peered":"2023-10-09T04:19:54.928517+0000","last_clean":"2023-10-09T04:19:54.928517+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:19:54.928517+0000","last_undegraded":"2023-10-09T04:19:54.928517+0000","last_fullsized":"2023-10-09T04:19:54.928517+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":41,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.938415+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:19:54.938415+0000","last_peered":"2023-10-09T04:19:54.938415+0000","last_clean":"2023-10-09T04:19:54.938415+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:19:54.938415+0000","last_undegraded":"2023-10-09T04:19:54.938415+0000","last_fullsized":"2023-10-09T04:19:54.938415+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":481,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929609+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:19:54.929609+0000","last_peered":"2023-10-09T04:19:54.929609+0000","last_clean":"2023-10-09T04:19:54.929609+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:19:54.929609+0000","last_undegraded":"2023-10-09T04:19:54.929609+0000","last_fullsized":"2023-10-09T04:19:54.929609+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":389,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.940000+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:19:54.940000+0000","last_peered":"2023-10-09T04:19:54.940000+0000","last_clean":"2023-10-09T04:19:54.940000+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:19:54.940000+0000","last_undegraded":"2023-10-09T04:19:54.940000+0000","last_fullsized":"2023-10-09T04:19:54.940000+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":28,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.971357+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:20:01.971357+0000","last_peered":"2023-10-09T04:20:01.971357+0000","last_clean":"2023-10-09T04:20:01.971357+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:20:01.971357+0000","last_undegraded":"2023-10-09T04:20:01.971357+0000","last_fullsized":"2023-10-09T04:20:01.971357+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":491,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.100163+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:20:02.100163+0000","last_peered":"2023-10-09T04:20:02.100163+0000","last_clean":"2023-10-09T04:20:02.100163+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:20:02.100163+0000","last_undegraded":"2023-10-09T04:20:02.100163+0000","last_fullsized":"2023-10-09T04:20:02.100163+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":126,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.016895+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:20:04.016895+0000","last_peered":"2023-10-09T04:20:04.016895+0000","last_clean":"2023-10-09T04:20:04.016895+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:20:04.016895+0000","last_undegraded":"2023-10-09T04:20:04.016895+0000","last_fullsized":"2023-10-09T04:20:04.016895+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.015518+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:20:04.015518+0000","last_peered":"2023-10-09T04:20:04.015518+0000","last_clean":"2023-10-09T04:20:04.015518+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:20:04.015518+0000","last_undegraded":"2023-10-09T04:20:04.015518+0000","last_fullsized":"2023-10-09T04:20:04.015518+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":497,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.978909+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:19:59.978909+0000","last_peered":"2023-10-09T04:19:59.978909+0000","last_clean":"2023-10-09T04:19:59.978909+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:19:59.978909+0000","last_undegraded":"2023-10-09T04:19:59.978909+0000","last_fullsized":"2023-10-09T04:19:59.978909+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":340,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939845+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:19:54.939845+0000","last_peered":"2023-10-09T04:19:54.939845+0000","last_clean":"2023-10-09T04:19:54.939845+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:19:54.939845+0000","last_undegraded":"2023-10-09T04:19:54.939845+0000","last_fullsized":"2023-10-09T04:19:54.939845+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":28,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.099585+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:20:02.099585+0000","last_peered":"2023-10-09T04:20:02.099585+0000","last_clean":"2023-10-09T04:20:02.099585+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:20:02.099585+0000","last_undegraded":"2023-10-09T04:20:02.099585+0000","last_fullsized":"2023-10-09T04:20:02.099585+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":491,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.935357+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:19:54.935357+0000","last_peered":"2023-10-09T04:19:54.935357+0000","last_clean":"2023-10-09T04:19:54.935357+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:19:54.935357+0000","last_undegraded":"2023-10-09T04:19:54.935357+0000","last_fullsized":"2023-10-09T04:19:54.935357+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":324,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979021+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:19:59.979021+0000","last_peered":"2023-10-09T04:19:59.979021+0000","last_clean":"2023-10-09T04:19:59.979021+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:19:59.979021+0000","last_undegraded":"2023-10-09T04:19:59.979021+0000","last_fullsized":"2023-10-09T04:19:59.979021+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":325,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929570+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:19:54.929570+0000","last_peered":"2023-10-09T04:19:54.929570+0000","last_clean":"2023-10-09T04:19:54.929570+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:19:54.929570+0000","last_undegraded":"2023-10-09T04:19:54.929570+0000","last_fullsized":"2023-10-09T04:19:54.929570+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":482,"reported_epoch":350,"state":"active+undersized+remapped","last_fresh":"2023-10-09T04:20:03.005487+0000","last_change":"2023-10-09T04:20:03.005427+0000","last_active":"2023-10-09T04:20:03.005487+0000","last_peered":"2023-10-09T04:20:03.005487+0000","last_clean":"2023-10-09T04:19:59.977192+0000","last_became_active":"2023-10-09T04:20:03.005427+0000","last_became_peered":"2023-10-09T04:20:03.005427+0000","last_unstale":"2023-10-09T04:20:03.005487+0000","last_undegraded":"2023-10-09T04:20:03.005487+0000","last_fullsized":"2023-10-09T04:20:02.982500+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:17.519518+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,4],"avail_no_missing":["1","0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.017916+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:20:04.017916+0000","last_peered":"2023-10-09T04:20:04.017916+0000","last_clean":"2023-10-09T04:20:04.017916+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:20:04.017916+0000","last_undegraded":"2023-10-09T04:20:04.017916+0000","last_fullsized":"2023-10-09T04:20:04.017916+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":324,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979610+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:19:59.979610+0000","last_peered":"2023-10-09T04:19:59.979610+0000","last_clean":"2023-10-09T04:19:59.979610+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:19:59.979610+0000","last_undegraded":"2023-10-09T04:19:59.979610+0000","last_fullsized":"2023-10-09T04:19:59.979610+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":491,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.935118+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:19:54.935118+0000","last_peered":"2023-10-09T04:19:54.935118+0000","last_clean":"2023-10-09T04:19:54.935118+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:19:54.935118+0000","last_undegraded":"2023-10-09T04:19:54.935118+0000","last_fullsized":"2023-10-09T04:19:54.935118+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":337,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.978951+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:19:59.978951+0000","last_peered":"2023-10-09T04:19:59.978951+0000","last_clean":"2023-10-09T04:19:59.978951+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:19:59.978951+0000","last_undegraded":"2023-10-09T04:19:59.978951+0000","last_fullsized":"2023-10-09T04:19:59.978951+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.040338+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:20:05.040338+0000","last_peered":"2023-10-09T04:20:05.040338+0000","last_clean":"2023-10-09T04:20:05.040338+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:20:05.040338+0000","last_undegraded":"2023-10-09T04:20:05.040338+0000","last_fullsized":"2023-10-09T04:20:05.040338+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"348'6849","reported_seq":6900,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.082518+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:20:01.082518+0000","last_peered":"2023-10-09T04:20:01.082518+0000","last_clean":"2023-10-09T04:20:01.082518+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:20:01.082518+0000","last_undegraded":"2023-10-09T04:20:01.082518+0000","last_fullsized":"2023-10-09T04:20:01.082518+0000","mapping_epoch":306,"log_start":"314'800","ondisk_log_start":"314'800","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349372416,"num_objects":5331,"num_object_clones":0,"num_object_copies":15993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6849,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":749,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939875+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:19:54.939875+0000","last_peered":"2023-10-09T04:19:54.939875+0000","last_clean":"2023-10-09T04:19:54.939875+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:19:54.939875+0000","last_undegraded":"2023-10-09T04:19:54.939875+0000","last_fullsized":"2023-10-09T04:19:54.939875+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":335,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939098+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:19:54.939098+0000","last_peered":"2023-10-09T04:19:54.939098+0000","last_clean":"2023-10-09T04:19:54.939098+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:19:54.939098+0000","last_undegraded":"2023-10-09T04:19:54.939098+0000","last_fullsized":"2023-10-09T04:19:54.939098+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":298,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.935507+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:19:54.935507+0000","last_peered":"2023-10-09T04:19:54.935507+0000","last_clean":"2023-10-09T04:19:54.935507+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:19:54.935507+0000","last_undegraded":"2023-10-09T04:19:54.935507+0000","last_fullsized":"2023-10-09T04:19:54.935507+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"350'6964","reported_seq":7026,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:03.202262+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:20:03.202262+0000","last_peered":"2023-10-09T04:20:03.202262+0000","last_clean":"2023-10-09T04:20:03.202262+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:20:03.202262+0000","last_undegraded":"2023-10-09T04:20:03.202262+0000","last_fullsized":"2023-10-09T04:20:03.202262+0000","mapping_epoch":310,"log_start":"318'1873","ondisk_log_start":"318'1873","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355991552,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6964,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":518,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.017131+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:20:04.017131+0000","last_peered":"2023-10-09T04:20:04.017131+0000","last_clean":"2023-10-09T04:20:04.017131+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:20:04.017131+0000","last_undegraded":"2023-10-09T04:20:04.017131+0000","last_fullsized":"2023-10-09T04:20:04.017131+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":339,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.938426+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:19:54.938426+0000","last_peered":"2023-10-09T04:19:54.938426+0000","last_clean":"2023-10-09T04:19:54.938426+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:19:54.938426+0000","last_undegraded":"2023-10-09T04:19:54.938426+0000","last_fullsized":"2023-10-09T04:19:54.938426+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":126,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.017848+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:20:04.017848+0000","last_peered":"2023-10-09T04:20:04.017848+0000","last_clean":"2023-10-09T04:20:04.017848+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:20:04.017848+0000","last_undegraded":"2023-10-09T04:20:04.017848+0000","last_fullsized":"2023-10-09T04:20:04.017848+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"347'7063","reported_seq":7112,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.792856+0000","last_change":"2023-10-09T04:18:07.758447+0000","last_active":"2023-10-09T04:20:00.792856+0000","last_peered":"2023-10-09T04:20:00.792856+0000","last_clean":"2023-10-09T04:20:00.792856+0000","last_became_active":"2023-10-09T04:18:07.758159+0000","last_became_peered":"2023-10-09T04:18:07.758159+0000","last_unstale":"2023-10-09T04:20:00.792856+0000","last_undegraded":"2023-10-09T04:20:00.792856+0000","last_fullsized":"2023-10-09T04:20:00.792856+0000","mapping_epoch":306,"log_start":"314'1218","ondisk_log_start":"314'1218","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5845,"log_dups_size":0,"ondisk_log_size":5845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:29.487444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7063,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":481,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929516+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:19:54.929516+0000","last_peered":"2023-10-09T04:19:54.929516+0000","last_clean":"2023-10-09T04:19:54.929516+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:19:54.929516+0000","last_undegraded":"2023-10-09T04:19:54.929516+0000","last_fullsized":"2023-10-09T04:19:54.929516+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":304,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939906+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:19:54.939906+0000","last_peered":"2023-10-09T04:19:54.939906+0000","last_clean":"2023-10-09T04:19:54.939906+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:19:54.939906+0000","last_undegraded":"2023-10-09T04:19:54.939906+0000","last_fullsized":"2023-10-09T04:19:54.939906+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":282,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.984621+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:20:02.984621+0000","last_peered":"2023-10-09T04:20:02.984621+0000","last_clean":"2023-10-09T04:20:02.984621+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:20:02.984621+0000","last_undegraded":"2023-10-09T04:20:02.984621+0000","last_fullsized":"2023-10-09T04:20:02.984621+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"347'6932","reported_seq":6981,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.843620+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:20:00.843620+0000","last_peered":"2023-10-09T04:20:00.843620+0000","last_clean":"2023-10-09T04:20:00.843620+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:20:00.843620+0000","last_undegraded":"2023-10-09T04:20:00.843620+0000","last_fullsized":"2023-10-09T04:20:00.843620+0000","mapping_epoch":306,"log_start":"314'1087","ondisk_log_start":"314'1087","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5845,"log_dups_size":0,"ondisk_log_size":5845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353370112,"num_objects":5392,"num_object_clones":0,"num_object_copies":16176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6932,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":496,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013708+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:20:04.013708+0000","last_peered":"2023-10-09T04:20:04.013708+0000","last_clean":"2023-10-09T04:20:04.013708+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:20:04.013708+0000","last_undegraded":"2023-10-09T04:20:04.013708+0000","last_fullsized":"2023-10-09T04:20:04.013708+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":336,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929084+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:19:54.929084+0000","last_peered":"2023-10-09T04:19:54.929084+0000","last_clean":"2023-10-09T04:19:54.929084+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:19:54.929084+0000","last_undegraded":"2023-10-09T04:19:54.929084+0000","last_fullsized":"2023-10-09T04:19:54.929084+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":325,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.980845+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:20:00.980845+0000","last_peered":"2023-10-09T04:20:00.980845+0000","last_clean":"2023-10-09T04:20:00.980845+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:20:00.980845+0000","last_undegraded":"2023-10-09T04:20:00.980845+0000","last_fullsized":"2023-10-09T04:20:00.980845+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"347'6635","reported_seq":10057,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.894543+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:20:00.894543+0000","last_peered":"2023-10-09T04:20:00.894543+0000","last_clean":"2023-10-09T04:20:00.894543+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:20:00.894543+0000","last_undegraded":"2023-10-09T04:20:00.894543+0000","last_fullsized":"2023-10-09T04:20:00.894543+0000","mapping_epoch":320,"log_start":"314'1300","ondisk_log_start":"314'1300","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343605248,"num_objects":5243,"num_object_clones":0,"num_object_copies":15729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6635,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":770,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.938244+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:19:54.938244+0000","last_peered":"2023-10-09T04:19:54.938244+0000","last_clean":"2023-10-09T04:19:54.938244+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:19:54.938244+0000","last_undegraded":"2023-10-09T04:19:54.938244+0000","last_fullsized":"2023-10-09T04:19:54.938244+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":307,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979975+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:19:59.979975+0000","last_peered":"2023-10-09T04:19:59.979975+0000","last_clean":"2023-10-09T04:19:59.979975+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:19:59.979975+0000","last_undegraded":"2023-10-09T04:19:59.979975+0000","last_fullsized":"2023-10-09T04:19:59.979975+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":126,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.016978+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:20:04.016978+0000","last_peered":"2023-10-09T04:20:04.016978+0000","last_clean":"2023-10-09T04:20:04.016978+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:20:04.016978+0000","last_undegraded":"2023-10-09T04:20:04.016978+0000","last_fullsized":"2023-10-09T04:20:04.016978+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"350'6899","reported_seq":10463,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:03.193145+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:20:03.193145+0000","last_peered":"2023-10-09T04:20:03.193145+0000","last_clean":"2023-10-09T04:20:03.193145+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:20:03.193145+0000","last_undegraded":"2023-10-09T04:20:03.193145+0000","last_fullsized":"2023-10-09T04:20:03.193145+0000","mapping_epoch":329,"log_start":"317'1760","ondisk_log_start":"317'1760","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5139,"log_dups_size":0,"ondisk_log_size":5139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352387072,"num_objects":5377,"num_object_clones":0,"num_object_copies":16131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6899,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":516,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.935225+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:19:54.935225+0000","last_peered":"2023-10-09T04:19:54.935225+0000","last_clean":"2023-10-09T04:19:54.935225+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:19:54.935225+0000","last_undegraded":"2023-10-09T04:19:54.935225+0000","last_fullsized":"2023-10-09T04:19:54.935225+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013828+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:20:04.013828+0000","last_peered":"2023-10-09T04:20:04.013828+0000","last_clean":"2023-10-09T04:20:04.013828+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:20:04.013828+0000","last_undegraded":"2023-10-09T04:20:04.013828+0000","last_fullsized":"2023-10-09T04:20:04.013828+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":96,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.015271+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:20:04.015271+0000","last_peered":"2023-10-09T04:20:04.015271+0000","last_clean":"2023-10-09T04:20:04.015271+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:20:04.015271+0000","last_undegraded":"2023-10-09T04:20:04.015271+0000","last_fullsized":"2023-10-09T04:20:04.015271+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"347'6836","reported_seq":6895,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.841997+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:20:00.841997+0000","last_peered":"2023-10-09T04:20:00.841997+0000","last_clean":"2023-10-09T04:20:00.841997+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:20:00.841997+0000","last_undegraded":"2023-10-09T04:20:00.841997+0000","last_fullsized":"2023-10-09T04:20:00.841997+0000","mapping_epoch":310,"log_start":"314'1500","ondisk_log_start":"314'1500","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5336,"log_dups_size":0,"ondisk_log_size":5336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347996160,"num_objects":5310,"num_object_clones":0,"num_object_copies":15930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6836,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":474,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.971594+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:20:01.971594+0000","last_peered":"2023-10-09T04:20:01.971594+0000","last_clean":"2023-10-09T04:20:01.971594+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:20:01.971594+0000","last_undegraded":"2023-10-09T04:20:01.971594+0000","last_fullsized":"2023-10-09T04:20:01.971594+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":96,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.017667+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:20:04.017667+0000","last_peered":"2023-10-09T04:20:04.017667+0000","last_clean":"2023-10-09T04:20:04.017667+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:20:04.017667+0000","last_undegraded":"2023-10-09T04:20:04.017667+0000","last_fullsized":"2023-10-09T04:20:04.017667+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":146,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013893+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:20:04.013893+0000","last_peered":"2023-10-09T04:20:04.013893+0000","last_clean":"2023-10-09T04:20:04.013893+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:20:04.013893+0000","last_undegraded":"2023-10-09T04:20:04.013893+0000","last_fullsized":"2023-10-09T04:20:04.013893+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"350'6954","reported_seq":7015,"reported_epoch":350,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:03.198415+0000","last_change":"2023-10-09T04:20:03.088309+0000","last_active":"2023-10-09T04:20:03.198415+0000","last_peered":"2023-10-09T04:20:03.198415+0000","last_clean":"2023-10-09T04:20:00.980108+0000","last_became_active":"2023-10-09T04:20:03.079675+0000","last_became_peered":"2023-10-09T04:20:03.079675+0000","last_unstale":"2023-10-09T04:20:03.198415+0000","last_undegraded":"2023-10-09T04:20:03.198415+0000","last_fullsized":"2023-10-09T04:20:03.198415+0000","mapping_epoch":349,"log_start":"317'1781","ondisk_log_start":"317'1781","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5173,"log_dups_size":0,"ondisk_log_size":5173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:58.283644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358350848,"num_objects":5468,"num_object_clones":0,"num_object_copies":16404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5468,"num_objects_unfound":0,"num_objects_dirty":5468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6954,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5468}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":491,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.934927+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:19:54.934927+0000","last_peered":"2023-10-09T04:19:54.934927+0000","last_clean":"2023-10-09T04:19:54.934927+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:19:54.934927+0000","last_undegraded":"2023-10-09T04:19:54.934927+0000","last_fullsized":"2023-10-09T04:19:54.934927+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.037913+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:20:05.037913+0000","last_peered":"2023-10-09T04:20:05.037913+0000","last_clean":"2023-10-09T04:20:05.037913+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:20:05.037913+0000","last_undegraded":"2023-10-09T04:20:05.037913+0000","last_fullsized":"2023-10-09T04:20:05.037913+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.998255+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:20:02.998255+0000","last_peered":"2023-10-09T04:20:02.998255+0000","last_clean":"2023-10-09T04:20:02.998255+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:20:02.998255+0000","last_undegraded":"2023-10-09T04:20:02.998255+0000","last_fullsized":"2023-10-09T04:20:02.998255+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"347'6966","reported_seq":7025,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.817947+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:20:00.817947+0000","last_peered":"2023-10-09T04:20:00.817947+0000","last_clean":"2023-10-09T04:20:00.817947+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:20:00.817947+0000","last_undegraded":"2023-10-09T04:20:00.817947+0000","last_fullsized":"2023-10-09T04:20:00.817947+0000","mapping_epoch":310,"log_start":"314'1134","ondisk_log_start":"314'1134","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5832,"log_dups_size":0,"ondisk_log_size":5832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356777984,"num_objects":5444,"num_object_clones":0,"num_object_copies":16332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6966,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":491,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929865+0000","last_change":"2023-10-09T04:18:31.849253+0000","last_active":"2023-10-09T04:19:54.929865+0000","last_peered":"2023-10-09T04:19:54.929865+0000","last_clean":"2023-10-09T04:19:54.929865+0000","last_became_active":"2023-10-09T04:18:31.848599+0000","last_became_peered":"2023-10-09T04:18:31.848599+0000","last_unstale":"2023-10-09T04:19:54.929865+0000","last_undegraded":"2023-10-09T04:19:54.929865+0000","last_fullsized":"2023-10-09T04:19:54.929865+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:47.206426+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":329,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.100244+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:20:02.100244+0000","last_peered":"2023-10-09T04:20:02.100244+0000","last_clean":"2023-10-09T04:20:02.100244+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:20:02.100244+0000","last_undegraded":"2023-10-09T04:20:02.100244+0000","last_fullsized":"2023-10-09T04:20:02.100244+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":343,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929426+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:19:54.929426+0000","last_peered":"2023-10-09T04:19:54.929426+0000","last_clean":"2023-10-09T04:19:54.929426+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:19:54.929426+0000","last_undegraded":"2023-10-09T04:19:54.929426+0000","last_fullsized":"2023-10-09T04:19:54.929426+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.999753+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:20:02.999753+0000","last_peered":"2023-10-09T04:20:02.999753+0000","last_clean":"2023-10-09T04:20:02.999753+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:20:02.999753+0000","last_undegraded":"2023-10-09T04:20:02.999753+0000","last_fullsized":"2023-10-09T04:20:02.999753+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"348'6900","reported_seq":6951,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.061196+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:20:01.061196+0000","last_peered":"2023-10-09T04:20:01.061196+0000","last_clean":"2023-10-09T04:20:01.061196+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:20:01.061196+0000","last_undegraded":"2023-10-09T04:20:01.061196+0000","last_fullsized":"2023-10-09T04:20:01.061196+0000","mapping_epoch":306,"log_start":"314'800","ondisk_log_start":"314'800","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6100,"log_dups_size":0,"ondisk_log_size":6100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354025472,"num_objects":5402,"num_object_clones":0,"num_object_copies":16206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6900,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":464,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.938718+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:19:54.938718+0000","last_peered":"2023-10-09T04:19:54.938718+0000","last_clean":"2023-10-09T04:19:54.938718+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:19:54.938718+0000","last_undegraded":"2023-10-09T04:19:54.938718+0000","last_fullsized":"2023-10-09T04:19:54.938718+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":320,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.934998+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:19:54.934998+0000","last_peered":"2023-10-09T04:19:54.934998+0000","last_clean":"2023-10-09T04:19:54.934998+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:19:54.934998+0000","last_undegraded":"2023-10-09T04:19:54.934998+0000","last_fullsized":"2023-10-09T04:19:54.934998+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.015435+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:20:04.015435+0000","last_peered":"2023-10-09T04:20:04.015435+0000","last_clean":"2023-10-09T04:20:04.015435+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:20:04.015435+0000","last_undegraded":"2023-10-09T04:20:04.015435+0000","last_fullsized":"2023-10-09T04:20:04.015435+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":357,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.100175+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:20:02.100175+0000","last_peered":"2023-10-09T04:20:02.100175+0000","last_clean":"2023-10-09T04:20:02.100175+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:20:02.100175+0000","last_undegraded":"2023-10-09T04:20:02.100175+0000","last_fullsized":"2023-10-09T04:20:02.100175+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"347'6981","reported_seq":7030,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.786910+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:20:00.786910+0000","last_peered":"2023-10-09T04:20:00.786910+0000","last_clean":"2023-10-09T04:20:00.786910+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:20:00.786910+0000","last_undegraded":"2023-10-09T04:20:00.786910+0000","last_fullsized":"2023-10-09T04:20:00.786910+0000","mapping_epoch":306,"log_start":"314'1168","ondisk_log_start":"314'1168","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5813,"log_dups_size":0,"ondisk_log_size":5813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360382464,"num_objects":5499,"num_object_clones":0,"num_object_copies":16497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6981,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":480,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979099+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:19:59.979099+0000","last_peered":"2023-10-09T04:19:59.979099+0000","last_clean":"2023-10-09T04:19:59.979099+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:19:59.979099+0000","last_undegraded":"2023-10-09T04:19:59.979099+0000","last_fullsized":"2023-10-09T04:19:59.979099+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":186,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.996932+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:20:02.996932+0000","last_peered":"2023-10-09T04:20:02.996932+0000","last_clean":"2023-10-09T04:20:02.996932+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:20:02.996932+0000","last_undegraded":"2023-10-09T04:20:02.996932+0000","last_fullsized":"2023-10-09T04:20:02.996932+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":332,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.100707+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:20:02.100707+0000","last_peered":"2023-10-09T04:20:02.100707+0000","last_clean":"2023-10-09T04:20:02.100707+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:20:02.100707+0000","last_undegraded":"2023-10-09T04:20:02.100707+0000","last_fullsized":"2023-10-09T04:20:02.100707+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":28,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.099738+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:20:02.099738+0000","last_peered":"2023-10-09T04:20:02.099738+0000","last_clean":"2023-10-09T04:20:02.099738+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:20:02.099738+0000","last_undegraded":"2023-10-09T04:20:02.099738+0000","last_fullsized":"2023-10-09T04:20:02.099738+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"347'6965","reported_seq":10473,"reported_epoch":348,"state":"remapped+peering","last_fresh":"2023-10-09T04:20:00.983064+0000","last_change":"2023-10-09T04:20:00.983263+0000","last_active":"2023-10-09T04:20:00.983064+0000","last_peered":"2023-10-09T04:20:00.976968+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:19:43.715695+0000","last_became_peered":"2023-10-09T04:19:43.715695+0000","last_unstale":"2023-10-09T04:20:00.983064+0000","last_undegraded":"2023-10-09T04:20:00.983064+0000","last_fullsized":"2023-10-09T04:20:00.983064+0000","mapping_epoch":348,"log_start":"314'900","ondisk_log_start":"314'900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:11.246258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355401728,"num_objects":5423,"num_object_clones":0,"num_object_copies":16269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6965,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1722,"num_bytes_recovered":112852992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,5,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":330,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.928725+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:19:54.928725+0000","last_peered":"2023-10-09T04:19:54.928725+0000","last_clean":"2023-10-09T04:19:54.928725+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:19:54.928725+0000","last_undegraded":"2023-10-09T04:19:54.928725+0000","last_fullsized":"2023-10-09T04:19:54.928725+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":741,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.937745+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:19:54.937745+0000","last_peered":"2023-10-09T04:19:54.937745+0000","last_clean":"2023-10-09T04:19:54.937745+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:19:54.937745+0000","last_undegraded":"2023-10-09T04:19:54.937745+0000","last_fullsized":"2023-10-09T04:19:54.937745+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":273,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.935030+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:19:54.935030+0000","last_peered":"2023-10-09T04:19:54.935030+0000","last_clean":"2023-10-09T04:19:54.935030+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:19:54.935030+0000","last_undegraded":"2023-10-09T04:19:54.935030+0000","last_fullsized":"2023-10-09T04:19:54.935030+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.015351+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:20:04.015351+0000","last_peered":"2023-10-09T04:20:04.015351+0000","last_clean":"2023-10-09T04:20:04.015351+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:20:04.015351+0000","last_undegraded":"2023-10-09T04:20:04.015351+0000","last_fullsized":"2023-10-09T04:20:04.015351+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"348'6951","reported_seq":10581,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-10-09T04:20:01.057107+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:20:01.057107+0000","last_peered":"2023-10-09T04:20:01.057107+0000","last_clean":"2023-10-09T04:20:01.057107+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:20:01.057107+0000","last_undegraded":"2023-10-09T04:20:01.057107+0000","last_fullsized":"2023-10-09T04:20:01.057107+0000","mapping_epoch":323,"log_start":"314'900","ondisk_log_start":"314'900","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6051,"log_dups_size":0,"ondisk_log_size":6051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352780288,"num_objects":5383,"num_object_clones":0,"num_object_copies":16149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6951,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":492,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.099980+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:20:02.099980+0000","last_peered":"2023-10-09T04:20:02.099980+0000","last_clean":"2023-10-09T04:20:02.099980+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:20:02.099980+0000","last_undegraded":"2023-10-09T04:20:02.099980+0000","last_fullsized":"2023-10-09T04:20:02.099980+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":327,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.937672+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:19:54.937672+0000","last_peered":"2023-10-09T04:19:54.937672+0000","last_clean":"2023-10-09T04:19:54.937672+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:19:54.937672+0000","last_undegraded":"2023-10-09T04:19:54.937672+0000","last_fullsized":"2023-10-09T04:19:54.937672+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":335,"reported_epoch":348,"state":"peering","last_fresh":"2023-10-09T04:20:00.985516+0000","last_change":"2023-10-09T04:20:00.985639+0000","last_active":"2023-10-09T04:20:00.985516+0000","last_peered":"2023-10-09T04:19:59.979426+0000","last_clean":"2023-10-09T04:19:59.979426+0000","last_became_active":"2023-10-09T04:16:24.470433+0000","last_became_peered":"2023-10-09T04:16:24.470433+0000","last_unstale":"2023-10-09T04:20:00.985516+0000","last_undegraded":"2023-10-09T04:20:00.985516+0000","last_fullsized":"2023-10-09T04:20:00.985516+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:32.294020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,5],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":340,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979371+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:19:59.979371+0000","last_peered":"2023-10-09T04:19:59.979371+0000","last_clean":"2023-10-09T04:19:59.979371+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:19:59.979371+0000","last_undegraded":"2023-10-09T04:19:59.979371+0000","last_fullsized":"2023-10-09T04:19:59.979371+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"347'6957","reported_seq":7006,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:20:00.883735+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:20:00.883735+0000","last_peered":"2023-10-09T04:20:00.883735+0000","last_clean":"2023-10-09T04:20:00.883735+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:20:00.883735+0000","last_undegraded":"2023-10-09T04:20:00.883735+0000","last_fullsized":"2023-10-09T04:20:00.883735+0000","mapping_epoch":306,"log_start":"314'1600","ondisk_log_start":"314'1600","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361562112,"num_objects":5517,"num_object_clones":0,"num_object_copies":16551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6957,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":505,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013571+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:20:04.013571+0000","last_peered":"2023-10-09T04:20:04.013571+0000","last_clean":"2023-10-09T04:20:04.013571+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:20:04.013571+0000","last_undegraded":"2023-10-09T04:20:04.013571+0000","last_fullsized":"2023-10-09T04:20:04.013571+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":330,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939960+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:19:54.939960+0000","last_peered":"2023-10-09T04:19:54.939960+0000","last_clean":"2023-10-09T04:19:54.939960+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:19:54.939960+0000","last_undegraded":"2023-10-09T04:19:54.939960+0000","last_fullsized":"2023-10-09T04:19:54.939960+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":323,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.100322+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:20:02.100322+0000","last_peered":"2023-10-09T04:20:02.100322+0000","last_clean":"2023-10-09T04:20:02.100322+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:20:02.100322+0000","last_undegraded":"2023-10-09T04:20:02.100322+0000","last_fullsized":"2023-10-09T04:20:02.100322+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":46,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.928654+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:19:54.928654+0000","last_peered":"2023-10-09T04:19:54.928654+0000","last_clean":"2023-10-09T04:19:54.928654+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:19:54.928654+0000","last_undegraded":"2023-10-09T04:19:54.928654+0000","last_fullsized":"2023-10-09T04:19:54.928654+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"347'6927","reported_seq":14655,"reported_epoch":347,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:20:00.893474+0000","last_change":"2023-10-09T04:19:50.952445+0000","last_active":"2023-10-09T04:20:00.893474+0000","last_peered":"2023-10-09T04:20:00.893474+0000","last_clean":"2023-10-09T04:19:25.901043+0000","last_became_active":"2023-10-09T04:19:28.825416+0000","last_became_peered":"2023-10-09T04:19:28.825416+0000","last_unstale":"2023-10-09T04:20:00.893474+0000","last_undegraded":"2023-10-09T04:20:00.893474+0000","last_fullsized":"2023-10-09T04:20:00.893474+0000","mapping_epoch":333,"log_start":"314'1600","ondisk_log_start":"314'1600","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5327,"log_dups_size":0,"ondisk_log_size":5327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:15.224579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352845848,"num_objects":5385,"num_object_clones":0,"num_object_copies":16155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1861,"num_objects_unfound":0,"num_objects_dirty":5385,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6927,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3831,"num_bytes_recovered":251002904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,4],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":5385}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":496,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.015071+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:20:04.015071+0000","last_peered":"2023-10-09T04:20:04.015071+0000","last_clean":"2023-10-09T04:20:04.015071+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:20:04.015071+0000","last_undegraded":"2023-10-09T04:20:04.015071+0000","last_fullsized":"2023-10-09T04:20:04.015071+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":279,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.939342+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:19:54.939342+0000","last_peered":"2023-10-09T04:19:54.939342+0000","last_clean":"2023-10-09T04:19:54.939342+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:19:54.939342+0000","last_undegraded":"2023-10-09T04:19:54.939342+0000","last_fullsized":"2023-10-09T04:19:54.939342+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":317,"reported_epoch":349,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.100285+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:20:02.100285+0000","last_peered":"2023-10-09T04:20:02.100285+0000","last_clean":"2023-10-09T04:20:02.100285+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:20:02.100285+0000","last_undegraded":"2023-10-09T04:20:02.100285+0000","last_fullsized":"2023-10-09T04:20:02.100285+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":287,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.937951+0000","last_change":"2023-10-09T04:16:24.469186+0000","last_active":"2023-10-09T04:19:54.937951+0000","last_peered":"2023-10-09T04:19:54.937951+0000","last_clean":"2023-10-09T04:19:54.937951+0000","last_became_active":"2023-10-09T04:16:24.468836+0000","last_became_peered":"2023-10-09T04:16:24.468836+0000","last_unstale":"2023-10-09T04:19:54.937951+0000","last_undegraded":"2023-10-09T04:19:54.937951+0000","last_fullsized":"2023-10-09T04:19:54.937951+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:42.902638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"352'7159","reported_seq":7223,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.432470+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:20:05.432470+0000","last_peered":"2023-10-09T04:20:05.432470+0000","last_clean":"2023-10-09T04:20:05.432470+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:20:05.432470+0000","last_undegraded":"2023-10-09T04:20:05.432470+0000","last_fullsized":"2023-10-09T04:20:05.432470+0000","mapping_epoch":310,"log_start":"314'1600","ondisk_log_start":"314'1600","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353304576,"num_objects":5391,"num_object_clones":0,"num_object_copies":16173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7159,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":489,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.017035+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:20:04.017035+0000","last_peered":"2023-10-09T04:20:04.017035+0000","last_clean":"2023-10-09T04:20:04.017035+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:20:04.017035+0000","last_undegraded":"2023-10-09T04:20:04.017035+0000","last_fullsized":"2023-10-09T04:20:04.017035+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":182,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.998254+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:20:02.998254+0000","last_peered":"2023-10-09T04:20:02.998254+0000","last_clean":"2023-10-09T04:20:02.998254+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:20:02.998254+0000","last_undegraded":"2023-10-09T04:20:02.998254+0000","last_fullsized":"2023-10-09T04:20:02.998254+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":353,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.979460+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:19:59.979460+0000","last_peered":"2023-10-09T04:19:59.979460+0000","last_clean":"2023-10-09T04:19:59.979460+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:19:59.979460+0000","last_undegraded":"2023-10-09T04:19:59.979460+0000","last_fullsized":"2023-10-09T04:19:59.979460+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.017762+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:20:04.017762+0000","last_peered":"2023-10-09T04:20:04.017762+0000","last_clean":"2023-10-09T04:20:04.017762+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:20:04.017762+0000","last_undegraded":"2023-10-09T04:20:04.017762+0000","last_fullsized":"2023-10-09T04:20:04.017762+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":506,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.978825+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:19:59.978825+0000","last_peered":"2023-10-09T04:19:59.978825+0000","last_clean":"2023-10-09T04:19:59.978825+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:19:59.978825+0000","last_undegraded":"2023-10-09T04:19:59.978825+0000","last_fullsized":"2023-10-09T04:19:59.978825+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":322,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.980062+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:19:59.980062+0000","last_peered":"2023-10-09T04:19:59.980062+0000","last_clean":"2023-10-09T04:19:59.980062+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:19:59.980062+0000","last_undegraded":"2023-10-09T04:19:59.980062+0000","last_fullsized":"2023-10-09T04:19:59.980062+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.997487+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:20:02.997487+0000","last_peered":"2023-10-09T04:20:02.997487+0000","last_clean":"2023-10-09T04:20:02.997487+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:20:02.997487+0000","last_undegraded":"2023-10-09T04:20:02.997487+0000","last_fullsized":"2023-10-09T04:20:02.997487+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":734,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.938343+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:19:54.938343+0000","last_peered":"2023-10-09T04:19:54.938343+0000","last_clean":"2023-10-09T04:19:54.938343+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:19:54.938343+0000","last_undegraded":"2023-10-09T04:19:54.938343+0000","last_fullsized":"2023-10-09T04:19:54.938343+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":336,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929170+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:19:54.929170+0000","last_peered":"2023-10-09T04:19:54.929170+0000","last_clean":"2023-10-09T04:19:54.929170+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:19:54.929170+0000","last_undegraded":"2023-10-09T04:19:54.929170+0000","last_fullsized":"2023-10-09T04:19:54.929170+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":116,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.015592+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:20:04.015592+0000","last_peered":"2023-10-09T04:20:04.015592+0000","last_clean":"2023-10-09T04:20:04.015592+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:20:04.015592+0000","last_undegraded":"2023-10-09T04:20:04.015592+0000","last_fullsized":"2023-10-09T04:20:04.015592+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":333,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.929352+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:19:54.929352+0000","last_peered":"2023-10-09T04:19:54.929352+0000","last_clean":"2023-10-09T04:19:54.929352+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:19:54.929352+0000","last_undegraded":"2023-10-09T04:19:54.929352+0000","last_fullsized":"2023-10-09T04:19:54.929352+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013962+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:20:04.013962+0000","last_peered":"2023-10-09T04:20:04.013962+0000","last_clean":"2023-10-09T04:20:04.013962+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:20:04.013962+0000","last_undegraded":"2023-10-09T04:20:04.013962+0000","last_fullsized":"2023-10-09T04:20:04.013962+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":769,"reported_epoch":346,"state":"active+clean","last_fresh":"2023-10-09T04:19:54.940568+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:19:54.940568+0000","last_peered":"2023-10-09T04:19:54.940568+0000","last_clean":"2023-10-09T04:19:54.940568+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:19:54.940568+0000","last_undegraded":"2023-10-09T04:19:54.940568+0000","last_fullsized":"2023-10-09T04:19:54.940568+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":195,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.997893+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:20:02.997893+0000","last_peered":"2023-10-09T04:20:02.997893+0000","last_clean":"2023-10-09T04:20:02.997893+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:20:02.997893+0000","last_undegraded":"2023-10-09T04:20:02.997893+0000","last_fullsized":"2023-10-09T04:20:02.997893+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":185,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T04:20:02.996962+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:20:02.996962+0000","last_peered":"2023-10-09T04:20:02.996962+0000","last_clean":"2023-10-09T04:20:02.996962+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:20:02.996962+0000","last_undegraded":"2023-10-09T04:20:02.996962+0000","last_fullsized":"2023-10-09T04:20:02.996962+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":482,"reported_epoch":350,"state":"active+undersized+remapped","last_fresh":"2023-10-09T04:20:03.004418+0000","last_change":"2023-10-09T04:20:03.004295+0000","last_active":"2023-10-09T04:20:03.004418+0000","last_peered":"2023-10-09T04:20:03.004418+0000","last_clean":"2023-10-09T04:19:59.977582+0000","last_became_active":"2023-10-09T04:20:03.004295+0000","last_became_peered":"2023-10-09T04:20:03.004295+0000","last_unstale":"2023-10-09T04:20:03.004418+0000","last_undegraded":"2023-10-09T04:20:03.004418+0000","last_fullsized":"2023-10-09T04:20:02.984062+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:07:49.431588+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,4],"avail_no_missing":["1","2","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":341,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-10-09T04:19:59.980611+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:19:59.980611+0000","last_peered":"2023-10-09T04:19:59.980611+0000","last_clean":"2023-10-09T04:19:59.980611+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:19:59.980611+0000","last_undegraded":"2023-10-09T04:19:59.980611+0000","last_fullsized":"2023-10-09T04:19:59.980611+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":106,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013493+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:20:04.013493+0000","last_peered":"2023-10-09T04:20:04.013493+0000","last_clean":"2023-10-09T04:20:04.013493+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:20:04.013493+0000","last_undegraded":"2023-10-09T04:20:04.013493+0000","last_fullsized":"2023-10-09T04:20:04.013493+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5669453848,"num_objects":86510,"num_object_clones":0,"num_object_copies":259530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7329,"num_objects_unfound":0,"num_objects_dirty":86510,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110938,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10749,"num_bytes_recovered":704380952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17385209856,"data_stored":17385193568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89917,"ondisk_log_size":89917,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201610,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4436616,"kb_used_data":3838592,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":89308536,"statfs":{"total":95995035648,"available":91451940864,"internally_reserved":0,"allocated":3930718208,"data_stored":3929750176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.4089999999999998,"15min":2.9249999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":16.440999999999999,"5min":16.440999999999999,"15min":126.752},"last":0.45200000000000001},{"interface":"front","average":{"1min":3.1869999999999998,"5min":2.2189999999999999,"15min":2.8820000000000001},"min":{"1min":0.27300000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":16.462,"5min":16.462,"15min":128.88399999999999},"last":3.5449999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:19:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":3.8620000000000001,"15min":3.9609999999999999},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":14.738,"5min":103.324,"15min":245.28700000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.948,"5min":3.8599999999999999,"15min":3.8679999999999999},"min":{"1min":0.308,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.159000000000001,"5min":104.248,"15min":244.626},"last":3.492}]},{"osd":2,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.7849999999999999,"15min":2.0979999999999999},"min":{"1min":0.30499999999999999,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.194,"5min":14.076000000000001,"15min":76.774000000000001},"last":2.9470000000000001},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.556,"15min":2.794},"min":{"1min":0.35799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":6.1749999999999998,"5min":15.597,"15min":99.372},"last":3.4630000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:19:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":2.601,"15min":2.5710000000000002},"min":{"1min":0.312,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":5.2300000000000004,"5min":85.575000000000003,"15min":99.134},"last":2.972},{"interface":"front","average":{"1min":1.6399999999999999,"5min":2.883,"15min":2.7559999999999998},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.4210000000000003,"5min":85.528000000000006,"15min":99.117999999999995},"last":3.5720000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.86899999999999999,"15min":0.82799999999999996},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.8529999999999998,"5min":8.9359999999999999,"15min":16.195},"last":2.9140000000000001},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6160000000000001,"15min":1.1120000000000001},"min":{"1min":0.27700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"last":3.5190000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.5680000000000001,"15min":1.321},"min":{"1min":0.29299999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":6.9390000000000001,"5min":120.066,"15min":120.066},"last":2.3039999999999998},{"interface":"front","average":{"1min":0.85299999999999998,"5min":2.2530000000000001,"15min":1.2789999999999999},"min":{"1min":0.34399999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8899999999999997,"5min":120.09699999999999,"15min":120.09699999999999},"last":3.3929999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.4590000000000001,"15min":1.0249999999999999},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":8.4550000000000001,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":2.6880000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.96999999999999997,"15min":0.88300000000000001},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.605,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":3.3559999999999999}]}]},{"osd":6,"up_from":0,"seq":201863463248,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089556,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3986300,"kb_used_data":3416052,"kb_used_omap":9,"kb_used_meta":570166,"kb_avail":89758852,"statfs":{"total":95995035648,"available":91913064448,"internally_reserved":0,"allocated":3498037248,"data_stored":3496575807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":583850985},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":4.7389999999999999,"15min":3.923},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.17199999999999999},"max":{"1min":6.5739999999999998,"5min":98.257999999999996,"15min":98.257999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.7689999999999999,"5min":7.306,"15min":5.077},"min":{"1min":0.215,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":8.7080000000000002,"5min":154.654,"15min":154.654},"last":0.89700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":2.1970000000000001,"15min":2.6019999999999999},"min":{"1min":0.26600000000000001,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":19.564,"5min":19.564,"15min":121.54600000000001},"last":1.994},{"interface":"front","average":{"1min":2,"5min":2.476,"15min":2.9239999999999999},"min":{"1min":0.22500000000000001,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":12.382999999999999,"5min":16.114999999999998,"15min":99.899000000000001},"last":1.0009999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.968,"15min":3.669},"min":{"1min":0.29899999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.548,"5min":68.054000000000002,"15min":138.87299999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.0569999999999999,"5min":4.0460000000000003,"15min":4.5579999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.6760000000000002,"5min":68.563999999999993,"15min":138.49799999999999},"last":0.94099999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.206,"15min":2.2189999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":6.3879999999999999,"5min":16.27,"15min":99.989999999999995},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.661,"5min":2.0270000000000001,"15min":2.1259999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":6.3090000000000002,"5min":13.428000000000001,"15min":100.249},"last":0.55200000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":2.9550000000000001,"15min":2.4340000000000002},"min":{"1min":0.34000000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.5419999999999998,"5min":38.848999999999997,"15min":99.933999999999997},"last":5.8099999999999996},{"interface":"front","average":{"1min":1.623,"5min":2.9340000000000002,"15min":2.5390000000000001},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.3479999999999999,"5min":39.593000000000004,"15min":100.34099999999999},"last":3.5680000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":2.3210000000000002,"15min":2.4220000000000002},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":6.4409999999999998,"5min":13.377000000000001,"15min":100.309},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.004,"5min":1.8140000000000001,"15min":2.0659999999999998},"min":{"1min":0.44800000000000001,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":2.6720000000000002,"5min":14.141999999999999,"15min":100.143},"last":0.71599999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":2.6589999999999998,"15min":2.5150000000000001},"min":{"1min":0.38200000000000001,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":6.375,"5min":17.460000000000001,"15min":100.214},"last":2.2749999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":2.2000000000000002,"15min":2.2040000000000002},"min":{"1min":0.498,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":2.6579999999999999,"5min":14.167,"15min":100.113},"last":2.2400000000000002}]}]},{"osd":0,"up_from":84,"seq":360777253082,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2249192,"kb_used_data":1727264,"kb_used_omap":28,"kb_used_meta":521891,"kb_avail":91495960,"statfs":{"total":95995035648,"available":93691863040,"internally_reserved":0,"allocated":1768718336,"data_stored":1767200330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":534417232},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:19:02 2023","interfaces":[{"interface":"back","average":{"1min":14.959,"5min":8.9269999999999996,"15min":5.1890000000000001},"min":{"1min":0.19500000000000001,"5min":0.187,"15min":0.17000000000000001},"max":{"1min":209.215,"5min":209.215,"15min":209.215},"last":0.219},{"interface":"front","average":{"1min":4.391,"5min":6.8300000000000001,"15min":4.8570000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":63.552999999999997,"5min":120.934,"15min":120.934},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.3359999999999999,"15min":3.141},"min":{"1min":0.32900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.025,"5min":17.466999999999999,"15min":118.54000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.0609999999999999,"5min":3.2429999999999999,"15min":3.536},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.20300000000000001},"max":{"1min":6.7460000000000004,"5min":20.262,"15min":118.727},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:19:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":3.0830000000000002,"15min":5.1539999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":4.375,"5min":99.709000000000003,"15min":156.488},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.244,"5min":3.2719999999999998,"15min":5.2320000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":7.718,"5min":99.560000000000002,"15min":156.50700000000001},"last":0.29699999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":4.899,"5min":2.9969999999999999,"15min":2.7250000000000001},"min":{"1min":0.35399999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":49.840000000000003,"5min":63.375999999999998,"15min":118.633},"last":0.374},{"interface":"front","average":{"1min":5.0609999999999999,"5min":3.3290000000000002,"15min":3.0670000000000002},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":49.607999999999997,"5min":49.607999999999997,"15min":118.913},"last":2.3959999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":5.4800000000000004,"15min":4.2960000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.0570000000000004,"5min":156.96100000000001,"15min":182.68899999999999},"last":2.2610000000000001},{"interface":"front","average":{"1min":2.0870000000000002,"5min":4.5540000000000003,"15min":3.7749999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.024,"5min":156.97300000000001,"15min":182.66300000000001},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.528,"15min":2.996},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":7.0709999999999997,"5min":63.468000000000004,"15min":137.36099999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.206,"5min":2.5859999999999999,"15min":3.0329999999999999},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":10.113,"5min":63.447000000000003,"15min":136.88999999999999},"last":2.9399999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.6419999999999999,"15min":3.677},"min":{"1min":0.40000000000000002,"5min":0.375,"15min":0.32100000000000001},"max":{"1min":9.1699999999999999,"5min":63.509,"15min":185.024},"last":1.8540000000000001},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.7770000000000001,"15min":3.3479999999999999},"min":{"1min":0.44700000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":10.077,"5min":63.481999999999999,"15min":184.98599999999999},"last":0.60299999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926030,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2288020,"kb_used_data":1729804,"kb_used_omap":81,"kb_used_meta":558062,"kb_avail":91457132,"statfs":{"total":95995035648,"available":93652103168,"internally_reserved":0,"allocated":1771319296,"data_stored":1769838889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":571455706},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":6.3810000000000002,"15min":3.875},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":5.0780000000000003,"5min":81.385999999999996,"15min":81.385999999999996},"last":2.657},{"interface":"front","average":{"1min":1.8,"5min":5.7300000000000004,"15min":3.899},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":10.457000000000001,"5min":79.453999999999994,"15min":79.453999999999994},"last":1.7709999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":11.619,"5min":6.9870000000000001,"15min":4.391},"min":{"1min":0.27600000000000002,"5min":0.191,"15min":0.191},"max":{"1min":158.13399999999999,"5min":158.13399999999999,"15min":158.13399999999999},"last":1.9450000000000001},{"interface":"front","average":{"1min":11.401999999999999,"5min":7.7629999999999999,"15min":4.6639999999999997},"min":{"1min":0.253,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":161.12,"5min":161.12,"15min":161.12},"last":1.2949999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.4700000000000002,"15min":2.8170000000000002},"min":{"1min":0.28000000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.409000000000001,"5min":58.997,"15min":58.997},"last":2.8279999999999998},{"interface":"front","average":{"1min":2.1499999999999999,"5min":3.5030000000000001,"15min":3.0270000000000001},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.488,"5min":66.135000000000005,"15min":66.135000000000005},"last":1.913}]},{"osd":4,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":2.6139999999999999,"15min":2.1309999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":6.0700000000000003,"5min":58.798000000000002,"15min":58.798000000000002},"last":3.1520000000000001},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.7290000000000001,"15min":2.0190000000000001},"min":{"1min":0.32000000000000001,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":3.1659999999999999,"5min":56.616,"15min":56.616},"last":3.3500000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.8620000000000001,"15min":2.327},"min":{"1min":0.28299999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.241,"5min":56.524000000000001,"15min":56.524000000000001},"last":3.5899999999999999},{"interface":"front","average":{"1min":3.625,"5min":3.157,"15min":2.456},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":51.472000000000001,"5min":58.854999999999997,"15min":58.854999999999997},"last":3.5800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":2.9729999999999999,"15min":2.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.8650000000000002,"5min":56.561,"15min":114.14},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.5579999999999998,"15min":2.5859999999999999},"min":{"1min":0.32100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.218,"5min":56.460999999999999,"15min":114.123},"last":1.802}]},{"osd":7,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":3.0350000000000001,"15min":2.7290000000000001},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.8479999999999999,"5min":56.591000000000001,"15min":117.32599999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.6950000000000001,"5min":2.7429999999999999,"15min":2.7599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.189,"5min":56.497999999999998,"15min":117.28700000000001},"last":3.9550000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762504,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2468540,"kb_used_data":1738484,"kb_used_omap":7,"kb_used_meta":730040,"kb_avail":91276612,"statfs":{"total":95995035648,"available":93467250688,"internally_reserved":0,"allocated":1780207616,"data_stored":1778752588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":747561576},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":2.952,"5min":5.8639999999999999,"15min":5.3819999999999997},"min":{"1min":0.20699999999999999,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":39.718000000000004,"5min":125.765,"15min":125.765},"last":0.34799999999999998},{"interface":"front","average":{"1min":3.5310000000000001,"5min":6.4059999999999997,"15min":5.6950000000000003},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.193},"max":{"1min":38.786000000000001,"5min":125.46899999999999,"15min":125.46899999999999},"last":0.27400000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":5.8559999999999999,"5min":8.1150000000000002,"15min":5.1760000000000002},"min":{"1min":0.22700000000000001,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":103.143,"5min":186.21299999999999,"15min":186.21299999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":7.0309999999999997,"5min":7.891,"15min":4.9470000000000001},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.193},"max":{"1min":105.19,"5min":191.78,"15min":191.78},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":2.891,"5min":4.7149999999999999,"15min":3.871},"min":{"1min":0.252,"5min":0.187,"15min":0.187},"max":{"1min":43.152999999999999,"5min":127.599,"15min":127.599},"last":0.432},{"interface":"front","average":{"1min":3.6349999999999998,"5min":6.5330000000000004,"15min":4.8789999999999996},"min":{"1min":0.246,"5min":0.214,"15min":0.214},"max":{"1min":43.134,"5min":129.98599999999999,"15min":129.98599999999999},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.1459999999999999,"15min":2.6139999999999999},"min":{"1min":0.41499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.306,"5min":27.539999999999999,"15min":106.85899999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.874,"5min":2.4540000000000002,"15min":2.1200000000000001},"min":{"1min":0.36699999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.6019999999999999,"5min":26.663,"15min":37.442},"last":0.36899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":2.766,"15min":2.137},"min":{"1min":0.36099999999999999,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":4.04,"5min":17.901,"15min":17.901},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.78900000000000003,"5min":1.8580000000000001,"15min":1.8899999999999999},"min":{"1min":0.35699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":2.6389999999999998,"5min":18.068000000000001,"15min":18.068000000000001},"last":0.59599999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":1.823,"15min":2.2559999999999998},"min":{"1min":0.42399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.2170000000000001,"5min":14.567,"15min":106.036},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.319,"15min":2.5339999999999998},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.335,"5min":14.529,"15min":106.741},"last":0.58299999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:19:04 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":2.2269999999999999,"15min":2.5590000000000002},"min":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":7.3639999999999999,"5min":14.291,"15min":106.82599999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.2230000000000001,"5min":2.6469999999999998,"15min":2.5790000000000002},"min":{"1min":0.45400000000000001,"5min":0.38900000000000001,"15min":0.35999999999999999},"max":{"1min":4.008,"5min":20.757999999999999,"15min":106.084},"last":0.45800000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986520,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2001972,"kb_used_data":1700588,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":91743180,"statfs":{"total":95995035648,"available":93945016320,"internally_reserved":0,"allocated":1741402112,"data_stored":1740430053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":10.717000000000001,"5min":10.526,"15min":5.4820000000000002},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":132.631,"5min":154.62100000000001,"15min":154.62100000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":10.893000000000001,"5min":10.782,"15min":5.5069999999999997},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":132.59100000000001,"5min":154.37299999999999,"15min":154.37299999999999},"last":0.252}]},{"osd":1,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":7.8129999999999997,"5min":9.4969999999999999,"15min":5.1180000000000003},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":69.706000000000003,"5min":135.25399999999999,"15min":135.25399999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":7.3220000000000001,"5min":9.1270000000000007,"15min":5.0060000000000002},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":69.561000000000007,"5min":134.44300000000001,"15min":134.44300000000001},"last":0.34200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.679,"15min":2.0720000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.245},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":123.598},"last":0.435},{"interface":"front","average":{"1min":1.708,"5min":1.546,"15min":1.986},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":6.21,"5min":8.4800000000000004,"15min":123.619},"last":0.45700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.052,"15min":2.29},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.0069999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.165,"15min":2.4849999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.7579999999999991,"5min":8.9169999999999998,"15min":127.90000000000001},"last":0.308}]},{"osd":5,"last update":"Mon Oct 9 04:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":3.6099999999999999,"15min":1.6279999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":13.599,"5min":111.809,"15min":111.809},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.532,"5min":3.7200000000000002,"15min":1.7929999999999999},"min":{"1min":0.316,"5min":0.252,"15min":0.25},"max":{"1min":9.1530000000000005,"5min":112.002,"15min":112.002},"last":0.41299999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.395,"15min":1.3680000000000001},"min":{"1min":0.35699999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":6.9470000000000001,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.877,"5min":2.2130000000000001,"15min":1.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9320000000000004,"5min":78.421999999999997,"15min":78.421999999999997},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.3049999999999999,"15min":0.95299999999999996},"min":{"1min":0.31900000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.8010000000000002,"5min":34.039000000000001,"15min":34.039000000000001},"last":0.497},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.5109999999999999,"15min":1.0680000000000001},"min":{"1min":0.375,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.384,"5min":33.433999999999997,"15min":33.433999999999997},"last":0.55300000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724883,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3190068,"kb_used_data":2888620,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":90555084,"statfs":{"total":95995035648,"available":92728406016,"internally_reserved":0,"allocated":2957946880,"data_stored":2956994769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":4.984,"15min":5.0759999999999996},"min":{"1min":0.223,"5min":0.193,"15min":0.193},"max":{"1min":13.675000000000001,"5min":118.98399999999999,"15min":151.23599999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.4079999999999999,"5min":5.3380000000000001,"15min":5.4470000000000001},"min":{"1min":0.23499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":13.885,"5min":118.846,"15min":151.566},"last":4.8300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":5.242,"15min":4.0270000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":29.632999999999999,"5min":111.194,"15min":111.194},"last":0.52100000000000002},{"interface":"front","average":{"1min":4.7249999999999996,"5min":5.8289999999999997,"15min":4.4180000000000001},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":29.672000000000001,"5min":112.226,"15min":112.226},"last":0.88200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.444,"15min":3.1299999999999999},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":8.8599999999999994,"5min":11.324,"15min":106.508},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.718,"15min":2.8879999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":3.8199999999999998,"5min":8.5280000000000005,"15min":106.462},"last":5.601}]},{"osd":3,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.0449999999999999,"15min":2.5939999999999999},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.555999999999999,"5min":11.555999999999999,"15min":91.834999999999994},"last":5.1529999999999996},{"interface":"front","average":{"1min":2.3260000000000001,"5min":2.2389999999999999,"15min":2.6890000000000001},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3219999999999992,"5min":12.122999999999999,"15min":91.802000000000007},"last":4.7869999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.242,"15min":0.93300000000000005},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.246},"max":{"1min":9.2579999999999991,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.1200000000000001,"15min":0.92900000000000005},"min":{"1min":0.30199999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":3.77,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.36399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.27,"15min":1.3779999999999999},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":9.4090000000000007,"5min":12.068,"15min":123.239},"last":4.9930000000000003},{"interface":"front","average":{"1min":1.615,"5min":1.45,"15min":1.4670000000000001},"min":{"1min":0.313,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":9.7720000000000002,"15min":123.014},"last":0.39300000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.202,"15min":0.995},"min":{"1min":0.33100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.3289999999999997,"5min":12.096,"15min":12.096},"last":0.378},{"interface":"front","average":{"1min":1.24,"5min":1.446,"15min":1.095},"min":{"1min":0.36699999999999999,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":9.3019999999999996,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.45000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1764229120,"data_stored":1764229120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3490844672,"data_stored":3490840600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1757413376,"data_stored":1757413376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1770651648,"data_stored":1770651648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1734348800,"data_stored":1734344728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2948272128,"data_stored":2948268056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":10289152,"data_stored":10289152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3909160960,"data_stored":3909156888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:20:07.501 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:20:07.502 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:07.540 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:07.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:07.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:07.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:07.778 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:07.793 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:20:07.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:07.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:07.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:08.064 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:08.090 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:08.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:08.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:08.277 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:08.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:08.371 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:08.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:08.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:08.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:08.546 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:08.585 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:08.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:08.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:08.793 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:08.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:08 smithi163 ceph-mon[221067]: pgmap v944: 119 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+undersized+remapped, 113 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 19640/253821 objects misplaced (7.738%); 21 MiB/s, 336 objects/s recovering 2023-10-09T04:20:08.827 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:08.827 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:08 smithi163 ceph-mon[213841]: pgmap v944: 119 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+undersized+remapped, 113 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 19640/253821 objects misplaced (7.738%); 21 MiB/s, 336 objects/s recovering 2023-10-09T04:20:08.828 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:08.832 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:09.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:08 smithi181 ceph-mon[196979]: pgmap v944: 119 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+undersized+remapped, 113 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 19640/253821 objects misplaced (7.738%); 21 MiB/s, 336 objects/s recovering 2023-10-09T04:20:09.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:09.576 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:20:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/984077655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:09 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:09 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:09.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/984077655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:09.837 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:09 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:09.838 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:09 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:09.875 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:20:09.875 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:20:09.909 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/984077655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:09.909 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:09 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:09.909 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:09 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:20:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:20:10] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: failed to encode map e353 with expected crc 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/984077655' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: osdmap e353: 8 total, 8 up, 7 in 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: failed to encode map e353 with expected crc 2023-10-09T04:20:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: pgmap v946: 119 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 115 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 19129/251655 objects misplaced (7.601%); 24 MiB/s, 380 objects/s recovering 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: failed to encode map e353 with expected crc 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/984077655' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: osdmap e353: 8 total, 8 up, 7 in 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: failed to encode map e353 with expected crc 2023-10-09T04:20:10.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: pgmap v946: 119 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 115 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 19129/251655 objects misplaced (7.601%); 24 MiB/s, 380 objects/s recovering 2023-10-09T04:20:10.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:10.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:10.975 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:10.975 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: failed to encode map e353 with expected crc 2023-10-09T04:20:10.975 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/984077655' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:10.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: osdmap e353: 8 total, 8 up, 7 in 2023-10-09T04:20:10.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: failed to encode map e353 with expected crc 2023-10-09T04:20:10.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: pgmap v946: 119 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 115 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 19129/251655 objects misplaced (7.601%); 24 MiB/s, 380 objects/s recovering 2023-10-09T04:20:10.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:10.976 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:11.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:20:11.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:11.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:11.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:11.319 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:11.449 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:11.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:11.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:11.554 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:11.584 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:11.584 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:20:11.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:11.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:11.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:11.776 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:11.881 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:11.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:11.908 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:20:11.932 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:20:11.965 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:20:11.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:11.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[213841]: osdmap e354: 8 total, 8 up, 7 in 2023-10-09T04:20:11.994 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[213841]: failed to encode map e354 with expected crc 2023-10-09T04:20:11.994 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[213841]: failed to encode map e354 with expected crc 2023-10-09T04:20:11.994 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/949237340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:11.999 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:20:12.029 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:20:12.045 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:12.070 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:20:12.126 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:20:12.157 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.173 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:20:12.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:11 smithi181 ceph-mon[196979]: osdmap e354: 8 total, 8 up, 7 in 2023-10-09T04:20:12.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:11 smithi181 ceph-mon[196979]: failed to encode map e354 with expected crc 2023-10-09T04:20:12.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:11 smithi181 ceph-mon[196979]: failed to encode map e354 with expected crc 2023-10-09T04:20:12.225 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/949237340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:12.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:12.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:12.263 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[221067]: osdmap e354: 8 total, 8 up, 7 in 2023-10-09T04:20:12.264 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[221067]: failed to encode map e354 with expected crc 2023-10-09T04:20:12.264 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[221067]: failed to encode map e354 with expected crc 2023-10-09T04:20:12.264 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/949237340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:12.324 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:12.333 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:20:12.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:12.387 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:12.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[213841]: pgmap v948: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+remapped, 1 active+remapped+backfilling, 115 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 18084/243711 objects misplaced (7.420%); 26 MiB/s, 409 objects/s recovering 2023-10-09T04:20:12.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[213841]: failed to encode map e355 with expected crc 2023-10-09T04:20:12.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[213841]: osdmap e355: 8 total, 8 up, 7 in 2023-10-09T04:20:12.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[213841]: failed to encode map e355 with expected crc 2023-10-09T04:20:12.993 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[213841]: from='client.35952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:12 smithi181 ceph-mon[196979]: pgmap v948: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+remapped, 1 active+remapped+backfilling, 115 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 18084/243711 objects misplaced (7.420%); 26 MiB/s, 409 objects/s recovering 2023-10-09T04:20:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:12 smithi181 ceph-mon[196979]: failed to encode map e355 with expected crc 2023-10-09T04:20:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:12 smithi181 ceph-mon[196979]: osdmap e355: 8 total, 8 up, 7 in 2023-10-09T04:20:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:12 smithi181 ceph-mon[196979]: failed to encode map e355 with expected crc 2023-10-09T04:20:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:12 smithi181 ceph-mon[196979]: from='client.35952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[221067]: pgmap v948: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+remapped, 1 active+remapped+backfilling, 115 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 18084/243711 objects misplaced (7.420%); 26 MiB/s, 409 objects/s recovering 2023-10-09T04:20:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[221067]: failed to encode map e355 with expected crc 2023-10-09T04:20:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[221067]: osdmap e355: 8 total, 8 up, 7 in 2023-10-09T04:20:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[221067]: failed to encode map e355 with expected crc 2023-10-09T04:20:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:12 smithi163 ceph-mon[221067]: from='client.35952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:13.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:20:13.692 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:20:13.692 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:20:13.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.692 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:117935 9.1 9:8153eeee:::benchmark_data_smithi163_464650_object19209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:13.685581+0000", 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.000117, 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014964500000000001, 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 117935 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.693 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685581+0000", 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685581+0000", 2023-10-09T04:20:13.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685579+0000", 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685589+0000", 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6690000000000001e-06 2023-10-09T04:20:13.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685611+0000", 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2291999999999999e-05 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685620+0000", 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4209999999999995e-06 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685676+0000", 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5804000000000003e-05 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:117936 9.3 9:c47ccc85:::benchmark_data_smithi163_464650_object19210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:13.685638+0000", 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 6.0223000000000003e-05, 2023-10-09T04:20:13.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017459999999999999, 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 117936 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:13.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685638+0000", 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685638+0000", 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685638+0000", 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685641+0000", 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.954e-06 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685646+0000", 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9769999999999997e-06 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685651+0000", 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.763e-06 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:13.685708+0000", 2023-10-09T04:20:13.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7089000000000002e-05 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 2 2023-10-09T04:20:13.702 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:20:14.260 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:13 smithi163 ceph-mon[213841]: pgmap v950: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+remapped, 115 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 16827/242160 objects misplaced (6.949%); 20 MiB/s, 319 objects/s recovering 2023-10-09T04:20:14.260 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:13 smithi163 ceph-mon[221067]: pgmap v950: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+remapped, 115 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 16827/242160 objects misplaced (6.949%); 20 MiB/s, 319 objects/s recovering 2023-10-09T04:20:14.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:14 smithi181 ceph-mon[196979]: pgmap v950: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+remapped, 115 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 16827/242160 objects misplaced (6.949%); 20 MiB/s, 319 objects/s recovering 2023-10-09T04:20:14.301 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":949,"stamp":"2023-10-09T04:20:11.990676+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5322981561,"num_objects":81237,"num_object_clones":0,"num_object_copies":243711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":18084,"num_objects_unfound":0,"num_objects_dirty":81237,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":116328,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13609,"num_bytes_recovered":887989985,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96860,"ondisk_log_size":96860,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19748844,"kb_used_data":16196340,"kb_used_omap":126,"kb_used_meta":3552193,"kb_avail":636467220,"statfs":{"total":671965249536,"available":651742433280,"internally_reserved":0,"allocated":16585052160,"data_stored":16576122626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3637446267},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-347144192,"num_objects":-5297,"num_object_clones":0,"num_object_copies":-15891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10755,"num_objects_unfound":0,"num_objects_dirty":-5297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2793,"num_bytes_recovered":183042048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.161742"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":477,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.019712+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:20:05.019712+0000","last_peered":"2023-10-09T04:20:05.019712+0000","last_clean":"2023-10-09T04:20:05.019712+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:20:05.019712+0000","last_undegraded":"2023-10-09T04:20:05.019712+0000","last_fullsized":"2023-10-09T04:20:05.019712+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":97,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015326+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:20:05.015326+0000","last_peered":"2023-10-09T04:20:05.015326+0000","last_clean":"2023-10-09T04:20:05.015326+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:20:05.015326+0000","last_undegraded":"2023-10-09T04:20:05.015326+0000","last_fullsized":"2023-10-09T04:20:05.015326+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":327,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.022159+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:20:04.022159+0000","last_peered":"2023-10-09T04:20:04.022159+0000","last_clean":"2023-10-09T04:20:04.022159+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:20:04.022159+0000","last_undegraded":"2023-10-09T04:20:04.022159+0000","last_fullsized":"2023-10-09T04:20:04.022159+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":470,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020452+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:20:05.020452+0000","last_peered":"2023-10-09T04:20:05.020452+0000","last_clean":"2023-10-09T04:20:05.020452+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:20:05.020452+0000","last_undegraded":"2023-10-09T04:20:05.020452+0000","last_fullsized":"2023-10-09T04:20:05.020452+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":342,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.008902+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:20:04.008902+0000","last_peered":"2023-10-09T04:20:04.008902+0000","last_clean":"2023-10-09T04:20:04.008902+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:20:04.008902+0000","last_undegraded":"2023-10-09T04:20:04.008902+0000","last_fullsized":"2023-10-09T04:20:04.008902+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":332,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778310+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:20:09.778310+0000","last_peered":"2023-10-09T04:20:09.778310+0000","last_clean":"2023-10-09T04:20:09.778310+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:20:09.778310+0000","last_undegraded":"2023-10-09T04:20:09.778310+0000","last_fullsized":"2023-10-09T04:20:09.778310+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":477,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015066+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:20:05.015066+0000","last_peered":"2023-10-09T04:20:05.015066+0000","last_clean":"2023-10-09T04:20:05.015066+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:20:05.015066+0000","last_undegraded":"2023-10-09T04:20:05.015066+0000","last_fullsized":"2023-10-09T04:20:05.015066+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":329,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.781841+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:20:09.781841+0000","last_peered":"2023-10-09T04:20:09.781841+0000","last_clean":"2023-10-09T04:20:09.781841+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:20:09.781841+0000","last_undegraded":"2023-10-09T04:20:09.781841+0000","last_fullsized":"2023-10-09T04:20:09.781841+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":296,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.021009+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:20:05.021009+0000","last_peered":"2023-10-09T04:20:05.021009+0000","last_clean":"2023-10-09T04:20:05.021009+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:20:05.021009+0000","last_undegraded":"2023-10-09T04:20:05.021009+0000","last_fullsized":"2023-10-09T04:20:05.021009+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":752,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025606+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:20:05.025606+0000","last_peered":"2023-10-09T04:20:05.025606+0000","last_clean":"2023-10-09T04:20:05.025606+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:20:05.025606+0000","last_undegraded":"2023-10-09T04:20:05.025606+0000","last_fullsized":"2023-10-09T04:20:05.025606+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":330,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778173+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:20:09.778173+0000","last_peered":"2023-10-09T04:20:09.778173+0000","last_clean":"2023-10-09T04:20:09.778173+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:20:09.778173+0000","last_undegraded":"2023-10-09T04:20:09.778173+0000","last_fullsized":"2023-10-09T04:20:09.778173+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.038434+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:20:05.038434+0000","last_peered":"2023-10-09T04:20:05.038434+0000","last_clean":"2023-10-09T04:20:05.038434+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:20:05.038434+0000","last_undegraded":"2023-10-09T04:20:05.038434+0000","last_fullsized":"2023-10-09T04:20:05.038434+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":477,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013690+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:20:04.013690+0000","last_peered":"2023-10-09T04:20:04.013690+0000","last_clean":"2023-10-09T04:20:04.013690+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:20:04.013690+0000","last_undegraded":"2023-10-09T04:20:04.013690+0000","last_fullsized":"2023-10-09T04:20:04.013690+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":312,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.022442+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:20:05.022442+0000","last_peered":"2023-10-09T04:20:05.022442+0000","last_clean":"2023-10-09T04:20:05.022442+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:20:05.022442+0000","last_undegraded":"2023-10-09T04:20:05.022442+0000","last_fullsized":"2023-10-09T04:20:05.022442+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":47,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020715+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:20:05.020715+0000","last_peered":"2023-10-09T04:20:05.020715+0000","last_clean":"2023-10-09T04:20:05.020715+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:20:05.020715+0000","last_undegraded":"2023-10-09T04:20:05.020715+0000","last_fullsized":"2023-10-09T04:20:05.020715+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":487,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020884+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:20:05.020884+0000","last_peered":"2023-10-09T04:20:05.020884+0000","last_clean":"2023-10-09T04:20:05.020884+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:20:05.020884+0000","last_undegraded":"2023-10-09T04:20:05.020884+0000","last_fullsized":"2023-10-09T04:20:05.020884+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":395,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.026002+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:20:05.026002+0000","last_peered":"2023-10-09T04:20:05.026002+0000","last_clean":"2023-10-09T04:20:05.026002+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:20:05.026002+0000","last_undegraded":"2023-10-09T04:20:05.026002+0000","last_fullsized":"2023-10-09T04:20:05.026002+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.008846+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:20:04.008846+0000","last_peered":"2023-10-09T04:20:04.008846+0000","last_clean":"2023-10-09T04:20:04.008846+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:20:04.008846+0000","last_undegraded":"2023-10-09T04:20:04.008846+0000","last_fullsized":"2023-10-09T04:20:04.008846+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":493,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.022998+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:20:04.022998+0000","last_peered":"2023-10-09T04:20:04.022998+0000","last_clean":"2023-10-09T04:20:04.022998+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:20:04.022998+0000","last_undegraded":"2023-10-09T04:20:04.022998+0000","last_fullsized":"2023-10-09T04:20:04.022998+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":127,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015618+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:20:05.015618+0000","last_peered":"2023-10-09T04:20:05.015618+0000","last_clean":"2023-10-09T04:20:05.015618+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:20:05.015618+0000","last_undegraded":"2023-10-09T04:20:05.015618+0000","last_fullsized":"2023-10-09T04:20:05.015618+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016095+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:20:05.016095+0000","last_peered":"2023-10-09T04:20:05.016095+0000","last_clean":"2023-10-09T04:20:05.016095+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:20:05.016095+0000","last_undegraded":"2023-10-09T04:20:05.016095+0000","last_fullsized":"2023-10-09T04:20:05.016095+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":503,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.777801+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:20:09.777801+0000","last_peered":"2023-10-09T04:20:09.777801+0000","last_clean":"2023-10-09T04:20:09.777801+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:20:09.777801+0000","last_undegraded":"2023-10-09T04:20:09.777801+0000","last_fullsized":"2023-10-09T04:20:09.777801+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":346,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025919+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:20:05.025919+0000","last_peered":"2023-10-09T04:20:05.025919+0000","last_clean":"2023-10-09T04:20:05.025919+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:20:05.025919+0000","last_undegraded":"2023-10-09T04:20:05.025919+0000","last_fullsized":"2023-10-09T04:20:05.025919+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":30,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.012485+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:20:04.012485+0000","last_peered":"2023-10-09T04:20:04.012485+0000","last_clean":"2023-10-09T04:20:04.012485+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:20:04.012485+0000","last_undegraded":"2023-10-09T04:20:04.012485+0000","last_fullsized":"2023-10-09T04:20:04.012485+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":497,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.019632+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:20:05.019632+0000","last_peered":"2023-10-09T04:20:05.019632+0000","last_clean":"2023-10-09T04:20:05.019632+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:20:05.019632+0000","last_undegraded":"2023-10-09T04:20:05.019632+0000","last_fullsized":"2023-10-09T04:20:05.019632+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":330,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778101+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:20:09.778101+0000","last_peered":"2023-10-09T04:20:09.778101+0000","last_clean":"2023-10-09T04:20:09.778101+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:20:09.778101+0000","last_undegraded":"2023-10-09T04:20:09.778101+0000","last_fullsized":"2023-10-09T04:20:09.778101+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":331,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020626+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:20:05.020626+0000","last_peered":"2023-10-09T04:20:05.020626+0000","last_clean":"2023-10-09T04:20:05.020626+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:20:05.020626+0000","last_undegraded":"2023-10-09T04:20:05.020626+0000","last_fullsized":"2023-10-09T04:20:05.020626+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":493,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.041214+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:20:05.041214+0000","last_peered":"2023-10-09T04:20:05.041214+0000","last_clean":"2023-10-09T04:20:05.041214+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:20:05.041214+0000","last_undegraded":"2023-10-09T04:20:05.041214+0000","last_fullsized":"2023-10-09T04:20:05.041214+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015088+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:20:05.015088+0000","last_peered":"2023-10-09T04:20:05.015088+0000","last_clean":"2023-10-09T04:20:05.015088+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:20:05.015088+0000","last_undegraded":"2023-10-09T04:20:05.015088+0000","last_fullsized":"2023-10-09T04:20:05.015088+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":330,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778298+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:20:09.778298+0000","last_peered":"2023-10-09T04:20:09.778298+0000","last_clean":"2023-10-09T04:20:09.778298+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:20:09.778298+0000","last_undegraded":"2023-10-09T04:20:09.778298+0000","last_fullsized":"2023-10-09T04:20:09.778298+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":497,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020183+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:20:05.020183+0000","last_peered":"2023-10-09T04:20:05.020183+0000","last_clean":"2023-10-09T04:20:05.020183+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:20:05.020183+0000","last_undegraded":"2023-10-09T04:20:05.020183+0000","last_fullsized":"2023-10-09T04:20:05.020183+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":343,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.781765+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:20:09.781765+0000","last_peered":"2023-10-09T04:20:09.781765+0000","last_clean":"2023-10-09T04:20:09.781765+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:20:09.781765+0000","last_undegraded":"2023-10-09T04:20:09.781765+0000","last_fullsized":"2023-10-09T04:20:09.781765+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.256330+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:20:05.256330+0000","last_peered":"2023-10-09T04:20:05.256330+0000","last_clean":"2023-10-09T04:20:05.256330+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:20:05.256330+0000","last_undegraded":"2023-10-09T04:20:05.256330+0000","last_fullsized":"2023-10-09T04:20:05.256330+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"354'7230","reported_seq":7288,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.215876+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:20:11.215876+0000","last_peered":"2023-10-09T04:20:11.215876+0000","last_clean":"2023-10-09T04:20:11.215876+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:20:11.215876+0000","last_undegraded":"2023-10-09T04:20:11.215876+0000","last_fullsized":"2023-10-09T04:20:11.215876+0000","mapping_epoch":306,"log_start":"314'900","ondisk_log_start":"314'900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6330,"log_dups_size":0,"ondisk_log_size":6330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324403200,"num_objects":4950,"num_object_clones":0,"num_object_copies":14850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7230,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":755,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025699+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:20:05.025699+0000","last_peered":"2023-10-09T04:20:05.025699+0000","last_clean":"2023-10-09T04:20:05.025699+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:20:05.025699+0000","last_undegraded":"2023-10-09T04:20:05.025699+0000","last_fullsized":"2023-10-09T04:20:05.025699+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":341,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025795+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:20:05.025795+0000","last_peered":"2023-10-09T04:20:05.025795+0000","last_clean":"2023-10-09T04:20:05.025795+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:20:05.025795+0000","last_undegraded":"2023-10-09T04:20:05.025795+0000","last_fullsized":"2023-10-09T04:20:05.025795+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":304,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.019853+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:20:05.019853+0000","last_peered":"2023-10-09T04:20:05.019853+0000","last_clean":"2023-10-09T04:20:05.019853+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:20:05.019853+0000","last_undegraded":"2023-10-09T04:20:05.019853+0000","last_fullsized":"2023-10-09T04:20:05.019853+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"352'7225","reported_seq":7287,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:08.231289+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:20:08.231289+0000","last_peered":"2023-10-09T04:20:08.231289+0000","last_clean":"2023-10-09T04:20:08.231289+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:20:08.231289+0000","last_undegraded":"2023-10-09T04:20:08.231289+0000","last_fullsized":"2023-10-09T04:20:08.231289+0000","mapping_epoch":310,"log_start":"318'1873","ondisk_log_start":"318'1873","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338886656,"num_objects":5171,"num_object_clones":0,"num_object_copies":15513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7225,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":519,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.036847+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:20:05.036847+0000","last_peered":"2023-10-09T04:20:05.036847+0000","last_clean":"2023-10-09T04:20:05.036847+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:20:05.036847+0000","last_undegraded":"2023-10-09T04:20:05.036847+0000","last_fullsized":"2023-10-09T04:20:05.036847+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":345,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025433+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:20:05.025433+0000","last_peered":"2023-10-09T04:20:05.025433+0000","last_clean":"2023-10-09T04:20:05.025433+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:20:05.025433+0000","last_undegraded":"2023-10-09T04:20:05.025433+0000","last_fullsized":"2023-10-09T04:20:05.025433+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015238+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:20:05.015238+0000","last_peered":"2023-10-09T04:20:05.015238+0000","last_clean":"2023-10-09T04:20:05.015238+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:20:05.015238+0000","last_undegraded":"2023-10-09T04:20:05.015238+0000","last_fullsized":"2023-10-09T04:20:05.015238+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"353'7449","reported_seq":7515,"reported_epoch":353,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:10.795765+0000","last_change":"2023-10-09T04:20:03.101091+0000","last_active":"2023-10-09T04:20:10.795765+0000","last_peered":"2023-10-09T04:20:10.795765+0000","last_clean":"2023-10-09T04:20:00.944119+0000","last_became_active":"2023-10-09T04:20:03.092473+0000","last_became_peered":"2023-10-09T04:20:03.092473+0000","last_unstale":"2023-10-09T04:20:10.795765+0000","last_undegraded":"2023-10-09T04:20:10.795765+0000","last_fullsized":"2023-10-09T04:20:10.795765+0000","mapping_epoch":349,"log_start":"314'1518","ondisk_log_start":"314'1518","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:49.431905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336003072,"num_objects":5127,"num_object_clones":0,"num_object_copies":15381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5127,"num_objects_unfound":0,"num_objects_dirty":5127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7449,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":5127}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":487,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020802+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:20:05.020802+0000","last_peered":"2023-10-09T04:20:05.020802+0000","last_clean":"2023-10-09T04:20:05.020802+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:20:05.020802+0000","last_undegraded":"2023-10-09T04:20:05.020802+0000","last_fullsized":"2023-10-09T04:20:05.020802+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":310,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025914+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:20:05.025914+0000","last_peered":"2023-10-09T04:20:05.025914+0000","last_clean":"2023-10-09T04:20:05.025914+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:20:05.025914+0000","last_undegraded":"2023-10-09T04:20:05.025914+0000","last_fullsized":"2023-10-09T04:20:05.025914+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":284,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.009315+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:20:04.009315+0000","last_peered":"2023-10-09T04:20:04.009315+0000","last_clean":"2023-10-09T04:20:04.009315+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:20:04.009315+0000","last_undegraded":"2023-10-09T04:20:04.009315+0000","last_fullsized":"2023-10-09T04:20:04.009315+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"353'7268","reported_seq":7323,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.704428+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:20:10.704428+0000","last_peered":"2023-10-09T04:20:10.704428+0000","last_clean":"2023-10-09T04:20:10.704428+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:20:10.704428+0000","last_undegraded":"2023-10-09T04:20:10.704428+0000","last_fullsized":"2023-10-09T04:20:10.704428+0000","mapping_epoch":306,"log_start":"314'1287","ondisk_log_start":"314'1287","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5981,"log_dups_size":0,"ondisk_log_size":5981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331350016,"num_objects":5056,"num_object_clones":0,"num_object_copies":15168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7268,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":497,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016159+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:20:05.016159+0000","last_peered":"2023-10-09T04:20:05.016159+0000","last_clean":"2023-10-09T04:20:05.016159+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:20:05.016159+0000","last_undegraded":"2023-10-09T04:20:05.016159+0000","last_fullsized":"2023-10-09T04:20:05.016159+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":342,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020077+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:20:05.020077+0000","last_peered":"2023-10-09T04:20:05.020077+0000","last_clean":"2023-10-09T04:20:05.020077+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:20:05.020077+0000","last_undegraded":"2023-10-09T04:20:05.020077+0000","last_fullsized":"2023-10-09T04:20:05.020077+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":331,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778471+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:20:09.778471+0000","last_peered":"2023-10-09T04:20:09.778471+0000","last_clean":"2023-10-09T04:20:09.778471+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:20:09.778471+0000","last_undegraded":"2023-10-09T04:20:09.778471+0000","last_fullsized":"2023-10-09T04:20:09.778471+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"353'6968","reported_seq":10396,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.796339+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:20:10.796339+0000","last_peered":"2023-10-09T04:20:10.796339+0000","last_clean":"2023-10-09T04:20:10.796339+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:20:10.796339+0000","last_undegraded":"2023-10-09T04:20:10.796339+0000","last_fullsized":"2023-10-09T04:20:10.796339+0000","mapping_epoch":320,"log_start":"317'1700","ondisk_log_start":"317'1700","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321781760,"num_objects":4910,"num_object_clones":0,"num_object_copies":14730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6968,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":776,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025000+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:20:05.025000+0000","last_peered":"2023-10-09T04:20:05.025000+0000","last_clean":"2023-10-09T04:20:05.025000+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:20:05.025000+0000","last_undegraded":"2023-10-09T04:20:05.025000+0000","last_fullsized":"2023-10-09T04:20:05.025000+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":313,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778519+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:20:09.778519+0000","last_peered":"2023-10-09T04:20:09.778519+0000","last_clean":"2023-10-09T04:20:09.778519+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:20:09.778519+0000","last_undegraded":"2023-10-09T04:20:09.778519+0000","last_fullsized":"2023-10-09T04:20:09.778519+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":127,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015650+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:20:05.015650+0000","last_peered":"2023-10-09T04:20:05.015650+0000","last_clean":"2023-10-09T04:20:05.015650+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:20:05.015650+0000","last_undegraded":"2023-10-09T04:20:05.015650+0000","last_fullsized":"2023-10-09T04:20:05.015650+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"352'7160","reported_seq":10727,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:08.240069+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:20:08.240069+0000","last_peered":"2023-10-09T04:20:08.240069+0000","last_clean":"2023-10-09T04:20:08.240069+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:20:08.240069+0000","last_undegraded":"2023-10-09T04:20:08.240069+0000","last_fullsized":"2023-10-09T04:20:08.240069+0000","mapping_epoch":329,"log_start":"318'1860","ondisk_log_start":"318'1860","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335282176,"num_objects":5116,"num_object_clones":0,"num_object_copies":15348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7160,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":522,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.019542+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:20:05.019542+0000","last_peered":"2023-10-09T04:20:05.019542+0000","last_clean":"2023-10-09T04:20:05.019542+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:20:05.019542+0000","last_undegraded":"2023-10-09T04:20:05.019542+0000","last_fullsized":"2023-10-09T04:20:05.019542+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015143+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:20:05.015143+0000","last_peered":"2023-10-09T04:20:05.015143+0000","last_clean":"2023-10-09T04:20:05.015143+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:20:05.015143+0000","last_undegraded":"2023-10-09T04:20:05.015143+0000","last_fullsized":"2023-10-09T04:20:05.015143+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":97,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016747+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:20:05.016747+0000","last_peered":"2023-10-09T04:20:05.016747+0000","last_clean":"2023-10-09T04:20:05.016747+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:20:05.016747+0000","last_undegraded":"2023-10-09T04:20:05.016747+0000","last_fullsized":"2023-10-09T04:20:05.016747+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"353'7191","reported_seq":7256,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.711146+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:20:10.711146+0000","last_peered":"2023-10-09T04:20:10.711146+0000","last_clean":"2023-10-09T04:20:10.711146+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:20:10.711146+0000","last_undegraded":"2023-10-09T04:20:10.711146+0000","last_fullsized":"2023-10-09T04:20:10.711146+0000","mapping_epoch":310,"log_start":"318'1900","ondisk_log_start":"318'1900","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324730880,"num_objects":4955,"num_object_clones":0,"num_object_copies":14865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7191,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":476,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.009612+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:20:04.009612+0000","last_peered":"2023-10-09T04:20:04.009612+0000","last_clean":"2023-10-09T04:20:04.009612+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:20:04.009612+0000","last_undegraded":"2023-10-09T04:20:04.009612+0000","last_fullsized":"2023-10-09T04:20:04.009612+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.014913+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:20:05.014913+0000","last_peered":"2023-10-09T04:20:05.014913+0000","last_clean":"2023-10-09T04:20:05.014913+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:20:05.014913+0000","last_undegraded":"2023-10-09T04:20:05.014913+0000","last_fullsized":"2023-10-09T04:20:05.014913+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":147,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016295+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:20:05.016295+0000","last_peered":"2023-10-09T04:20:05.016295+0000","last_clean":"2023-10-09T04:20:05.016295+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:20:05.016295+0000","last_undegraded":"2023-10-09T04:20:05.016295+0000","last_fullsized":"2023-10-09T04:20:05.016295+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"352'7154","reported_seq":7843,"reported_epoch":352,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:20:08.229447+0000","last_change":"2023-10-09T04:20:05.061955+0000","last_active":"2023-10-09T04:20:08.229447+0000","last_peered":"2023-10-09T04:20:08.229447+0000","last_clean":"2023-10-09T04:20:00.980108+0000","last_became_active":"2023-10-09T04:20:03.079675+0000","last_became_peered":"2023-10-09T04:20:03.079675+0000","last_unstale":"2023-10-09T04:20:08.229447+0000","last_undegraded":"2023-10-09T04:20:08.229447+0000","last_fullsized":"2023-10-09T04:20:08.229447+0000","mapping_epoch":349,"log_start":"318'1881","ondisk_log_start":"318'1881","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5273,"log_dups_size":0,"ondisk_log_size":5273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:58.283644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4957,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7154,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":5268}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":497,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.019905+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:20:05.019905+0000","last_peered":"2023-10-09T04:20:05.019905+0000","last_clean":"2023-10-09T04:20:05.019905+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:20:05.019905+0000","last_undegraded":"2023-10-09T04:20:05.019905+0000","last_fullsized":"2023-10-09T04:20:05.019905+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.256264+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:20:05.256264+0000","last_peered":"2023-10-09T04:20:05.256264+0000","last_clean":"2023-10-09T04:20:05.256264+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:20:05.256264+0000","last_undegraded":"2023-10-09T04:20:05.256264+0000","last_fullsized":"2023-10-09T04:20:05.256264+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":19,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.014905+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:20:04.014905+0000","last_peered":"2023-10-09T04:20:04.014905+0000","last_clean":"2023-10-09T04:20:04.014905+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:20:04.014905+0000","last_undegraded":"2023-10-09T04:20:04.014905+0000","last_fullsized":"2023-10-09T04:20:04.014905+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"353'7352","reported_seq":7417,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.705368+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:20:10.705368+0000","last_peered":"2023-10-09T04:20:10.705368+0000","last_clean":"2023-10-09T04:20:10.705368+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:20:10.705368+0000","last_undegraded":"2023-10-09T04:20:10.705368+0000","last_fullsized":"2023-10-09T04:20:10.705368+0000","mapping_epoch":310,"log_start":"314'1434","ondisk_log_start":"314'1434","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331481088,"num_objects":5058,"num_object_clones":0,"num_object_copies":15174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7352,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":517,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.041914+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:20:05.041914+0000","last_peered":"2023-10-09T04:20:05.041914+0000","last_clean":"2023-10-09T04:20:05.041914+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:20:05.041914+0000","last_undegraded":"2023-10-09T04:20:05.041914+0000","last_fullsized":"2023-10-09T04:20:05.041914+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":331,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.022991+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:20:04.022991+0000","last_peered":"2023-10-09T04:20:04.022991+0000","last_clean":"2023-10-09T04:20:04.022991+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:20:04.022991+0000","last_undegraded":"2023-10-09T04:20:04.022991+0000","last_fullsized":"2023-10-09T04:20:04.022991+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":349,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020457+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:20:05.020457+0000","last_peered":"2023-10-09T04:20:05.020457+0000","last_clean":"2023-10-09T04:20:05.020457+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:20:05.020457+0000","last_undegraded":"2023-10-09T04:20:05.020457+0000","last_fullsized":"2023-10-09T04:20:05.020457+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":19,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013926+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:20:04.013926+0000","last_peered":"2023-10-09T04:20:04.013926+0000","last_clean":"2023-10-09T04:20:04.013926+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:20:04.013926+0000","last_undegraded":"2023-10-09T04:20:04.013926+0000","last_fullsized":"2023-10-09T04:20:04.013926+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"354'7296","reported_seq":7353,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.245902+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:20:11.245902+0000","last_peered":"2023-10-09T04:20:11.245902+0000","last_clean":"2023-10-09T04:20:11.245902+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:20:11.245902+0000","last_undegraded":"2023-10-09T04:20:11.245902+0000","last_fullsized":"2023-10-09T04:20:11.245902+0000","mapping_epoch":306,"log_start":"314'1000","ondisk_log_start":"314'1000","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328073216,"num_objects":5006,"num_object_clones":0,"num_object_copies":15018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7296,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":470,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020970+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:20:05.020970+0000","last_peered":"2023-10-09T04:20:05.020970+0000","last_clean":"2023-10-09T04:20:05.020970+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:20:05.020970+0000","last_undegraded":"2023-10-09T04:20:05.020970+0000","last_fullsized":"2023-10-09T04:20:05.020970+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":326,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.019979+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:20:05.019979+0000","last_peered":"2023-10-09T04:20:05.019979+0000","last_clean":"2023-10-09T04:20:05.019979+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:20:05.019979+0000","last_undegraded":"2023-10-09T04:20:05.019979+0000","last_fullsized":"2023-10-09T04:20:05.019979+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016024+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:20:05.016024+0000","last_peered":"2023-10-09T04:20:05.016024+0000","last_clean":"2023-10-09T04:20:05.016024+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:20:05.016024+0000","last_undegraded":"2023-10-09T04:20:05.016024+0000","last_fullsized":"2023-10-09T04:20:05.016024+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":359,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.022921+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:20:04.022921+0000","last_peered":"2023-10-09T04:20:04.022921+0000","last_clean":"2023-10-09T04:20:04.022921+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:20:04.022921+0000","last_undegraded":"2023-10-09T04:20:04.022921+0000","last_fullsized":"2023-10-09T04:20:04.022921+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"353'7375","reported_seq":7430,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.710682+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:20:10.710682+0000","last_peered":"2023-10-09T04:20:10.710682+0000","last_clean":"2023-10-09T04:20:10.710682+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:20:10.710682+0000","last_undegraded":"2023-10-09T04:20:10.710682+0000","last_fullsized":"2023-10-09T04:20:10.710682+0000","mapping_epoch":306,"log_start":"314'1468","ondisk_log_start":"314'1468","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5907,"log_dups_size":0,"ondisk_log_size":5907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7375,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":487,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778178+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:20:09.778178+0000","last_peered":"2023-10-09T04:20:09.778178+0000","last_clean":"2023-10-09T04:20:09.778178+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:20:09.778178+0000","last_undegraded":"2023-10-09T04:20:09.778178+0000","last_fullsized":"2023-10-09T04:20:09.778178+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":187,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013598+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:20:04.013598+0000","last_peered":"2023-10-09T04:20:04.013598+0000","last_clean":"2023-10-09T04:20:04.013598+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:20:04.013598+0000","last_undegraded":"2023-10-09T04:20:04.013598+0000","last_fullsized":"2023-10-09T04:20:04.013598+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":334,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.011906+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:20:04.011906+0000","last_peered":"2023-10-09T04:20:04.011906+0000","last_clean":"2023-10-09T04:20:04.011906+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:20:04.011906+0000","last_undegraded":"2023-10-09T04:20:04.011906+0000","last_fullsized":"2023-10-09T04:20:04.011906+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":30,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.022197+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:20:04.022197+0000","last_peered":"2023-10-09T04:20:04.022197+0000","last_clean":"2023-10-09T04:20:04.022197+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:20:04.022197+0000","last_undegraded":"2023-10-09T04:20:04.022197+0000","last_fullsized":"2023-10-09T04:20:04.022197+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"354'7320","reported_seq":12157,"reported_epoch":354,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:11.387631+0000","last_change":"2023-10-09T04:20:05.060638+0000","last_active":"2023-10-09T04:20:11.387631+0000","last_peered":"2023-10-09T04:20:11.387631+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:20:02.106154+0000","last_became_peered":"2023-10-09T04:20:02.106154+0000","last_unstale":"2023-10-09T04:20:11.387631+0000","last_undegraded":"2023-10-09T04:20:11.387631+0000","last_fullsized":"2023-10-09T04:20:11.387631+0000","mapping_epoch":348,"log_start":"314'1000","ondisk_log_start":"314'1000","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6320,"log_dups_size":0,"ondisk_log_size":6320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:45.805086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332136448,"num_objects":5068,"num_object_clones":0,"num_object_copies":15204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8000,"num_objects_unfound":0,"num_objects_dirty":5068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7320,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2377,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":5068}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":336,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020374+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:20:05.020374+0000","last_peered":"2023-10-09T04:20:05.020374+0000","last_clean":"2023-10-09T04:20:05.020374+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:20:05.020374+0000","last_undegraded":"2023-10-09T04:20:05.020374+0000","last_fullsized":"2023-10-09T04:20:05.020374+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":747,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025770+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:20:05.025770+0000","last_peered":"2023-10-09T04:20:05.025770+0000","last_clean":"2023-10-09T04:20:05.025770+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:20:05.025770+0000","last_undegraded":"2023-10-09T04:20:05.025770+0000","last_fullsized":"2023-10-09T04:20:05.025770+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":279,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020052+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:20:05.020052+0000","last_peered":"2023-10-09T04:20:05.020052+0000","last_clean":"2023-10-09T04:20:05.020052+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:20:05.020052+0000","last_undegraded":"2023-10-09T04:20:05.020052+0000","last_fullsized":"2023-10-09T04:20:05.020052+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016822+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:20:05.016822+0000","last_peered":"2023-10-09T04:20:05.016822+0000","last_clean":"2023-10-09T04:20:05.016822+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:20:05.016822+0000","last_undegraded":"2023-10-09T04:20:05.016822+0000","last_fullsized":"2023-10-09T04:20:05.016822+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"354'7304","reported_seq":10940,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.223982+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:20:11.223982+0000","last_peered":"2023-10-09T04:20:11.223982+0000","last_clean":"2023-10-09T04:20:11.223982+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:20:11.223982+0000","last_undegraded":"2023-10-09T04:20:11.223982+0000","last_fullsized":"2023-10-09T04:20:11.223982+0000","mapping_epoch":323,"log_start":"314'1000","ondisk_log_start":"314'1000","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6304,"log_dups_size":0,"ondisk_log_size":6304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329646080,"num_objects":5030,"num_object_clones":0,"num_object_copies":15090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7304,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":494,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.010254+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:20:04.010254+0000","last_peered":"2023-10-09T04:20:04.010254+0000","last_clean":"2023-10-09T04:20:04.010254+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:20:04.010254+0000","last_undegraded":"2023-10-09T04:20:04.010254+0000","last_fullsized":"2023-10-09T04:20:04.010254+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":333,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.024903+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:20:05.024903+0000","last_peered":"2023-10-09T04:20:05.024903+0000","last_clean":"2023-10-09T04:20:05.024903+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:20:05.024903+0000","last_undegraded":"2023-10-09T04:20:05.024903+0000","last_fullsized":"2023-10-09T04:20:05.024903+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":349,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778058+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:20:09.778058+0000","last_peered":"2023-10-09T04:20:09.778058+0000","last_clean":"2023-10-09T04:20:09.778058+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:20:09.778058+0000","last_undegraded":"2023-10-09T04:20:09.778058+0000","last_fullsized":"2023-10-09T04:20:09.778058+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":346,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778089+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:20:09.778089+0000","last_peered":"2023-10-09T04:20:09.778089+0000","last_clean":"2023-10-09T04:20:09.778089+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:20:09.778089+0000","last_undegraded":"2023-10-09T04:20:09.778089+0000","last_fullsized":"2023-10-09T04:20:09.778089+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"353'7340","reported_seq":7395,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.710884+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:20:10.710884+0000","last_peered":"2023-10-09T04:20:10.710884+0000","last_clean":"2023-10-09T04:20:10.710884+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:20:10.710884+0000","last_undegraded":"2023-10-09T04:20:10.710884+0000","last_fullsized":"2023-10-09T04:20:10.710884+0000","mapping_epoch":306,"log_start":"319'2000","ondisk_log_start":"319'2000","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336461824,"num_objects":5134,"num_object_clones":0,"num_object_copies":15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7340,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":506,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016024+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:20:05.016024+0000","last_peered":"2023-10-09T04:20:05.016024+0000","last_clean":"2023-10-09T04:20:05.016024+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:20:05.016024+0000","last_undegraded":"2023-10-09T04:20:05.016024+0000","last_fullsized":"2023-10-09T04:20:05.016024+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":336,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025973+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:20:05.025973+0000","last_peered":"2023-10-09T04:20:05.025973+0000","last_clean":"2023-10-09T04:20:05.025973+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:20:05.025973+0000","last_undegraded":"2023-10-09T04:20:05.025973+0000","last_fullsized":"2023-10-09T04:20:05.025973+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":325,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.023067+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:20:04.023067+0000","last_peered":"2023-10-09T04:20:04.023067+0000","last_clean":"2023-10-09T04:20:04.023067+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:20:04.023067+0000","last_undegraded":"2023-10-09T04:20:04.023067+0000","last_fullsized":"2023-10-09T04:20:04.023067+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020303+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:20:05.020303+0000","last_peered":"2023-10-09T04:20:05.020303+0000","last_clean":"2023-10-09T04:20:05.020303+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:20:05.020303+0000","last_undegraded":"2023-10-09T04:20:05.020303+0000","last_fullsized":"2023-10-09T04:20:05.020303+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"353'7261","reported_seq":18648,"reported_epoch":353,"state":"active+clean+remapped","last_fresh":"2023-10-09T04:20:10.810230+0000","last_change":"2023-10-09T04:20:10.037558+0000","last_active":"2023-10-09T04:20:10.810230+0000","last_peered":"2023-10-09T04:20:10.810230+0000","last_clean":"2023-10-09T04:20:10.810230+0000","last_became_active":"2023-10-09T04:19:28.825416+0000","last_became_peered":"2023-10-09T04:19:28.825416+0000","last_unstale":"2023-10-09T04:20:10.810230+0000","last_undegraded":"2023-10-09T04:20:10.810230+0000","last_fullsized":"2023-10-09T04:20:10.810230+0000","mapping_epoch":333,"log_start":"318'1900","ondisk_log_start":"318'1900","created":306,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5361,"log_dups_size":0,"ondisk_log_size":5361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:15.224579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330956824,"num_objects":5051,"num_object_clones":0,"num_object_copies":15153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5051,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7261,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5,4],"avail_no_missing":["7","1","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":5051}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":497,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016544+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:20:05.016544+0000","last_peered":"2023-10-09T04:20:05.016544+0000","last_clean":"2023-10-09T04:20:05.016544+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:20:05.016544+0000","last_undegraded":"2023-10-09T04:20:05.016544+0000","last_fullsized":"2023-10-09T04:20:05.016544+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":285,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025944+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:20:05.025944+0000","last_peered":"2023-10-09T04:20:05.025944+0000","last_clean":"2023-10-09T04:20:05.025944+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:20:05.025944+0000","last_undegraded":"2023-10-09T04:20:05.025944+0000","last_fullsized":"2023-10-09T04:20:05.025944+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":319,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.023152+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:20:04.023152+0000","last_peered":"2023-10-09T04:20:04.023152+0000","last_clean":"2023-10-09T04:20:04.023152+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:20:04.023152+0000","last_undegraded":"2023-10-09T04:20:04.023152+0000","last_fullsized":"2023-10-09T04:20:04.023152+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":303,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.026156+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:20:05.026156+0000","last_peered":"2023-10-09T04:20:05.026156+0000","last_clean":"2023-10-09T04:20:05.026156+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:20:05.026156+0000","last_undegraded":"2023-10-09T04:20:05.026156+0000","last_fullsized":"2023-10-09T04:20:05.026156+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"353'7342","reported_seq":7407,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:10.386641+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:20:10.386641+0000","last_peered":"2023-10-09T04:20:10.386641+0000","last_clean":"2023-10-09T04:20:10.386641+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:20:10.386641+0000","last_undegraded":"2023-10-09T04:20:10.386641+0000","last_fullsized":"2023-10-09T04:20:10.386641+0000","mapping_epoch":310,"log_start":"314'1700","ondisk_log_start":"314'1700","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341311488,"num_objects":5208,"num_object_clones":0,"num_object_copies":15624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7342,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":490,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.036728+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:20:05.036728+0000","last_peered":"2023-10-09T04:20:05.036728+0000","last_clean":"2023-10-09T04:20:05.036728+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:20:05.036728+0000","last_undegraded":"2023-10-09T04:20:05.036728+0000","last_fullsized":"2023-10-09T04:20:05.036728+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":183,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.014839+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:20:04.014839+0000","last_peered":"2023-10-09T04:20:04.014839+0000","last_clean":"2023-10-09T04:20:04.014839+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:20:04.014839+0000","last_undegraded":"2023-10-09T04:20:04.014839+0000","last_fullsized":"2023-10-09T04:20:04.014839+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":359,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778214+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:20:09.778214+0000","last_peered":"2023-10-09T04:20:09.778214+0000","last_clean":"2023-10-09T04:20:09.778214+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:20:09.778214+0000","last_undegraded":"2023-10-09T04:20:09.778214+0000","last_fullsized":"2023-10-09T04:20:09.778214+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015164+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:20:05.015164+0000","last_peered":"2023-10-09T04:20:05.015164+0000","last_clean":"2023-10-09T04:20:05.015164+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:20:05.015164+0000","last_undegraded":"2023-10-09T04:20:05.015164+0000","last_fullsized":"2023-10-09T04:20:05.015164+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":512,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.777792+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:20:09.777792+0000","last_peered":"2023-10-09T04:20:09.777792+0000","last_clean":"2023-10-09T04:20:09.777792+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:20:09.777792+0000","last_undegraded":"2023-10-09T04:20:09.777792+0000","last_fullsized":"2023-10-09T04:20:09.777792+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":328,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778564+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:20:09.778564+0000","last_peered":"2023-10-09T04:20:09.778564+0000","last_clean":"2023-10-09T04:20:09.778564+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:20:09.778564+0000","last_undegraded":"2023-10-09T04:20:09.778564+0000","last_fullsized":"2023-10-09T04:20:09.778564+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":186,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013753+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:20:04.013753+0000","last_peered":"2023-10-09T04:20:04.013753+0000","last_clean":"2023-10-09T04:20:04.013753+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:20:04.013753+0000","last_undegraded":"2023-10-09T04:20:04.013753+0000","last_fullsized":"2023-10-09T04:20:04.013753+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":740,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.025356+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:20:05.025356+0000","last_peered":"2023-10-09T04:20:05.025356+0000","last_clean":"2023-10-09T04:20:05.025356+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:20:05.025356+0000","last_undegraded":"2023-10-09T04:20:05.025356+0000","last_fullsized":"2023-10-09T04:20:05.025356+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":342,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020166+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:20:05.020166+0000","last_peered":"2023-10-09T04:20:05.020166+0000","last_clean":"2023-10-09T04:20:05.020166+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:20:05.020166+0000","last_undegraded":"2023-10-09T04:20:05.020166+0000","last_fullsized":"2023-10-09T04:20:05.020166+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":117,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.016169+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:20:05.016169+0000","last_peered":"2023-10-09T04:20:05.016169+0000","last_clean":"2023-10-09T04:20:05.016169+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:20:05.016169+0000","last_undegraded":"2023-10-09T04:20:05.016169+0000","last_fullsized":"2023-10-09T04:20:05.016169+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":339,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.020388+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:20:05.020388+0000","last_peered":"2023-10-09T04:20:05.020388+0000","last_clean":"2023-10-09T04:20:05.020388+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:20:05.020388+0000","last_undegraded":"2023-10-09T04:20:05.020388+0000","last_fullsized":"2023-10-09T04:20:05.020388+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":117,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.015212+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:20:05.015212+0000","last_peered":"2023-10-09T04:20:05.015212+0000","last_clean":"2023-10-09T04:20:05.015212+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:20:05.015212+0000","last_undegraded":"2023-10-09T04:20:05.015212+0000","last_fullsized":"2023-10-09T04:20:05.015212+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":775,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.024993+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:20:05.024993+0000","last_peered":"2023-10-09T04:20:05.024993+0000","last_clean":"2023-10-09T04:20:05.024993+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:20:05.024993+0000","last_undegraded":"2023-10-09T04:20:05.024993+0000","last_fullsized":"2023-10-09T04:20:05.024993+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":196,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.018748+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:20:04.018748+0000","last_peered":"2023-10-09T04:20:04.018748+0000","last_clean":"2023-10-09T04:20:04.018748+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:20:04.018748+0000","last_undegraded":"2023-10-09T04:20:04.018748+0000","last_fullsized":"2023-10-09T04:20:04.018748+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":186,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T04:20:04.013625+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:20:04.013625+0000","last_peered":"2023-10-09T04:20:04.013625+0000","last_clean":"2023-10-09T04:20:04.013625+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:20:04.013625+0000","last_undegraded":"2023-10-09T04:20:04.013625+0000","last_fullsized":"2023-10-09T04:20:04.013625+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":493,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.043020+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:20:05.043020+0000","last_peered":"2023-10-09T04:20:05.043020+0000","last_clean":"2023-10-09T04:20:05.043020+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:20:05.043020+0000","last_undegraded":"2023-10-09T04:20:05.043020+0000","last_fullsized":"2023-10-09T04:20:05.043020+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":347,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-10-09T04:20:09.778220+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:20:09.778220+0000","last_peered":"2023-10-09T04:20:09.778220+0000","last_clean":"2023-10-09T04:20:09.778220+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:20:09.778220+0000","last_undegraded":"2023-10-09T04:20:09.778220+0000","last_fullsized":"2023-10-09T04:20:09.778220+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":107,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-10-09T04:20:05.014831+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:20:05.014831+0000","last_peered":"2023-10-09T04:20:05.014831+0000","last_clean":"2023-10-09T04:20:05.014831+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:20:05.014831+0000","last_undegraded":"2023-10-09T04:20:05.014831+0000","last_fullsized":"2023-10-09T04:20:05.014831+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5322309656,"num_objects":81213,"num_object_clones":0,"num_object_copies":243639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":18084,"num_objects_unfound":0,"num_objects_dirty":81213,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":116235,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13542,"num_bytes_recovered":887423000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16502046720,"data_stored":16502030432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91814,"ondisk_log_size":91814,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201612,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4174864,"kb_used_data":3580936,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":89570288,"statfs":{"total":95995035648,"available":91719974912,"internally_reserved":0,"allocated":3666878464,"data_stored":3665887080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.4089999999999998,"15min":2.9249999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":16.440999999999999,"5min":16.440999999999999,"15min":126.752},"last":5.8220000000000001},{"interface":"front","average":{"1min":3.1869999999999998,"5min":2.2189999999999999,"15min":2.8820000000000001},"min":{"1min":0.27300000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":16.462,"5min":16.462,"15min":128.88399999999999},"last":3.5539999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2430000000000003,"5min":4.3739999999999997,"15min":4.21},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":29.792000000000002,"5min":103.324,"15min":245.28700000000001},"last":6.5890000000000004},{"interface":"front","average":{"1min":4.9909999999999997,"5min":4.6210000000000004,"15min":4.1660000000000004},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":29.268000000000001,"5min":104.248,"15min":244.626},"last":3.6230000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.7849999999999999,"15min":2.0979999999999999},"min":{"1min":0.30499999999999999,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.194,"5min":14.076000000000001,"15min":76.774000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.556,"15min":2.794},"min":{"1min":0.35799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":6.1749999999999998,"5min":15.597,"15min":99.372},"last":4.0499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":2.6419999999999999,"15min":2.633},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":3.7149999999999999,"5min":85.575000000000003,"15min":99.134},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.1080000000000001,"15min":2.8769999999999998},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":6.1459999999999999,"5min":85.528000000000006,"15min":99.117999999999995},"last":5.9809999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.86899999999999999,"15min":0.82799999999999996},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.8529999999999998,"5min":8.9359999999999999,"15min":16.195},"last":2.9449999999999998},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6160000000000001,"15min":1.1120000000000001},"min":{"1min":0.27700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"last":4.4500000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.5680000000000001,"15min":1.321},"min":{"1min":0.29299999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":6.9390000000000001,"5min":120.066,"15min":120.066},"last":6.5490000000000004},{"interface":"front","average":{"1min":0.85299999999999998,"5min":2.2530000000000001,"15min":1.2789999999999999},"min":{"1min":0.34399999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8899999999999997,"5min":120.09699999999999,"15min":120.09699999999999},"last":6.8360000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.4590000000000001,"15min":1.0249999999999999},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":8.4550000000000001,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":5.7850000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.96999999999999997,"15min":0.88300000000000001},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.605,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":4.5590000000000002}]}]},{"osd":6,"up_from":0,"seq":201863463249,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089557,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3879516,"kb_used_data":3317524,"kb_used_omap":9,"kb_used_meta":561974,"kb_avail":89865636,"statfs":{"total":95995035648,"available":92022411264,"internally_reserved":0,"allocated":3397144576,"data_stored":3395668112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":575462377},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":3.8759999999999999,"15min":4.0629999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":19.617000000000001,"5min":98.257999999999996,"15min":98.257999999999996},"last":3.1379999999999999},{"interface":"front","average":{"1min":2.1949999999999998,"5min":6.3319999999999999,"15min":5.2000000000000002},"min":{"1min":0.27100000000000002,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":9.9960000000000004,"5min":154.654,"15min":154.654},"last":5.8049999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":2.1970000000000001,"15min":2.6019999999999999},"min":{"1min":0.26600000000000001,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":19.564,"5min":19.564,"15min":121.54600000000001},"last":2.3140000000000001},{"interface":"front","average":{"1min":2,"5min":2.476,"15min":2.9239999999999999},"min":{"1min":0.22500000000000001,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":12.382999999999999,"5min":16.114999999999998,"15min":99.899000000000001},"last":9.5169999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.8660000000000001,"15min":3.7770000000000001},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":19.640000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":2.3490000000000002},{"interface":"front","average":{"1min":2.4609999999999999,"5min":3.8170000000000002,"15min":4.6970000000000001},"min":{"1min":0.307,"5min":0.251,"15min":0.251},"max":{"1min":12.353999999999999,"5min":68.563999999999993,"15min":138.49799999999999},"last":5.9880000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.2290000000000001,"15min":2.3370000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":7.5250000000000004,"5min":16.27,"15min":99.989999999999995},"last":5.8470000000000004},{"interface":"front","average":{"1min":1.458,"5min":1.9530000000000001,"15min":2.1829999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":5.282,"5min":13.428000000000001,"15min":100.249},"last":5.0469999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":3.899,"5min":3.2280000000000002,"15min":2.6539999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":25.696999999999999,"5min":38.848999999999997,"15min":99.933999999999997},"last":10.222},{"interface":"front","average":{"1min":2.8700000000000001,"5min":3.145,"15min":2.6909999999999998},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":19.977,"5min":39.593000000000004,"15min":100.34099999999999},"last":21.893999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.2120000000000002,"15min":2.5},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":5.3579999999999997,"5min":13.377000000000001,"15min":100.309},"last":8.3529999999999998},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.9570000000000001,"15min":2.1749999999999998},"min":{"1min":0.41399999999999998,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":100.143},"last":2.399}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.7050000000000001,"15min":2.6419999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":5.7430000000000003,"5min":17.460000000000001,"15min":100.214},"last":9.6869999999999994},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.3359999999999999,"15min":2.319},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":19.593,"5min":19.593,"15min":100.113},"last":2.3740000000000001}]}]},{"osd":0,"up_from":84,"seq":360777253084,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2118076,"kb_used_data":1608436,"kb_used_omap":28,"kb_used_meta":509603,"kb_avail":91627076,"statfs":{"total":95995035648,"available":93826125824,"internally_reserved":0,"allocated":1647038464,"data_stored":1645486488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":521834320},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":8.9990000000000006,"15min":5.2709999999999999},"min":{"1min":0.20200000000000001,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":6.1609999999999996,"5min":209.215,"15min":209.215},"last":3.198},{"interface":"front","average":{"1min":3.0499999999999998,"5min":6.8630000000000004,"15min":5.032},"min":{"1min":0.29499999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.547000000000001,"5min":120.934,"15min":120.934},"last":8.7799999999999994}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.3359999999999999,"15min":3.141},"min":{"1min":0.32900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.025,"5min":17.466999999999999,"15min":118.54000000000001},"last":4.1280000000000001},{"interface":"front","average":{"1min":2.0609999999999999,"5min":3.2429999999999999,"15min":3.536},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.20300000000000001},"max":{"1min":6.7460000000000004,"5min":20.262,"15min":118.727},"last":4.8490000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":3.1400000000000001,"15min":5.2759999999999998},"min":{"1min":0.25700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":9.1669999999999998,"5min":99.709000000000003,"15min":156.488},"last":9.1669999999999998},{"interface":"front","average":{"1min":2.3919999999999999,"5min":3.1720000000000002,"15min":5.3620000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":10.577,"5min":99.560000000000002,"15min":156.50700000000001},"last":4.7759999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":4.899,"5min":2.9969999999999999,"15min":2.7250000000000001},"min":{"1min":0.35399999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":49.840000000000003,"5min":63.375999999999998,"15min":118.633},"last":4},{"interface":"front","average":{"1min":5.0609999999999999,"5min":3.3290000000000002,"15min":3.0670000000000002},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":49.607999999999997,"5min":49.607999999999997,"15min":118.913},"last":4.9279999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":5.4800000000000004,"15min":4.2960000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.0570000000000004,"5min":156.96100000000001,"15min":182.68899999999999},"last":4.899},{"interface":"front","average":{"1min":2.0870000000000002,"5min":4.5540000000000003,"15min":3.7749999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.024,"5min":156.97300000000001,"15min":182.66300000000001},"last":3.2519999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.528,"15min":2.996},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":7.0709999999999997,"5min":63.468000000000004,"15min":137.36099999999999},"last":3.2839999999999998},{"interface":"front","average":{"1min":2.206,"5min":2.5859999999999999,"15min":3.0329999999999999},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":10.113,"5min":63.447000000000003,"15min":136.88999999999999},"last":4.0359999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.6419999999999999,"15min":3.677},"min":{"1min":0.40000000000000002,"5min":0.375,"15min":0.32100000000000001},"max":{"1min":9.1699999999999999,"5min":63.509,"15min":185.024},"last":3.3199999999999998},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.7770000000000001,"15min":3.3479999999999999},"min":{"1min":0.44700000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":10.077,"5min":63.481999999999999,"15min":184.98599999999999},"last":4.069}]}]},{"osd":2,"up_from":94,"seq":403726926031,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2220340,"kb_used_data":1658028,"kb_used_omap":81,"kb_used_meta":562158,"kb_avail":91524812,"statfs":{"total":95995035648,"available":93721407488,"internally_reserved":0,"allocated":1697820672,"data_stored":1696325242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":575650010},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":6.3810000000000002,"15min":3.875},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":5.0780000000000003,"5min":81.385999999999996,"15min":81.385999999999996},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.8,"5min":5.7300000000000004,"15min":3.899},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":10.457000000000001,"5min":79.453999999999994,"15min":79.453999999999994},"last":1.298}]},{"osd":1,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":11.619,"5min":6.9870000000000001,"15min":4.391},"min":{"1min":0.27600000000000002,"5min":0.191,"15min":0.191},"max":{"1min":158.13399999999999,"5min":158.13399999999999,"15min":158.13399999999999},"last":1.337},{"interface":"front","average":{"1min":11.401999999999999,"5min":7.7629999999999999,"15min":4.6639999999999997},"min":{"1min":0.253,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":161.12,"5min":161.12,"15min":161.12},"last":0.35899999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.4700000000000002,"15min":2.8170000000000002},"min":{"1min":0.28000000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.409000000000001,"5min":58.997,"15min":58.997},"last":2.1230000000000002},{"interface":"front","average":{"1min":2.1499999999999999,"5min":3.5030000000000001,"15min":3.0270000000000001},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.488,"5min":66.135000000000005,"15min":66.135000000000005},"last":1.6970000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":2.6139999999999999,"15min":2.1309999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":6.0700000000000003,"5min":58.798000000000002,"15min":58.798000000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.7290000000000001,"15min":2.0190000000000001},"min":{"1min":0.32000000000000001,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":3.1659999999999999,"5min":56.616,"15min":56.616},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.8620000000000001,"15min":2.327},"min":{"1min":0.28299999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.241,"5min":56.524000000000001,"15min":56.524000000000001},"last":1.367},{"interface":"front","average":{"1min":3.625,"5min":3.157,"15min":2.456},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":51.472000000000001,"5min":58.854999999999997,"15min":58.854999999999997},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":2.9729999999999999,"15min":2.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.8650000000000002,"5min":56.561,"15min":114.14},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.5579999999999998,"15min":2.5859999999999999},"min":{"1min":0.32100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.218,"5min":56.460999999999999,"15min":114.123},"last":5.2110000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":3.0350000000000001,"15min":2.7290000000000001},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.8479999999999999,"5min":56.591000000000001,"15min":117.32599999999999},"last":3.431},{"interface":"front","average":{"1min":1.6950000000000001,"5min":2.7429999999999999,"15min":2.7599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.189,"5min":56.497999999999998,"15min":117.28700000000001},"last":5.1660000000000004}]}]},{"osd":3,"up_from":99,"seq":425201762505,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2426516,"kb_used_data":1704652,"kb_used_omap":7,"kb_used_meta":721848,"kb_avail":91318636,"statfs":{"total":95995035648,"available":93510283264,"internally_reserved":0,"allocated":1745563648,"data_stored":1744100800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":739172968},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":3.3730000000000002,"15min":5.4059999999999997},"min":{"1min":0.214,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":6.4169999999999998,"5min":78.771000000000001,"15min":125.765},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.899,"5min":4.2080000000000002,"15min":5.8280000000000003},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.119999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":8.7490000000000006}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":5.8979999999999997,"15min":5.2619999999999996},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":8.3230000000000004,"5min":103.143,"15min":186.21299999999999},"last":0.22500000000000001},{"interface":"front","average":{"1min":1.6060000000000001,"5min":5.7789999999999999,"15min":5.0309999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.193},"max":{"1min":5.9219999999999997,"5min":105.19,"15min":191.78},"last":2.073}]},{"osd":2,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.1660000000000004,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.8760000000000003,"5min":127.599,"15min":127.599},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.359,"5min":6.0670000000000002,"15min":5.0119999999999996},"min":{"1min":0.29799999999999999,"5min":0.221,"15min":0.214},"max":{"1min":10.143000000000001,"5min":129.98599999999999,"15min":129.98599999999999},"last":5.5540000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":2.274,"15min":2.7240000000000002},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.013999999999999,"5min":27.539999999999999,"15min":106.85899999999999},"last":5.625},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.2919999999999998,"15min":2.1760000000000002},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.738,"5min":26.663,"15min":37.442},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.6280000000000001,"15min":2.2010000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":3.7749999999999999,"5min":17.901,"15min":17.901},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.083,"5min":1.766,"15min":1.887},"min":{"1min":0.375,"5min":0.248,"15min":0.248},"max":{"1min":4.6340000000000003,"5min":18.068000000000001,"15min":18.068000000000001},"last":0.50700000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.7509999999999999,"15min":2.2610000000000001},"min":{"1min":0.378,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4589999999999996,"5min":14.567,"15min":106.036},"last":2.9060000000000001},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.48,"15min":2.6509999999999998},"min":{"1min":0.42699999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.071,"5min":14.529,"15min":106.741},"last":5.6520000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":2.6720000000000002,"15min":2.786},"min":{"1min":0.39000000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":28.184000000000001,"5min":28.184000000000001,"15min":106.82599999999999},"last":5.593},{"interface":"front","average":{"1min":3.1480000000000001,"5min":2.827,"15min":2.7490000000000001},"min":{"1min":0.44800000000000001,"5min":0.40000000000000002,"15min":0.35999999999999999},"max":{"1min":26.648,"5min":26.648,"15min":106.084},"last":1.1699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986521,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1940544,"kb_used_data":1635064,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":91804608,"statfs":{"total":95995035648,"available":94007918592,"internally_reserved":0,"allocated":1674305536,"data_stored":1673329567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":10.717000000000001,"5min":10.526,"15min":5.4820000000000002},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":132.631,"5min":154.62100000000001,"15min":154.62100000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":10.893000000000001,"5min":10.782,"15min":5.5069999999999997},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":132.59100000000001,"5min":154.37299999999999,"15min":154.37299999999999},"last":0.23799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":7.8129999999999997,"5min":9.4969999999999999,"15min":5.1180000000000003},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":69.706000000000003,"5min":135.25399999999999,"15min":135.25399999999999},"last":0.88},{"interface":"front","average":{"1min":7.3220000000000001,"5min":9.1270000000000007,"15min":5.0060000000000002},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":69.561000000000007,"5min":134.44300000000001,"15min":134.44300000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.679,"15min":2.0720000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.245},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":123.598},"last":0.308},{"interface":"front","average":{"1min":1.708,"5min":1.546,"15min":1.986},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":6.21,"5min":8.4800000000000004,"15min":123.619},"last":0.69799999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.052,"15min":2.29},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.0069999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.165,"15min":2.4849999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.7579999999999991,"5min":8.9169999999999998,"15min":127.90000000000001},"last":0.34100000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":3.6099999999999999,"15min":1.6279999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":13.599,"5min":111.809,"15min":111.809},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.532,"5min":3.7200000000000002,"15min":1.7929999999999999},"min":{"1min":0.316,"5min":0.252,"15min":0.25},"max":{"1min":9.1530000000000005,"5min":112.002,"15min":112.002},"last":0.84399999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.395,"15min":1.3680000000000001},"min":{"1min":0.35699999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":6.9470000000000001,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.877,"5min":2.2130000000000001,"15min":1.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9320000000000004,"5min":78.421999999999997,"15min":78.421999999999997},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.3049999999999999,"15min":0.95299999999999996},"min":{"1min":0.31900000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.8010000000000002,"5min":34.039000000000001,"15min":34.039000000000001},"last":4.6719999999999997},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.5109999999999999,"15min":1.0680000000000001},"min":{"1min":0.375,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.384,"5min":33.433999999999997,"15min":33.433999999999997},"last":1.444}]}]},{"osd":5,"up_from":39,"seq":167503724885,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2988988,"kb_used_data":2691700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":90756164,"statfs":{"total":95995035648,"available":92934311936,"internally_reserved":0,"allocated":2756300800,"data_stored":2755325337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":4.984,"15min":5.0759999999999996},"min":{"1min":0.223,"5min":0.193,"15min":0.193},"max":{"1min":13.675000000000001,"5min":118.98399999999999,"15min":151.23599999999999},"last":2.1230000000000002},{"interface":"front","average":{"1min":2.4079999999999999,"5min":5.3380000000000001,"15min":5.4470000000000001},"min":{"1min":0.23499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":13.885,"5min":118.846,"15min":151.566},"last":3.6890000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":5.242,"15min":4.0270000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":29.632999999999999,"5min":111.194,"15min":111.194},"last":6.2779999999999996},{"interface":"front","average":{"1min":4.7249999999999996,"5min":5.8289999999999997,"15min":4.4180000000000001},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":29.672000000000001,"5min":112.226,"15min":112.226},"last":4.3410000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.444,"15min":3.1299999999999999},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":8.8599999999999994,"5min":11.324,"15min":106.508},"last":3.5139999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.718,"15min":2.8879999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":3.8199999999999998,"5min":8.5280000000000005,"15min":106.462},"last":3.3809999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.0449999999999999,"15min":2.5939999999999999},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.555999999999999,"5min":11.555999999999999,"15min":91.834999999999994},"last":0.73799999999999999},{"interface":"front","average":{"1min":2.3260000000000001,"5min":2.2389999999999999,"15min":2.6890000000000001},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3219999999999992,"5min":12.122999999999999,"15min":91.802000000000007},"last":6.1319999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.242,"15min":0.93300000000000005},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.246},"max":{"1min":9.2579999999999991,"5min":10.162000000000001,"15min":10.162000000000001},"last":3.1160000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.1200000000000001,"15min":0.92900000000000005},"min":{"1min":0.30199999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":3.77,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.61599999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.27,"15min":1.3779999999999999},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":9.4090000000000007,"5min":12.068,"15min":123.239},"last":6.2519999999999998},{"interface":"front","average":{"1min":1.615,"5min":1.45,"15min":1.4670000000000001},"min":{"1min":0.313,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":9.7720000000000002,"15min":123.014},"last":3.1549999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.202,"15min":0.995},"min":{"1min":0.33100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.3289999999999997,"5min":12.096,"15min":12.096},"last":6.1719999999999997},{"interface":"front","average":{"1min":1.24,"5min":1.446,"15min":1.095},"min":{"1min":0.36699999999999999,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":9.3019999999999996,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":3.181}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1641742336,"data_stored":1641742336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3373010944,"data_stored":3373006872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1682309120,"data_stored":1682309120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1739063296,"data_stored":1739063296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1666781184,"data_stored":1666777112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2747994112,"data_stored":2747990040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3650686976,"data_stored":3650682904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:20:14.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:20:14.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:14.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:14.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:14.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:14.555 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:14.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:20:14.740 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:14.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:14.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:14.875 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:14.931 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:15.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:15.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:15.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:15.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:15.268 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:15.293 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:15.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:15.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:15.470 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:20:15.496 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:15.569 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:16.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:16.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:16.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:16.358 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:16.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:20:17.083 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:17.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:17.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:16 smithi181 ceph-mon[196979]: pgmap v951: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+remapped, 115 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 16827/241563 objects misplaced (6.966%); 16 MiB/s, 256 objects/s recovering 2023-10-09T04:20:17.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:16 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:17.164 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:16 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:17.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:17.245 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:17.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:17.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:17.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:16 smithi163 ceph-mon[221067]: pgmap v951: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+remapped, 115 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 16827/241563 objects misplaced (6.966%); 16 MiB/s, 256 objects/s recovering 2023-10-09T04:20:17.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:17.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:16 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:16 smithi163 ceph-mon[213841]: pgmap v951: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+clean+remapped, 115 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 16827/241563 objects misplaced (6.966%); 16 MiB/s, 256 objects/s recovering 2023-10-09T04:20:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:16 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:17.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:17.455 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:17.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:17.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:17.661 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:17.730 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:17.760 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:20:17.760 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:20:17.760 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:20:17.760 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:20:17.761 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:20:17.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:17.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:18.019 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:18.083 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:18.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:18.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:18.428 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:18.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:18.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3707426027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:20:18.542 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:18 smithi163 ceph-mon[213841]: pgmap v952: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 16243/233970 objects misplaced (6.942%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:20:18.572 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:18.572 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:20:18.735 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:18.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:18.761 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:20:18.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3707426027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:20:18.812 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:18 smithi181 ceph-mon[196979]: pgmap v952: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 16243/233970 objects misplaced (6.942%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:20:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3707426027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:20:18.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:18 smithi163 ceph-mon[221067]: pgmap v952: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 16243/233970 objects misplaced (6.942%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:20:18.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:18.941 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:19.103 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:19.104 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1183,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":356,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":7,"osd_in_since":1696825107,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":116},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":119,"num_pools":5,"num_objects":77990,"data_bytes":5110186169,"bytes_used":20015792128,"bytes_avail":651949457408,"bytes_total":671965249536,"misplaced_objects":16243,"misplaced_total":233970,"misplaced_ratio":0.06942343035431893,"recovering_objects_per_sec":200,"recovering_bytes_per_sec":13144938,"recovering_keys_per_sec":0,"num_objects_recovered":1605,"num_bytes_recovered":105185280,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":736},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T04:19:35.820638+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"cbdf697a-aea6-4b77-8dba-e1904d62d8bd":{"message":"Global Recovery Event (40s)\n [===========================.] (remaining: 1s)","progress":0.97478991746902466,"add_to_ceph_s":true}}} 2023-10-09T04:20:19.265 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:20:19.700 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:19.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:19.701 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: failed to encode map e356 with expected crc 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: failed to encode map e356 with expected crc 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3707426027' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: osdmap e356: 8 total, 8 up, 7 in 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1887913684' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: failed to encode map e356 with expected crc 2023-10-09T04:20:19.702 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: failed to encode map e356 with expected crc 2023-10-09T04:20:19.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3707426027' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:20:19.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: osdmap e356: 8 total, 8 up, 7 in 2023-10-09T04:20:19.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1887913684' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:20:19.703 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:19.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:19.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: failed to encode map e356 with expected crc 2023-10-09T04:20:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: failed to encode map e356 with expected crc 2023-10-09T04:20:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3707426027' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:20:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: osdmap e356: 8 total, 8 up, 7 in 2023-10-09T04:20:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1887913684' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:20:19.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:19.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:19.879 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:20.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:20.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:20.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:20.415 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:20.686 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:20 smithi163 ceph-mon[221067]: pgmap v954: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 15538/231708 objects misplaced (6.706%); 13 MiB/s, 203 objects/s recovering 2023-10-09T04:20:20.687 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:20 smithi163 ceph-mon[213841]: pgmap v954: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 15538/231708 objects misplaced (6.706%); 13 MiB/s, 203 objects/s recovering 2023-10-09T04:20:20.687 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:20:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:20:20] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:20:20.945 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:20:20.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:20:20.997 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:21.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:21.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:21.009 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:20 smithi181 ceph-mon[196979]: pgmap v954: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 15538/231708 objects misplaced (6.706%); 13 MiB/s, 203 objects/s recovering 2023-10-09T04:20:21.103 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:21.163 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:21.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:21.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:21.287 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:20:21.287 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:20:21.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:20:21.287 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35115.0:80484 7.15 7:ade26b62:::benchmark_data_smithi163_346082_object80483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 584.11692999599995, 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.249257176, 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35115", 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1510316755", 2023-10-09T04:20:21.288 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 80484 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165251+0000", 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165250+0000", 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165302+0000", 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2011000000000001e-05 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165315+0000", 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2316e-05 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165321+0000", 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0920000000000003e-06 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165361+0000", 2023-10-09T04:20:21.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9743000000000002e-05 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165448+0000", 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7162000000000003e-05 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.165538+0000", 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0328999999999995e-05 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.166751+0000", 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012134100000000001 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.167144+0000", 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000392896 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310638+0000", 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14349324699999999 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:37.310649+0000", 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1039999999999999e-05 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326699+0000", 2023-10-09T04:20:21.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.016050111 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:38.326711+0000", 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2063e-05 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414399+0000", 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0876884 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:10:39.414508+0000", 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109128 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28475 9.1 9:8009e758:::benchmark_data_smithi163_464650_object28474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:20:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 113.480999584, 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.096956874, 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28475 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801180+0000", 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801209+0000", 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8646e-05 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801226+0000", 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.715e-05 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801234+0000", 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3459999999999996e-06 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801291+0000", 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7438999999999997e-05 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801409+0000", 2023-10-09T04:20:21.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011781399999999999 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801670+0000", 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026062299999999997 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809884+0000", 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082140670000000002 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809890+0000", 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6460000000000004e-06 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812682+0000", 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002791615 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812694+0000", 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.171e-05 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.884929+0000", 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0722356550000001 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.885962+0000", 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010329860000000001 2023-10-09T04:20:21.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898005+0000", 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012042361999999999 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898138+0000", 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013387100000000001 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28492 9.1 9:80d8c0c4:::benchmark_data_smithi163_464650_object28491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 113.370004568, 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9871532940000001, 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.305 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28492 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:20:21.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912175+0000", 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912211+0000", 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6192000000000002e-05 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912224+0000", 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2965999999999999e-05 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912230+0000", 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6540000000000004e-06 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912283+0000", 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3115999999999998e-05 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912289+0000", 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2700000000000001e-06 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812717+0000", 2023-10-09T04:20:21.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90042778800000001 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812726+0000", 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5189999999999992e-06 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886022+0000", 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.073296604 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886114+0000", 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1397999999999995e-05 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886285+0000", 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171807 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891737+0000", 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005451661 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898177+0000", 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064403500000000001 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898550+0000", 2023-10-09T04:20:21.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037258199999999999 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898598+0000", 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8214000000000003e-05 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899330+0000", 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000731593 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.313 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28496 9.1 9:834685fc:::benchmark_data_smithi163_464650_object28495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 113.36235381100001, 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9805368130000001, 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28496 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:20:21.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919880+0000", 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3053000000000002e-05 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919893+0000", 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3439e-05 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919899+0000", 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.161e-06 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919957+0000", 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8118999999999997e-05 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919969+0000", 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2065000000000001e-05 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812748+0000", 2023-10-09T04:20:21.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89277856099999997 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812756+0000", 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6079999999999998e-06 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887444+0000", 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.074688672 2023-10-09T04:20:21.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887596+0000", 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000151533 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887736+0000", 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013991200000000001 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891747+0000", 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040110479999999997 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898230+0000", 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064832809999999996 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899391+0000", 2023-10-09T04:20:21.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001160863 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899443+0000", 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1969000000000003e-05 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.900364+0000", 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00092131999999999999 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.322 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87402 9.d 9:b58a4af0:::benchmark_data_smithi163_464650_object87401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.385576495000002, 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9327528470000002, 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.323 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87402 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896603+0000", 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896662+0000", 2023-10-09T04:20:21.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8721999999999998e-05 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896672+0000", 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0342e-05 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896678+0000", 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5890000000000002e-06 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897746+0000", 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001067901 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897884+0000", 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013837699999999999 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.898027+0000", 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014309899999999999 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.903780+0000", 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057525900000000001 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913854+0000", 2023-10-09T04:20:21.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010074411 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913865+0000", 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0667e-05 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904284+0000", 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99041863600000002 2023-10-09T04:20:21.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904295+0000", 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1622e-05 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829288+0000", 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9249930480000002 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829357+0000", 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9139000000000003e-05 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87427 9.d 9:b85b518c:::benchmark_data_smithi163_464650_object87426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.352437485000003, 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9083351629999998, 2023-10-09T04:20:21.331 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87427 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929742+0000", 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929812+0000", 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9973000000000002e-05 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929833+0000", 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0372000000000001e-05 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929840+0000", 2023-10-09T04:20:21.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0380000000000002e-06 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929955+0000", 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115658 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929964+0000", 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.02e-06 2023-10-09T04:20:21.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904451+0000", 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97448704799999997 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904459+0000", 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829570+0000", 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9251109199999998 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829657+0000", 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6693000000000005e-05 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829772+0000", 2023-10-09T04:20:21.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115109 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837839+0000", 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080666330000000001 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837847+0000", 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5720000000000003e-06 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837892+0000", 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4297e-05 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837929+0000", 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7622e-05 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837980+0000", 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0816000000000003e-05 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838079+0000", 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8758000000000006e-05 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87460 9.d 9:be691015:::benchmark_data_smithi163_464650_object87459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.311509870000002, 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8737964319999998, 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87460 2023-10-09T04:20:21.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970670+0000", 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970704+0000", 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3708999999999999e-05 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970712+0000", 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2910000000000007e-06 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970718+0000", 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2909999999999998e-06 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971579+0000", 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086131599999999999 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971589+0000", 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0108e-05 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904489+0000", 2023-10-09T04:20:21.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93289962699999995 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904496+0000", 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3939999999999998e-06 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832602+0000", 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9281065480000001 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832830+0000", 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022731300000000001 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.833032+0000", 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202256 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.347 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837853+0000", 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048209000000000004 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838118+0000", 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000265083 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838174+0000", 2023-10-09T04:20:21.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5485999999999997e-05 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844377+0000", 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062031949999999999 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844410+0000", 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3266999999999998e-05 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844468+0000", 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7698000000000002e-05 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87469 9.d 9:b0d8b7f7:::benchmark_data_smithi163_464650_object87468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.308301215999997, 2023-10-09T04:20:21.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8824357369999998, 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87469 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:20:21.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973879+0000", 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973909+0000", 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0063999999999999e-05 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973921+0000", 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.204e-05 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973927+0000", 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2049999999999999e-06 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974432+0000", 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050552799999999999 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974441+0000", 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:20:21.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904518+0000", 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93007710099999996 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904525+0000", 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9280000000000004e-06 2023-10-09T04:20:21.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835034+0000", 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9305090869999999 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835189+0000", 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015497800000000001 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835319+0000", 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012983499999999999 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838122+0000", 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028034420000000002 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838149+0000", 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6636999999999999e-05 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838193+0000", 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4202999999999997e-05 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856143+0000", 2023-10-09T04:20:21.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017949975 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856198+0000", 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4968e-05 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856316+0000", 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117589 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.359 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87510 9.d 9:b1e25812:::benchmark_data_smithi163_464650_object87509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.258943803000001, 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8391414909999999, 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.360 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87510 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023236+0000", 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023264+0000", 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7639000000000001e-05 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023280+0000", 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6189999999999999e-05 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023285+0000", 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4380000000000003e-06 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023338+0000", 2023-10-09T04:20:21.363 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2713999999999997e-05 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023350+0000", 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1912999999999999e-05 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904548+0000", 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88119805299999998 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.364 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904555+0000", 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1690000000000003e-06 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838206+0000", 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9336506519999999 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.365 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838292+0000", 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6051999999999995e-05 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838426+0000", 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013363900000000001 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844098+0000", 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056724699999999998 2023-10-09T04:20:21.366 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844218+0000", 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012015000000000001 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844519+0000", 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000300842 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.367 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862218+0000", 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017698935999999998 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862272+0000", 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4085999999999997e-05 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.368 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862379+0000", 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87520 9.d 9:b10313ad:::benchmark_data_smithi163_464650_object87519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.250315978000003, 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.830771683, 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.369 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87520 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.370 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031864+0000", 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.371 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031940+0000", 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6348999999999997e-05 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031956+0000", 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5733e-05 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031962+0000", 2023-10-09T04:20:21.372 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1330000000000004e-06 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032020+0000", 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8168999999999998e-05 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032031+0000", 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0546e-05 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.373 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904577+0000", 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87254678500000005 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904584+0000", 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8279999999999997e-06 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.374 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841092+0000", 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.936507684 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841230+0000", 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138245 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841385+0000", 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015503399999999999 2023-10-09T04:20:21.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844512+0000", 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031265519999999999 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850232+0000", 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057197300000000001 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.376 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850519+0000", 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000287342 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862488+0000", 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011968807999999999 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862529+0000", 2023-10-09T04:20:21.377 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1087999999999997e-05 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862637+0000", 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108094 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.378 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87523 9.d 9:b2b1ff3b:::benchmark_data_smithi163_464650_object87522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.244467227999998, 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8252149690000001, 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.379 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87523 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.380 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037713+0000", 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037758+0000", 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5336000000000001e-05 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.381 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037769+0000", 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1139e-05 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037774+0000", 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6249999999999998e-06 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.382 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037811+0000", 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7234999999999998e-05 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037819+0000", 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8599999999999993e-06 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904623+0000", 2023-10-09T04:20:21.383 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86680350299999998 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904630+0000", 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5780000000000001e-06 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844552+0000", 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9399218949999999 2023-10-09T04:20:21.384 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844685+0000", 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013337900000000001 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844852+0000", 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166682 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.385 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.386 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850445+0000", 2023-10-09T04:20:21.386 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055933379999999998 2023-10-09T04:20:21.386 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850487+0000", 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1508000000000003e-05 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856104+0000", 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056174129999999999 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.387 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862788+0000", 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066833480000000004 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862832+0000", 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000002e-05 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.388 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862929+0000", 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7040999999999995e-05 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87524 9.d 9:bf5a5ce2:::benchmark_data_smithi163_464650_object87523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.241743116999999, 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8227152589999998, 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.389 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87524 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.390 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040437+0000", 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.391 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040544+0000", 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010666399999999999 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040556+0000", 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697999999999999e-05 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040562+0000", 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0839999999999997e-06 2023-10-09T04:20:21.392 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040647+0000", 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4971999999999999e-05 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040656+0000", 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9479999999999997e-06 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.393 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904660+0000", 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86400389600000005 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904667+0000", 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847526+0000", 2023-10-09T04:20:21.394 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9428587660000001 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847651+0000", 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125136 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847838+0000", 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018702700000000001 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.395 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850552+0000", 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002713853 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856113+0000", 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055611580000000001 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862179+0000", 2023-10-09T04:20:21.396 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060656729999999997 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863021+0000", 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000842759 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863060+0000", 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8272000000000002e-05 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.397 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863153+0000", 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3684000000000006e-05 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87531 9.d 9:b4d1cf98:::benchmark_data_smithi163_464650_object87530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.240715606000002, 2023-10-09T04:20:21.398 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.836533765, 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87531 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:20:21.399 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041465+0000", 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.400 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041513+0000", 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8566999999999998e-05 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041522+0000", 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9020000000000005e-06 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.401 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041528+0000", 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041602+0000", 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4530000000000006e-05 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041611+0000", 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2819999999999996e-06 2023-10-09T04:20:21.402 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904688+0000", 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86307752000000004 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904695+0000", 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.403 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850572+0000", 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9458773759999999 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850685+0000", 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011277699999999999 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850827+0000", 2023-10-09T04:20:21.404 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014244099999999999 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862146+0000", 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011318701 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862451+0000", 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030508599999999999 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.405 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862741+0000", 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029011400000000001 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877820+0000", 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015078775000000001 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.406 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877869+0000", 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9407999999999999e-05 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877999+0000", 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129868 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.407 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87536 9.d 9:b13d2384:::benchmark_data_smithi163_464650_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.230233192999997, 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.826285253, 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87536 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.408 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051947+0000", 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052032+0000", 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5885999999999997e-05 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052049+0000", 2023-10-09T04:20:21.410 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6867e-05 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052056+0000", 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9349999999999997e-06 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052117+0000", 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0758999999999999e-05 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.411 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052127+0000", 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301e-05 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904719+0000", 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85259186799999997 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.412 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904726+0000", 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7989999999999996e-06 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853627+0000", 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9489006930000001 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853769+0000", 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142619 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.413 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853901+0000", 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205399999999999 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862421+0000", 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085194190000000003 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.414 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862735+0000", 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031399200000000003 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862747+0000", 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2238e-05 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878064+0000", 2023-10-09T04:20:21.415 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015316585000000001 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878120+0000", 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6338999999999997e-05 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878233+0000", 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113268 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.416 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87542 9.d 9:b8ae8bf6:::benchmark_data_smithi163_464650_object87541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.220731612999998, 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.817112244, 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.417 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87542 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:20:21.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061448+0000", 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061495+0000", 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6932999999999999e-05 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061515+0000", 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061522+0000", 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8499999999999996e-06 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.420 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061580+0000", 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8037e-05 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061591+0000", 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1099e-05 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904748+0000", 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84315744100000001 2023-10-09T04:20:21.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904755+0000", 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7880000000000001e-06 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856351+0000", 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9515959619999999 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856435+0000", 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3497000000000004e-05 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856590+0000", 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015579299999999999 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862697+0000", 2023-10-09T04:20:21.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0061066360000000004 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862753+0000", 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6106000000000001e-05 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862991+0000", 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023799100000000001 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878381+0000", 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015390452000000001 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878432+0000", 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0779999999999998e-05 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878562+0000", 2023-10-09T04:20:21.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012949399999999999 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87544 9.d 9:b2959611:::benchmark_data_smithi163_464650_object87543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.220586588000003, 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8172143809999999, 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.426 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87544 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061594+0000", 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061631+0000", 2023-10-09T04:20:21.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6940000000000002e-05 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061641+0000", 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5650000000000007e-06 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061645+0000", 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9400000000000001e-06 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061692+0000", 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6638000000000003e-05 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061705+0000", 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2525e-05 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904777+0000", 2023-10-09T04:20:21.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84307196799999995 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904783+0000", 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859252+0000", 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.954468756 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859377+0000", 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124423 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859542+0000", 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016522800000000001 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862758+0000", 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032164730000000001 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862963+0000", 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000205105 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871931+0000", 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089675880000000003 2023-10-09T04:20:21.433 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878651+0000", 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067196959999999998 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878706+0000", 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5047999999999999e-05 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878809+0000", 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103266 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87579 9.d 9:b6955889:::benchmark_data_smithi163_464650_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.163289585000001, 2023-10-09T04:20:21.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.760215788, 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87579 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118890+0000", 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:20:21.437 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118943+0000", 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2917000000000002e-05 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118957+0000", 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3553000000000001e-05 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118963+0000", 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9329999999999998e-06 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119027+0000", 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4450999999999997e-05 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119039+0000", 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1657999999999999e-05 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904808+0000", 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78576894799999997 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904814+0000", 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.472e-06 2023-10-09T04:20:21.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863179+0000", 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.958364789 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863297+0000", 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011818 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863439+0000", 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000141836 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871995+0000", 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085565290000000002 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872008+0000", 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2779000000000001e-05 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878306+0000", 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062979710000000003 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878882+0000", 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057614199999999997 2023-10-09T04:20:21.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878977+0000", 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4944000000000004e-05 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879107+0000", 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013005199999999999 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.444 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87597 9.d 9:ba722c6f:::benchmark_data_smithi163_464650_object87596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.134910869000002, 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.741504747, 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.445 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87597 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:20:21.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147306+0000", 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6227999999999999e-05 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147315+0000", 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4420000000000004e-06 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147321+0000", 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3900000000000001e-06 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147373+0000", 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1922000000000002e-05 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147383+0000", 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8160000000000005e-06 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904836+0000", 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75745349399999995 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904843+0000", 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9360000000000002e-06 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866166+0000", 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9613231419999999 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.450 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866285+0000", 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011870399999999999 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866435+0000", 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015030299999999999 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.875034+0000", 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085985039999999999 2023-10-09T04:20:21.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878346+0000", 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033125379999999999 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878607+0000", 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000261158 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888598+0000", 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099901480000000008 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888652+0000", 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4688999999999998e-05 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888775+0000", 2023-10-09T04:20:21.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012282100000000001 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87606 9.d 9:bd98b1c5:::benchmark_data_smithi163_464650_object87605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.114061462999999, 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7216060710000001, 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87606 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168119+0000", 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168170+0000", 2023-10-09T04:20:21.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1724000000000001e-05 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168183+0000", 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2322e-05 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168188+0000", 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.011e-06 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168245+0000", 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7197999999999997e-05 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168257+0000", 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1742999999999999e-05 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.458 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904867+0000", 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.73660977999999999 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904873+0000", 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3450000000000002e-06 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.868939+0000", 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9640662180000001 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869059+0000", 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012025 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869200+0000", 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014040799999999999 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878017+0000", 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0088170209999999995 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878849+0000", 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00083247799999999999 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879166+0000", 2023-10-09T04:20:21.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000316373 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889551+0000", 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010385553 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889609+0000", 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-05 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.462 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889726+0000", 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116743 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87628 9.d 9:b09d3eb4:::benchmark_data_smithi163_464650_object87627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.029600811999998, 2023-10-09T04:20:21.463 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6374180369999998, 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87628 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:20:21.464 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252579+0000", 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.465 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252624+0000", 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5071999999999998e-05 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252637+0000", 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.272e-05 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252642+0000", 2023-10-09T04:20:21.466 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8999999999999997e-06 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252685+0000", 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3056000000000002e-05 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252691+0000", 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3790000000000003e-06 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.467 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905181+0000", 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.65248916000000001 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905191+0000", 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0231000000000001e-05 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872040+0000", 2023-10-09T04:20:21.468 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.966849243 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872156+0000", 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115611 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872308+0000", 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152707 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.469 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878257+0000", 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059484389999999998 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879226+0000", 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096947500000000002 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.470 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888556+0000", 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093301310000000002 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889825+0000", 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001268948 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889882+0000", 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6885999999999997e-05 2023-10-09T04:20:21.471 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889998+0000", 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011613899999999999 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:20:21.472 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:20:21.473 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:21.548 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:21.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:21.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:21.937 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:22.006 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:22.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:22.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:22.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:22.219 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:22.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:22.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:20:22.300 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:22.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:22.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:22.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:22.705 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:22.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:20:22.858 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:22.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:22.931 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:20:22.975 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:22.993 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:20:23.011 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:23.036 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:20:23.071 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:23.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:23.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[213841]: pgmap v955: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 15004/223314 objects misplaced (6.719%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1892327693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[221067]: pgmap v955: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 15004/223314 objects misplaced (6.719%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:23.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1892327693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:23.084 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:20:23.125 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:20:23.156 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:20:23.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:23.189 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:20:23.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:22 smithi181 ceph-mon[196979]: pgmap v955: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 15004/223314 objects misplaced (6.719%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:20:23.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:23.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:23.226 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1892327693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:23.229 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:23.241 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:20:23.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.260 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:20:23.278 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:23.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:23.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:23.424 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:23.509 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:23.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:23.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:23.687 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:24.011 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:24.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:24.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:24.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:24.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:24.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:24.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:24.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:24.156 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:25.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:25.076 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:25.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:25.186 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:25.186 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:25.265 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:24 smithi163 ceph-mon[213841]: pgmap v956: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 14242/221421 objects misplaced (6.432%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:25.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:24 smithi163 ceph-mon[221067]: pgmap v956: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 14242/221421 objects misplaced (6.432%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:25.266 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:25.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:24 smithi181 ceph-mon[196979]: pgmap v956: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 14242/221421 objects misplaced (6.432%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:25.478 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:25.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:25.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:25.689 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:26.392 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:26.409 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:20:26.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:26.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:26.730 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:26.758 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:26.919 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:26 smithi163 ceph-mon[221067]: pgmap v957: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 14242/220689 objects misplaced (6.453%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:27.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:20:27.173 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:26 smithi163 ceph-mon[213841]: pgmap v957: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 14242/220689 objects misplaced (6.453%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:27.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:26 smithi181 ceph-mon[196979]: pgmap v957: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 14242/220689 objects misplaced (6.453%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:27.712 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:27.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:27.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:27.850 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:27.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:27.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:27.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:28.058 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:28.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:27 smithi181 ceph-mon[196979]: from='client.35970 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:28.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:27 smithi163 ceph-mon[213841]: from='client.35970 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:28.310 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:27 smithi163 ceph-mon[221067]: from='client.35970 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:28.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:28.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:28.432 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":957,"stamp":"2023-10-09T04:20:25.838807+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4820058297,"num_objects":73563,"num_object_clones":0,"num_object_copies":220689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":14242,"num_objects_unfound":0,"num_objects_dirty":73563,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":124002,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15882,"num_bytes_recovered":1036953313,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96834,"ondisk_log_size":96834,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18578948,"kb_used_data":14691468,"kb_used_omap":126,"kb_used_meta":3887297,"kb_avail":637637116,"statfs":{"total":671965249536,"available":652940406784,"internally_reserved":0,"allocated":15044063232,"data_stored":15035032475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":3980592763},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,4,7,3,1,0,0,9],"upper_bound":512},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-455999488,"num_objects":-6958,"num_object_clones":0,"num_object_copies":-20874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2585,"num_objects_unfound":0,"num_objects_dirty":-6958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6958,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1175,"num_bytes_recovered":77004800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007262"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":480,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999088+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:20:11.999088+0000","last_peered":"2023-10-09T04:20:11.999088+0000","last_clean":"2023-10-09T04:20:11.999088+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:20:11.999088+0000","last_undegraded":"2023-10-09T04:20:11.999088+0000","last_fullsized":"2023-10-09T04:20:11.999088+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":100,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025729+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:20:12.025729+0000","last_peered":"2023-10-09T04:20:12.025729+0000","last_clean":"2023-10-09T04:20:12.025729+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:20:12.025729+0000","last_undegraded":"2023-10-09T04:20:12.025729+0000","last_fullsized":"2023-10-09T04:20:12.025729+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":331,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998256+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:20:11.998256+0000","last_peered":"2023-10-09T04:20:11.998256+0000","last_clean":"2023-10-09T04:20:11.998256+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:20:11.998256+0000","last_undegraded":"2023-10-09T04:20:11.998256+0000","last_fullsized":"2023-10-09T04:20:11.998256+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":473,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999504+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:20:11.999504+0000","last_peered":"2023-10-09T04:20:11.999504+0000","last_clean":"2023-10-09T04:20:11.999504+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:20:11.999504+0000","last_undegraded":"2023-10-09T04:20:11.999504+0000","last_fullsized":"2023-10-09T04:20:11.999504+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":346,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998011+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:20:11.998011+0000","last_peered":"2023-10-09T04:20:11.998011+0000","last_clean":"2023-10-09T04:20:11.998011+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:20:11.998011+0000","last_undegraded":"2023-10-09T04:20:11.998011+0000","last_fullsized":"2023-10-09T04:20:11.998011+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":334,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.031820+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:20:12.031820+0000","last_peered":"2023-10-09T04:20:12.031820+0000","last_clean":"2023-10-09T04:20:12.031820+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:20:12.031820+0000","last_undegraded":"2023-10-09T04:20:12.031820+0000","last_fullsized":"2023-10-09T04:20:12.031820+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":480,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025451+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:20:12.025451+0000","last_peered":"2023-10-09T04:20:12.025451+0000","last_clean":"2023-10-09T04:20:12.025451+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:20:12.025451+0000","last_undegraded":"2023-10-09T04:20:12.025451+0000","last_fullsized":"2023-10-09T04:20:12.025451+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":331,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.028144+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:20:12.028144+0000","last_peered":"2023-10-09T04:20:12.028144+0000","last_clean":"2023-10-09T04:20:12.028144+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:20:12.028144+0000","last_undegraded":"2023-10-09T04:20:12.028144+0000","last_fullsized":"2023-10-09T04:20:12.028144+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":299,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.000301+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:20:12.000301+0000","last_peered":"2023-10-09T04:20:12.000301+0000","last_clean":"2023-10-09T04:20:12.000301+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:20:12.000301+0000","last_undegraded":"2023-10-09T04:20:12.000301+0000","last_fullsized":"2023-10-09T04:20:12.000301+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":755,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997627+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:20:11.997627+0000","last_peered":"2023-10-09T04:20:11.997627+0000","last_clean":"2023-10-09T04:20:11.997627+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:20:11.997627+0000","last_undegraded":"2023-10-09T04:20:11.997627+0000","last_fullsized":"2023-10-09T04:20:11.997627+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":332,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.032200+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:20:12.032200+0000","last_peered":"2023-10-09T04:20:12.032200+0000","last_clean":"2023-10-09T04:20:12.032200+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:20:12.032200+0000","last_undegraded":"2023-10-09T04:20:12.032200+0000","last_fullsized":"2023-10-09T04:20:12.032200+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":65,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.995132+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:20:11.995132+0000","last_peered":"2023-10-09T04:20:11.995132+0000","last_clean":"2023-10-09T04:20:11.995132+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:20:11.995132+0000","last_undegraded":"2023-10-09T04:20:11.995132+0000","last_fullsized":"2023-10-09T04:20:11.995132+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":482,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.587338+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:20:18.587338+0000","last_peered":"2023-10-09T04:20:18.587338+0000","last_clean":"2023-10-09T04:20:18.587338+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:20:18.587338+0000","last_undegraded":"2023-10-09T04:20:18.587338+0000","last_fullsized":"2023-10-09T04:20:18.587338+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":315,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.000067+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:20:12.000067+0000","last_peered":"2023-10-09T04:20:12.000067+0000","last_clean":"2023-10-09T04:20:12.000067+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:20:12.000067+0000","last_undegraded":"2023-10-09T04:20:12.000067+0000","last_fullsized":"2023-10-09T04:20:12.000067+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999050+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:20:11.999050+0000","last_peered":"2023-10-09T04:20:11.999050+0000","last_clean":"2023-10-09T04:20:11.999050+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:20:11.999050+0000","last_undegraded":"2023-10-09T04:20:11.999050+0000","last_fullsized":"2023-10-09T04:20:11.999050+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":490,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.000225+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:20:12.000225+0000","last_peered":"2023-10-09T04:20:12.000225+0000","last_clean":"2023-10-09T04:20:12.000225+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:20:12.000225+0000","last_undegraded":"2023-10-09T04:20:12.000225+0000","last_fullsized":"2023-10-09T04:20:12.000225+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":398,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997875+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:20:11.997875+0000","last_peered":"2023-10-09T04:20:11.997875+0000","last_clean":"2023-10-09T04:20:11.997875+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:20:11.997875+0000","last_undegraded":"2023-10-09T04:20:11.997875+0000","last_fullsized":"2023-10-09T04:20:11.997875+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":34,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999408+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:20:11.999408+0000","last_peered":"2023-10-09T04:20:11.999408+0000","last_clean":"2023-10-09T04:20:11.999408+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:20:11.999408+0000","last_undegraded":"2023-10-09T04:20:11.999408+0000","last_fullsized":"2023-10-09T04:20:11.999408+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":497,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998306+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:20:11.998306+0000","last_peered":"2023-10-09T04:20:11.998306+0000","last_clean":"2023-10-09T04:20:11.998306+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:20:11.998306+0000","last_undegraded":"2023-10-09T04:20:11.998306+0000","last_fullsized":"2023-10-09T04:20:11.998306+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":130,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.024882+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:20:12.024882+0000","last_peered":"2023-10-09T04:20:12.024882+0000","last_clean":"2023-10-09T04:20:12.024882+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:20:12.024882+0000","last_undegraded":"2023-10-09T04:20:12.024882+0000","last_fullsized":"2023-10-09T04:20:12.024882+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026335+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:20:12.026335+0000","last_peered":"2023-10-09T04:20:12.026335+0000","last_clean":"2023-10-09T04:20:12.026335+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:20:12.026335+0000","last_undegraded":"2023-10-09T04:20:12.026335+0000","last_fullsized":"2023-10-09T04:20:12.026335+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":506,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.566142+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:20:18.566142+0000","last_peered":"2023-10-09T04:20:18.566142+0000","last_clean":"2023-10-09T04:20:18.566142+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:20:18.566142+0000","last_undegraded":"2023-10-09T04:20:18.566142+0000","last_fullsized":"2023-10-09T04:20:18.566142+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":349,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997245+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:20:11.997245+0000","last_peered":"2023-10-09T04:20:11.997245+0000","last_clean":"2023-10-09T04:20:11.997245+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:20:11.997245+0000","last_undegraded":"2023-10-09T04:20:11.997245+0000","last_fullsized":"2023-10-09T04:20:11.997245+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":34,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999356+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:20:11.999356+0000","last_peered":"2023-10-09T04:20:11.999356+0000","last_clean":"2023-10-09T04:20:11.999356+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:20:11.999356+0000","last_undegraded":"2023-10-09T04:20:11.999356+0000","last_fullsized":"2023-10-09T04:20:11.999356+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":500,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999172+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:20:11.999172+0000","last_peered":"2023-10-09T04:20:11.999172+0000","last_clean":"2023-10-09T04:20:11.999172+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:20:11.999172+0000","last_undegraded":"2023-10-09T04:20:11.999172+0000","last_fullsized":"2023-10-09T04:20:11.999172+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":332,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.028035+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:20:12.028035+0000","last_peered":"2023-10-09T04:20:12.028035+0000","last_clean":"2023-10-09T04:20:12.028035+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:20:12.028035+0000","last_undegraded":"2023-10-09T04:20:12.028035+0000","last_fullsized":"2023-10-09T04:20:12.028035+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999578+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:20:11.999578+0000","last_peered":"2023-10-09T04:20:11.999578+0000","last_clean":"2023-10-09T04:20:11.999578+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:20:11.999578+0000","last_undegraded":"2023-10-09T04:20:11.999578+0000","last_fullsized":"2023-10-09T04:20:11.999578+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":497,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999025+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:20:11.999025+0000","last_peered":"2023-10-09T04:20:11.999025+0000","last_clean":"2023-10-09T04:20:11.999025+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:20:11.999025+0000","last_undegraded":"2023-10-09T04:20:11.999025+0000","last_fullsized":"2023-10-09T04:20:11.999025+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025090+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:20:12.025090+0000","last_peered":"2023-10-09T04:20:12.025090+0000","last_clean":"2023-10-09T04:20:12.025090+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:20:12.025090+0000","last_undegraded":"2023-10-09T04:20:12.025090+0000","last_fullsized":"2023-10-09T04:20:12.025090+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":332,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.031890+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:20:12.031890+0000","last_peered":"2023-10-09T04:20:12.031890+0000","last_clean":"2023-10-09T04:20:12.031890+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:20:12.031890+0000","last_undegraded":"2023-10-09T04:20:12.031890+0000","last_fullsized":"2023-10-09T04:20:12.031890+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":500,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999469+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:20:11.999469+0000","last_peered":"2023-10-09T04:20:11.999469+0000","last_clean":"2023-10-09T04:20:11.999469+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:20:11.999469+0000","last_undegraded":"2023-10-09T04:20:11.999469+0000","last_fullsized":"2023-10-09T04:20:11.999469+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":345,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.027993+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:20:12.027993+0000","last_peered":"2023-10-09T04:20:12.027993+0000","last_clean":"2023-10-09T04:20:12.027993+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:20:12.027993+0000","last_undegraded":"2023-10-09T04:20:12.027993+0000","last_fullsized":"2023-10-09T04:20:12.027993+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.995245+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:20:11.995245+0000","last_peered":"2023-10-09T04:20:11.995245+0000","last_clean":"2023-10-09T04:20:11.995245+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:20:11.995245+0000","last_undegraded":"2023-10-09T04:20:11.995245+0000","last_fullsized":"2023-10-09T04:20:11.995245+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"356'7641","reported_seq":7700,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:21.652361+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:20:21.652361+0000","last_peered":"2023-10-09T04:20:21.652361+0000","last_clean":"2023-10-09T04:20:21.652361+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:20:21.652361+0000","last_undegraded":"2023-10-09T04:20:21.652361+0000","last_fullsized":"2023-10-09T04:20:21.652361+0000","mapping_epoch":306,"log_start":"314'1300","ondisk_log_start":"314'1300","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7641,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":758,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997448+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:20:11.997448+0000","last_peered":"2023-10-09T04:20:11.997448+0000","last_clean":"2023-10-09T04:20:11.997448+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:20:11.997448+0000","last_undegraded":"2023-10-09T04:20:11.997448+0000","last_fullsized":"2023-10-09T04:20:11.997448+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":344,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997706+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:20:11.997706+0000","last_peered":"2023-10-09T04:20:11.997706+0000","last_clean":"2023-10-09T04:20:11.997706+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:20:11.997706+0000","last_undegraded":"2023-10-09T04:20:11.997706+0000","last_fullsized":"2023-10-09T04:20:11.997706+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":307,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999261+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:20:11.999261+0000","last_peered":"2023-10-09T04:20:11.999261+0000","last_clean":"2023-10-09T04:20:11.999261+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:20:11.999261+0000","last_undegraded":"2023-10-09T04:20:11.999261+0000","last_fullsized":"2023-10-09T04:20:11.999261+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"356'7847","reported_seq":7916,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:23.274563+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:20:23.274563+0000","last_peered":"2023-10-09T04:20:23.274563+0000","last_clean":"2023-10-09T04:20:23.274563+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:20:23.274563+0000","last_undegraded":"2023-10-09T04:20:23.274563+0000","last_fullsized":"2023-10-09T04:20:23.274563+0000","mapping_epoch":310,"log_start":"319'2573","ondisk_log_start":"319'2573","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298123264,"num_objects":4549,"num_object_clones":0,"num_object_copies":13647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7847,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":522,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.995631+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:20:11.995631+0000","last_peered":"2023-10-09T04:20:11.995631+0000","last_clean":"2023-10-09T04:20:11.995631+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:20:11.995631+0000","last_undegraded":"2023-10-09T04:20:11.995631+0000","last_fullsized":"2023-10-09T04:20:11.995631+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":348,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997647+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:20:11.997647+0000","last_peered":"2023-10-09T04:20:11.997647+0000","last_clean":"2023-10-09T04:20:11.997647+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:20:11.997647+0000","last_undegraded":"2023-10-09T04:20:11.997647+0000","last_fullsized":"2023-10-09T04:20:11.997647+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":130,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025159+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:20:12.025159+0000","last_peered":"2023-10-09T04:20:12.025159+0000","last_clean":"2023-10-09T04:20:12.025159+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:20:12.025159+0000","last_undegraded":"2023-10-09T04:20:12.025159+0000","last_fullsized":"2023-10-09T04:20:12.025159+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"356'7876","reported_seq":7945,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:20.823933+0000","last_change":"2023-10-09T04:20:03.101091+0000","last_active":"2023-10-09T04:20:20.823933+0000","last_peered":"2023-10-09T04:20:20.823933+0000","last_clean":"2023-10-09T04:20:00.944119+0000","last_became_active":"2023-10-09T04:20:03.092473+0000","last_became_peered":"2023-10-09T04:20:03.092473+0000","last_unstale":"2023-10-09T04:20:20.823933+0000","last_undegraded":"2023-10-09T04:20:20.823933+0000","last_fullsized":"2023-10-09T04:20:20.823933+0000","mapping_epoch":349,"log_start":"317'1918","ondisk_log_start":"317'1918","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:49.431905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308019200,"num_objects":4700,"num_object_clones":0,"num_object_copies":14100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4700,"num_objects_unfound":0,"num_objects_dirty":4700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7876,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":4700}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":490,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.000163+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:20:12.000163+0000","last_peered":"2023-10-09T04:20:12.000163+0000","last_clean":"2023-10-09T04:20:12.000163+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:20:12.000163+0000","last_undegraded":"2023-10-09T04:20:12.000163+0000","last_fullsized":"2023-10-09T04:20:12.000163+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":313,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997739+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:20:11.997739+0000","last_peered":"2023-10-09T04:20:11.997739+0000","last_clean":"2023-10-09T04:20:11.997739+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:20:11.997739+0000","last_undegraded":"2023-10-09T04:20:11.997739+0000","last_fullsized":"2023-10-09T04:20:11.997739+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":288,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999825+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:20:11.999825+0000","last_peered":"2023-10-09T04:20:11.999825+0000","last_clean":"2023-10-09T04:20:11.999825+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:20:11.999825+0000","last_undegraded":"2023-10-09T04:20:11.999825+0000","last_fullsized":"2023-10-09T04:20:11.999825+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"356'7738","reported_seq":7796,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.833787+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:20:20.833787+0000","last_peered":"2023-10-09T04:20:20.833787+0000","last_clean":"2023-10-09T04:20:20.833787+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:20:20.833787+0000","last_undegraded":"2023-10-09T04:20:20.833787+0000","last_fullsized":"2023-10-09T04:20:20.833787+0000","mapping_epoch":306,"log_start":"314'1787","ondisk_log_start":"314'1787","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300548096,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7738,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":500,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026942+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:20:12.026942+0000","last_peered":"2023-10-09T04:20:12.026942+0000","last_clean":"2023-10-09T04:20:12.026942+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:20:12.026942+0000","last_undegraded":"2023-10-09T04:20:12.026942+0000","last_fullsized":"2023-10-09T04:20:12.026942+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":345,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999838+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:20:11.999838+0000","last_peered":"2023-10-09T04:20:11.999838+0000","last_clean":"2023-10-09T04:20:11.999838+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:20:11.999838+0000","last_undegraded":"2023-10-09T04:20:11.999838+0000","last_fullsized":"2023-10-09T04:20:11.999838+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":333,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.032104+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:20:12.032104+0000","last_peered":"2023-10-09T04:20:12.032104+0000","last_clean":"2023-10-09T04:20:12.032104+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:20:12.032104+0000","last_undegraded":"2023-10-09T04:20:12.032104+0000","last_fullsized":"2023-10-09T04:20:12.032104+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"356'7371","reported_seq":10802,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.905652+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:20:20.905652+0000","last_peered":"2023-10-09T04:20:20.905652+0000","last_clean":"2023-10-09T04:20:20.905652+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:20:20.905652+0000","last_undegraded":"2023-10-09T04:20:20.905652+0000","last_fullsized":"2023-10-09T04:20:20.905652+0000","mapping_epoch":320,"log_start":"319'2100","ondisk_log_start":"319'2100","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5271,"log_dups_size":0,"ondisk_log_size":5271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7371,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":779,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997061+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:20:11.997061+0000","last_peered":"2023-10-09T04:20:11.997061+0000","last_clean":"2023-10-09T04:20:11.997061+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:20:11.997061+0000","last_undegraded":"2023-10-09T04:20:11.997061+0000","last_fullsized":"2023-10-09T04:20:11.997061+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":316,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:16.752076+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:20:16.752076+0000","last_peered":"2023-10-09T04:20:16.752076+0000","last_clean":"2023-10-09T04:20:16.752076+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:20:16.752076+0000","last_undegraded":"2023-10-09T04:20:16.752076+0000","last_fullsized":"2023-10-09T04:20:16.752076+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":130,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026535+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:20:12.026535+0000","last_peered":"2023-10-09T04:20:12.026535+0000","last_clean":"2023-10-09T04:20:12.026535+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:20:12.026535+0000","last_undegraded":"2023-10-09T04:20:12.026535+0000","last_fullsized":"2023-10-09T04:20:12.026535+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"356'7774","reported_seq":11345,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:23.241238+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:20:23.241238+0000","last_peered":"2023-10-09T04:20:23.241238+0000","last_clean":"2023-10-09T04:20:23.241238+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:20:23.241238+0000","last_undegraded":"2023-10-09T04:20:23.241238+0000","last_fullsized":"2023-10-09T04:20:23.241238+0000","mapping_epoch":329,"log_start":"319'2460","ondisk_log_start":"319'2460","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295043072,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7774,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":525,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998995+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:20:11.998995+0000","last_peered":"2023-10-09T04:20:11.998995+0000","last_clean":"2023-10-09T04:20:11.998995+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:20:11.998995+0000","last_undegraded":"2023-10-09T04:20:11.998995+0000","last_fullsized":"2023-10-09T04:20:11.998995+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025572+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:20:12.025572+0000","last_peered":"2023-10-09T04:20:12.025572+0000","last_clean":"2023-10-09T04:20:12.025572+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:20:12.025572+0000","last_undegraded":"2023-10-09T04:20:12.025572+0000","last_fullsized":"2023-10-09T04:20:12.025572+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":100,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.027013+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:20:12.027013+0000","last_peered":"2023-10-09T04:20:12.027013+0000","last_clean":"2023-10-09T04:20:12.027013+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:20:12.027013+0000","last_undegraded":"2023-10-09T04:20:12.027013+0000","last_fullsized":"2023-10-09T04:20:12.027013+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"356'7621","reported_seq":7688,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.889999+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:20:20.889999+0000","last_peered":"2023-10-09T04:20:20.889999+0000","last_clean":"2023-10-09T04:20:20.889999+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:20:20.889999+0000","last_undegraded":"2023-10-09T04:20:20.889999+0000","last_fullsized":"2023-10-09T04:20:20.889999+0000","mapping_epoch":310,"log_start":"319'2300","ondisk_log_start":"319'2300","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5321,"log_dups_size":0,"ondisk_log_size":5321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296550400,"num_objects":4525,"num_object_clones":0,"num_object_copies":13575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7621,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":481,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.579496+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:20:18.579496+0000","last_peered":"2023-10-09T04:20:18.579496+0000","last_clean":"2023-10-09T04:20:18.579496+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:20:18.579496+0000","last_undegraded":"2023-10-09T04:20:18.579496+0000","last_fullsized":"2023-10-09T04:20:18.579496+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":100,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.024967+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:20:12.024967+0000","last_peered":"2023-10-09T04:20:12.024967+0000","last_clean":"2023-10-09T04:20:12.024967+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:20:12.024967+0000","last_undegraded":"2023-10-09T04:20:12.024967+0000","last_fullsized":"2023-10-09T04:20:12.024967+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":150,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.027101+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:20:12.027101+0000","last_peered":"2023-10-09T04:20:12.027101+0000","last_clean":"2023-10-09T04:20:12.027101+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:20:12.027101+0000","last_undegraded":"2023-10-09T04:20:12.027101+0000","last_fullsized":"2023-10-09T04:20:12.027101+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"356'7820","reported_seq":13057,"reported_epoch":356,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:20:23.277378+0000","last_change":"2023-10-09T04:20:05.061955+0000","last_active":"2023-10-09T04:20:23.277378+0000","last_peered":"2023-10-09T04:20:23.277378+0000","last_clean":"2023-10-09T04:20:00.980108+0000","last_became_active":"2023-10-09T04:20:03.079675+0000","last_became_peered":"2023-10-09T04:20:03.079675+0000","last_unstale":"2023-10-09T04:20:23.277378+0000","last_undegraded":"2023-10-09T04:20:23.277378+0000","last_fullsized":"2023-10-09T04:20:23.277378+0000","mapping_epoch":349,"log_start":"319'2481","ondisk_log_start":"319'2481","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5339,"log_dups_size":0,"ondisk_log_size":5339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:58.283644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301596672,"num_objects":4602,"num_object_clones":0,"num_object_copies":13806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2233,"num_objects_unfound":0,"num_objects_dirty":4602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7820,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2585,"num_bytes_recovered":169410560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4602}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":500,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999220+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:20:11.999220+0000","last_peered":"2023-10-09T04:20:11.999220+0000","last_clean":"2023-10-09T04:20:11.999220+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:20:11.999220+0000","last_undegraded":"2023-10-09T04:20:11.999220+0000","last_fullsized":"2023-10-09T04:20:11.999220+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":18,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.994830+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:20:11.994830+0000","last_peered":"2023-10-09T04:20:11.994830+0000","last_clean":"2023-10-09T04:20:11.994830+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:20:11.994830+0000","last_undegraded":"2023-10-09T04:20:11.994830+0000","last_fullsized":"2023-10-09T04:20:11.994830+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":23,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.014148+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:20:12.014148+0000","last_peered":"2023-10-09T04:20:12.014148+0000","last_clean":"2023-10-09T04:20:12.014148+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:20:12.014148+0000","last_undegraded":"2023-10-09T04:20:12.014148+0000","last_fullsized":"2023-10-09T04:20:12.014148+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"356'7746","reported_seq":7814,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.846046+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:20:20.846046+0000","last_peered":"2023-10-09T04:20:20.846046+0000","last_clean":"2023-10-09T04:20:20.846046+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:20:20.846046+0000","last_undegraded":"2023-10-09T04:20:20.846046+0000","last_fullsized":"2023-10-09T04:20:20.846046+0000","mapping_epoch":310,"log_start":"317'1834","ondisk_log_start":"317'1834","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305659904,"num_objects":4664,"num_object_clones":0,"num_object_copies":13992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7746,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":520,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.000378+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:20:12.000378+0000","last_peered":"2023-10-09T04:20:12.000378+0000","last_clean":"2023-10-09T04:20:12.000378+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:20:12.000378+0000","last_undegraded":"2023-10-09T04:20:12.000378+0000","last_fullsized":"2023-10-09T04:20:12.000378+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":335,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999734+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:20:11.999734+0000","last_peered":"2023-10-09T04:20:11.999734+0000","last_clean":"2023-10-09T04:20:11.999734+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:20:11.999734+0000","last_undegraded":"2023-10-09T04:20:11.999734+0000","last_fullsized":"2023-10-09T04:20:11.999734+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":352,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999494+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:20:11.999494+0000","last_peered":"2023-10-09T04:20:11.999494+0000","last_clean":"2023-10-09T04:20:11.999494+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:20:11.999494+0000","last_undegraded":"2023-10-09T04:20:11.999494+0000","last_fullsized":"2023-10-09T04:20:11.999494+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":23,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.013451+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:20:12.013451+0000","last_peered":"2023-10-09T04:20:12.013451+0000","last_clean":"2023-10-09T04:20:12.013451+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:20:12.013451+0000","last_undegraded":"2023-10-09T04:20:12.013451+0000","last_fullsized":"2023-10-09T04:20:12.013451+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"356'7724","reported_seq":7784,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:21.660037+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:20:21.660037+0000","last_peered":"2023-10-09T04:20:21.660037+0000","last_clean":"2023-10-09T04:20:21.660037+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:20:21.660037+0000","last_undegraded":"2023-10-09T04:20:21.660037+0000","last_fullsized":"2023-10-09T04:20:21.660037+0000","mapping_epoch":306,"log_start":"314'1400","ondisk_log_start":"314'1400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6324,"log_dups_size":0,"ondisk_log_size":6324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300023808,"num_objects":4578,"num_object_clones":0,"num_object_copies":13734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7724,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":473,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999340+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:20:11.999340+0000","last_peered":"2023-10-09T04:20:11.999340+0000","last_clean":"2023-10-09T04:20:11.999340+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:20:11.999340+0000","last_undegraded":"2023-10-09T04:20:11.999340+0000","last_fullsized":"2023-10-09T04:20:11.999340+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":329,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999317+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:20:11.999317+0000","last_peered":"2023-10-09T04:20:11.999317+0000","last_clean":"2023-10-09T04:20:11.999317+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:20:11.999317+0000","last_undegraded":"2023-10-09T04:20:11.999317+0000","last_fullsized":"2023-10-09T04:20:11.999317+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026253+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:20:12.026253+0000","last_peered":"2023-10-09T04:20:12.026253+0000","last_clean":"2023-10-09T04:20:12.026253+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:20:12.026253+0000","last_undegraded":"2023-10-09T04:20:12.026253+0000","last_fullsized":"2023-10-09T04:20:12.026253+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":363,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999665+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:20:11.999665+0000","last_peered":"2023-10-09T04:20:11.999665+0000","last_clean":"2023-10-09T04:20:11.999665+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:20:11.999665+0000","last_undegraded":"2023-10-09T04:20:11.999665+0000","last_fullsized":"2023-10-09T04:20:11.999665+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"356'7778","reported_seq":7836,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.846663+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:20:20.846663+0000","last_peered":"2023-10-09T04:20:20.846663+0000","last_clean":"2023-10-09T04:20:20.846663+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:20:20.846663+0000","last_undegraded":"2023-10-09T04:20:20.846663+0000","last_fullsized":"2023-10-09T04:20:20.846663+0000","mapping_epoch":306,"log_start":"318'1868","ondisk_log_start":"318'1868","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308150272,"num_objects":4702,"num_object_clones":0,"num_object_copies":14106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7778,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":489,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.028118+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:20:12.028118+0000","last_peered":"2023-10-09T04:20:12.028118+0000","last_clean":"2023-10-09T04:20:12.028118+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:20:12.028118+0000","last_undegraded":"2023-10-09T04:20:12.028118+0000","last_fullsized":"2023-10-09T04:20:12.028118+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":191,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.013510+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:20:12.013510+0000","last_peered":"2023-10-09T04:20:12.013510+0000","last_clean":"2023-10-09T04:20:12.013510+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:20:12.013510+0000","last_undegraded":"2023-10-09T04:20:12.013510+0000","last_fullsized":"2023-10-09T04:20:12.013510+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":338,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997907+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:20:11.997907+0000","last_peered":"2023-10-09T04:20:11.997907+0000","last_clean":"2023-10-09T04:20:11.997907+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:20:11.997907+0000","last_undegraded":"2023-10-09T04:20:11.997907+0000","last_fullsized":"2023-10-09T04:20:11.997907+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":34,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998087+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:20:11.998087+0000","last_peered":"2023-10-09T04:20:11.998087+0000","last_clean":"2023-10-09T04:20:11.998087+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:20:11.998087+0000","last_undegraded":"2023-10-09T04:20:11.998087+0000","last_fullsized":"2023-10-09T04:20:11.998087+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"356'7766","reported_seq":12606,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:21.659877+0000","last_change":"2023-10-09T04:20:05.060638+0000","last_active":"2023-10-09T04:20:21.659877+0000","last_peered":"2023-10-09T04:20:21.659877+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:20:02.106154+0000","last_became_peered":"2023-10-09T04:20:02.106154+0000","last_unstale":"2023-10-09T04:20:21.659877+0000","last_undegraded":"2023-10-09T04:20:21.659877+0000","last_fullsized":"2023-10-09T04:20:21.659877+0000","mapping_epoch":348,"log_start":"314'1500","ondisk_log_start":"314'1500","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6266,"log_dups_size":0,"ondisk_log_size":6266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:45.805086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302907392,"num_objects":4622,"num_object_clones":0,"num_object_copies":13866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7309,"num_objects_unfound":0,"num_objects_dirty":4622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7766,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2377,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":4622}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":339,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999415+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:20:11.999415+0000","last_peered":"2023-10-09T04:20:11.999415+0000","last_clean":"2023-10-09T04:20:11.999415+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:20:11.999415+0000","last_undegraded":"2023-10-09T04:20:11.999415+0000","last_fullsized":"2023-10-09T04:20:11.999415+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":750,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997065+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:20:11.997065+0000","last_peered":"2023-10-09T04:20:11.997065+0000","last_clean":"2023-10-09T04:20:11.997065+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:20:11.997065+0000","last_undegraded":"2023-10-09T04:20:11.997065+0000","last_fullsized":"2023-10-09T04:20:11.997065+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":282,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999992+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:20:11.999992+0000","last_peered":"2023-10-09T04:20:11.999992+0000","last_clean":"2023-10-09T04:20:11.999992+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:20:11.999992+0000","last_undegraded":"2023-10-09T04:20:11.999992+0000","last_fullsized":"2023-10-09T04:20:11.999992+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.027060+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:20:12.027060+0000","last_peered":"2023-10-09T04:20:12.027060+0000","last_clean":"2023-10-09T04:20:12.027060+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:20:12.027060+0000","last_undegraded":"2023-10-09T04:20:12.027060+0000","last_fullsized":"2023-10-09T04:20:12.027060+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"356'7734","reported_seq":11373,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:21.663926+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:20:21.663926+0000","last_peered":"2023-10-09T04:20:21.663926+0000","last_clean":"2023-10-09T04:20:21.663926+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:20:21.663926+0000","last_undegraded":"2023-10-09T04:20:21.663926+0000","last_fullsized":"2023-10-09T04:20:21.663926+0000","mapping_epoch":323,"log_start":"314'1400","ondisk_log_start":"314'1400","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6334,"log_dups_size":0,"ondisk_log_size":6334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301465600,"num_objects":4600,"num_object_clones":0,"num_object_copies":13800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7734,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":499,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:13.241898+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:20:13.241898+0000","last_peered":"2023-10-09T04:20:13.241898+0000","last_clean":"2023-10-09T04:20:13.241898+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:20:13.241898+0000","last_undegraded":"2023-10-09T04:20:13.241898+0000","last_fullsized":"2023-10-09T04:20:13.241898+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":336,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.996979+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:20:11.996979+0000","last_peered":"2023-10-09T04:20:11.996979+0000","last_clean":"2023-10-09T04:20:11.996979+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:20:11.996979+0000","last_undegraded":"2023-10-09T04:20:11.996979+0000","last_fullsized":"2023-10-09T04:20:11.996979+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":351,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.034101+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:20:12.034101+0000","last_peered":"2023-10-09T04:20:12.034101+0000","last_clean":"2023-10-09T04:20:12.034101+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:20:12.034101+0000","last_undegraded":"2023-10-09T04:20:12.034101+0000","last_fullsized":"2023-10-09T04:20:12.034101+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":348,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.031649+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:20:12.031649+0000","last_peered":"2023-10-09T04:20:12.031649+0000","last_clean":"2023-10-09T04:20:12.031649+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:20:12.031649+0000","last_undegraded":"2023-10-09T04:20:12.031649+0000","last_fullsized":"2023-10-09T04:20:12.031649+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"356'7758","reported_seq":7816,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.871666+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:20:20.871666+0000","last_peered":"2023-10-09T04:20:20.871666+0000","last_clean":"2023-10-09T04:20:20.871666+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:20:20.871666+0000","last_undegraded":"2023-10-09T04:20:20.871666+0000","last_fullsized":"2023-10-09T04:20:20.871666+0000","mapping_epoch":306,"log_start":"319'2400","ondisk_log_start":"319'2400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309067776,"num_objects":4716,"num_object_clones":0,"num_object_copies":14148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7758,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":509,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026806+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:20:12.026806+0000","last_peered":"2023-10-09T04:20:12.026806+0000","last_clean":"2023-10-09T04:20:12.026806+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:20:12.026806+0000","last_undegraded":"2023-10-09T04:20:12.026806+0000","last_fullsized":"2023-10-09T04:20:12.026806+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":339,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997811+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:20:11.997811+0000","last_peered":"2023-10-09T04:20:11.997811+0000","last_clean":"2023-10-09T04:20:11.997811+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:20:11.997811+0000","last_undegraded":"2023-10-09T04:20:11.997811+0000","last_fullsized":"2023-10-09T04:20:11.997811+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":329,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999809+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:20:11.999809+0000","last_peered":"2023-10-09T04:20:11.999809+0000","last_clean":"2023-10-09T04:20:11.999809+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:20:11.999809+0000","last_undegraded":"2023-10-09T04:20:11.999809+0000","last_fullsized":"2023-10-09T04:20:11.999809+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999333+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:20:11.999333+0000","last_peered":"2023-10-09T04:20:11.999333+0000","last_clean":"2023-10-09T04:20:11.999333+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:20:11.999333+0000","last_undegraded":"2023-10-09T04:20:11.999333+0000","last_fullsized":"2023-10-09T04:20:11.999333+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"356'7668","reported_seq":19069,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:20.882792+0000","last_change":"2023-10-09T04:20:12.010618+0000","last_active":"2023-10-09T04:20:20.882792+0000","last_peered":"2023-10-09T04:20:20.882792+0000","last_clean":"2023-10-09T04:20:20.882792+0000","last_became_active":"2023-10-09T04:20:12.002110+0000","last_became_peered":"2023-10-09T04:20:12.002110+0000","last_unstale":"2023-10-09T04:20:20.882792+0000","last_undegraded":"2023-10-09T04:20:20.882792+0000","last_fullsized":"2023-10-09T04:20:20.882792+0000","mapping_epoch":354,"log_start":"319'2400","ondisk_log_start":"319'2400","created":306,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:12.102225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304283672,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7668,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":500,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026856+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:20:12.026856+0000","last_peered":"2023-10-09T04:20:12.026856+0000","last_clean":"2023-10-09T04:20:12.026856+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:20:12.026856+0000","last_undegraded":"2023-10-09T04:20:12.026856+0000","last_fullsized":"2023-10-09T04:20:12.026856+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":288,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997885+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:20:11.997885+0000","last_peered":"2023-10-09T04:20:11.997885+0000","last_clean":"2023-10-09T04:20:11.997885+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:20:11.997885+0000","last_undegraded":"2023-10-09T04:20:11.997885+0000","last_fullsized":"2023-10-09T04:20:11.997885+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":323,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998504+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:20:11.998504+0000","last_peered":"2023-10-09T04:20:11.998504+0000","last_clean":"2023-10-09T04:20:11.998504+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:20:11.998504+0000","last_undegraded":"2023-10-09T04:20:11.998504+0000","last_fullsized":"2023-10-09T04:20:11.998504+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":306,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997167+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:20:11.997167+0000","last_peered":"2023-10-09T04:20:11.997167+0000","last_clean":"2023-10-09T04:20:11.997167+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:20:11.997167+0000","last_undegraded":"2023-10-09T04:20:11.997167+0000","last_fullsized":"2023-10-09T04:20:11.997167+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"356'8047","reported_seq":8115,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:25.431014+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:20:25.431014+0000","last_peered":"2023-10-09T04:20:25.431014+0000","last_clean":"2023-10-09T04:20:25.431014+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:20:25.431014+0000","last_undegraded":"2023-10-09T04:20:25.431014+0000","last_fullsized":"2023-10-09T04:20:25.431014+0000","mapping_epoch":310,"log_start":"319'2400","ondisk_log_start":"319'2400","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295108608,"num_objects":4503,"num_object_clones":0,"num_object_copies":13509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8047,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":494,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.593794+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:20:18.593794+0000","last_peered":"2023-10-09T04:20:18.593794+0000","last_clean":"2023-10-09T04:20:18.593794+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:20:18.593794+0000","last_undegraded":"2023-10-09T04:20:18.593794+0000","last_fullsized":"2023-10-09T04:20:18.593794+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":188,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:14.383683+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:20:14.383683+0000","last_peered":"2023-10-09T04:20:14.383683+0000","last_clean":"2023-10-09T04:20:14.383683+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:20:14.383683+0000","last_undegraded":"2023-10-09T04:20:14.383683+0000","last_fullsized":"2023-10-09T04:20:14.383683+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":361,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.031744+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:20:12.031744+0000","last_peered":"2023-10-09T04:20:12.031744+0000","last_clean":"2023-10-09T04:20:12.031744+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:20:12.031744+0000","last_undegraded":"2023-10-09T04:20:12.031744+0000","last_fullsized":"2023-10-09T04:20:12.031744+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025134+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:20:12.025134+0000","last_peered":"2023-10-09T04:20:12.025134+0000","last_clean":"2023-10-09T04:20:12.025134+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:20:12.025134+0000","last_undegraded":"2023-10-09T04:20:12.025134+0000","last_fullsized":"2023-10-09T04:20:12.025134+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":514,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.033585+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:20:12.033585+0000","last_peered":"2023-10-09T04:20:12.033585+0000","last_clean":"2023-10-09T04:20:12.033585+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:20:12.033585+0000","last_undegraded":"2023-10-09T04:20:12.033585+0000","last_fullsized":"2023-10-09T04:20:12.033585+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":330,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.032301+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:20:12.032301+0000","last_peered":"2023-10-09T04:20:12.032301+0000","last_clean":"2023-10-09T04:20:12.032301+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:20:12.032301+0000","last_undegraded":"2023-10-09T04:20:12.032301+0000","last_fullsized":"2023-10-09T04:20:12.032301+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":190,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.013924+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:20:12.013924+0000","last_peered":"2023-10-09T04:20:12.013924+0000","last_clean":"2023-10-09T04:20:12.013924+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:20:12.013924+0000","last_undegraded":"2023-10-09T04:20:12.013924+0000","last_fullsized":"2023-10-09T04:20:12.013924+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":743,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997607+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:20:11.997607+0000","last_peered":"2023-10-09T04:20:11.997607+0000","last_clean":"2023-10-09T04:20:11.997607+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:20:11.997607+0000","last_undegraded":"2023-10-09T04:20:11.997607+0000","last_fullsized":"2023-10-09T04:20:11.997607+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":345,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999884+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:20:11.999884+0000","last_peered":"2023-10-09T04:20:11.999884+0000","last_clean":"2023-10-09T04:20:11.999884+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:20:11.999884+0000","last_undegraded":"2023-10-09T04:20:11.999884+0000","last_fullsized":"2023-10-09T04:20:11.999884+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.026411+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:20:12.026411+0000","last_peered":"2023-10-09T04:20:12.026411+0000","last_clean":"2023-10-09T04:20:12.026411+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:20:12.026411+0000","last_undegraded":"2023-10-09T04:20:12.026411+0000","last_fullsized":"2023-10-09T04:20:12.026411+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":342,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.999472+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:20:11.999472+0000","last_peered":"2023-10-09T04:20:11.999472+0000","last_clean":"2023-10-09T04:20:11.999472+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:20:11.999472+0000","last_undegraded":"2023-10-09T04:20:11.999472+0000","last_fullsized":"2023-10-09T04:20:11.999472+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025645+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:20:12.025645+0000","last_peered":"2023-10-09T04:20:12.025645+0000","last_clean":"2023-10-09T04:20:12.025645+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:20:12.025645+0000","last_undegraded":"2023-10-09T04:20:12.025645+0000","last_fullsized":"2023-10-09T04:20:12.025645+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":778,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.997253+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:20:11.997253+0000","last_peered":"2023-10-09T04:20:11.997253+0000","last_clean":"2023-10-09T04:20:11.997253+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:20:11.997253+0000","last_undegraded":"2023-10-09T04:20:11.997253+0000","last_fullsized":"2023-10-09T04:20:11.997253+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":200,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.016200+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:20:12.016200+0000","last_peered":"2023-10-09T04:20:12.016200+0000","last_clean":"2023-10-09T04:20:12.016200+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:20:12.016200+0000","last_undegraded":"2023-10-09T04:20:12.016200+0000","last_fullsized":"2023-10-09T04:20:12.016200+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":191,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.587178+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:20:18.587178+0000","last_peered":"2023-10-09T04:20:18.587178+0000","last_clean":"2023-10-09T04:20:18.587178+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:20:18.587178+0000","last_undegraded":"2023-10-09T04:20:18.587178+0000","last_fullsized":"2023-10-09T04:20:18.587178+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":497,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:11.998468+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:20:11.998468+0000","last_peered":"2023-10-09T04:20:11.998468+0000","last_clean":"2023-10-09T04:20:11.998468+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:20:11.998468+0000","last_undegraded":"2023-10-09T04:20:11.998468+0000","last_fullsized":"2023-10-09T04:20:11.998468+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":349,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.032284+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:20:12.032284+0000","last_peered":"2023-10-09T04:20:12.032284+0000","last_clean":"2023-10-09T04:20:12.032284+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:20:12.032284+0000","last_undegraded":"2023-10-09T04:20:12.032284+0000","last_fullsized":"2023-10-09T04:20:12.032284+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:20:12.025040+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:20:12.025040+0000","last_peered":"2023-10-09T04:20:12.025040+0000","last_clean":"2023-10-09T04:20:12.025040+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:20:12.025040+0000","last_undegraded":"2023-10-09T04:20:12.025040+0000","last_fullsized":"2023-10-09T04:20:12.025040+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4819386392,"num_objects":73539,"num_object_clones":0,"num_object_copies":220617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":14242,"num_objects_unfound":0,"num_objects_dirty":73539,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123909,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15815,"num_bytes_recovered":1036386328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14937243648,"data_stored":14937227360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91788,"ondisk_log_size":91788,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201614,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3887092,"kb_used_data":3293228,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":89858060,"statfs":{"total":95995035648,"available":92014653440,"internally_reserved":0,"allocated":3372265472,"data_stored":3371262259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.4089999999999998,"15min":2.9249999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":16.440999999999999,"5min":16.440999999999999,"15min":126.752},"last":0.70999999999999996},{"interface":"front","average":{"1min":3.1869999999999998,"5min":2.2189999999999999,"15min":2.8820000000000001},"min":{"1min":0.27300000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":16.462,"5min":16.462,"15min":128.88399999999999},"last":4.2770000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2430000000000003,"5min":4.3739999999999997,"15min":4.21},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":29.792000000000002,"5min":103.324,"15min":245.28700000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":4.9909999999999997,"5min":4.6210000000000004,"15min":4.1660000000000004},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":29.268000000000001,"5min":104.248,"15min":244.626},"last":0.47599999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.7849999999999999,"15min":2.0979999999999999},"min":{"1min":0.30499999999999999,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.194,"5min":14.076000000000001,"15min":76.774000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.556,"15min":2.794},"min":{"1min":0.35799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":6.1749999999999998,"5min":15.597,"15min":99.372},"last":0.86599999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":2.6419999999999999,"15min":2.633},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":3.7149999999999999,"5min":85.575000000000003,"15min":99.134},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.1080000000000001,"15min":2.8769999999999998},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":6.1459999999999999,"5min":85.528000000000006,"15min":99.117999999999995},"last":0.61899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.86899999999999999,"15min":0.82799999999999996},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.8529999999999998,"5min":8.9359999999999999,"15min":16.195},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6160000000000001,"15min":1.1120000000000001},"min":{"1min":0.27700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.375}]},{"osd":5,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.5680000000000001,"15min":1.321},"min":{"1min":0.29299999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":6.9390000000000001,"5min":120.066,"15min":120.066},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.85299999999999998,"5min":2.2530000000000001,"15min":1.2789999999999999},"min":{"1min":0.34399999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8899999999999997,"5min":120.09699999999999,"15min":120.09699999999999},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.4590000000000001,"15min":1.0249999999999999},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":8.4550000000000001,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.96999999999999997,"15min":0.88300000000000001},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.605,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.45100000000000001}]}]},{"osd":6,"up_from":0,"seq":201863463252,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089560,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3501456,"kb_used_data":2931272,"kb_used_omap":9,"kb_used_meta":570166,"kb_avail":90243696,"statfs":{"total":95995035648,"available":92409544704,"internally_reserved":0,"allocated":3001622528,"data_stored":3000126684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":583850985},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,9],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":3.8759999999999999,"15min":4.0629999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":19.617000000000001,"5min":98.257999999999996,"15min":98.257999999999996},"last":0.246},{"interface":"front","average":{"1min":2.1949999999999998,"5min":6.3319999999999999,"15min":5.2000000000000002},"min":{"1min":0.27100000000000002,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":9.9960000000000004,"5min":154.654,"15min":154.654},"last":0.30199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":2.1970000000000001,"15min":2.6019999999999999},"min":{"1min":0.26600000000000001,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":19.564,"5min":19.564,"15min":121.54600000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":2,"5min":2.476,"15min":2.9239999999999999},"min":{"1min":0.22500000000000001,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":12.382999999999999,"5min":16.114999999999998,"15min":99.899000000000001},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.8660000000000001,"15min":3.7770000000000001},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":19.640000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":0.95399999999999996},{"interface":"front","average":{"1min":2.4609999999999999,"5min":3.8170000000000002,"15min":4.6970000000000001},"min":{"1min":0.307,"5min":0.251,"15min":0.251},"max":{"1min":12.353999999999999,"5min":68.563999999999993,"15min":138.49799999999999},"last":0.52200000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.2290000000000001,"15min":2.3370000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":7.5250000000000004,"5min":16.27,"15min":99.989999999999995},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.458,"5min":1.9530000000000001,"15min":2.1829999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":5.282,"5min":13.428000000000001,"15min":100.249},"last":0.66400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":3.899,"5min":3.2280000000000002,"15min":2.6539999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":25.696999999999999,"5min":38.848999999999997,"15min":99.933999999999997},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.8700000000000001,"5min":3.145,"15min":2.6909999999999998},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":19.977,"5min":39.593000000000004,"15min":100.34099999999999},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.2120000000000002,"15min":2.5},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":5.3579999999999997,"5min":13.377000000000001,"15min":100.309},"last":0.71699999999999997},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.9570000000000001,"15min":2.1749999999999998},"min":{"1min":0.41399999999999998,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":100.143},"last":0.99299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.7050000000000001,"15min":2.6419999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":5.7430000000000003,"5min":17.460000000000001,"15min":100.214},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.3359999999999999,"15min":2.319},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":19.593,"5min":19.593,"15min":100.113},"last":1.016}]}]},{"osd":0,"up_from":84,"seq":360777253086,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1992920,"kb_used_data":1475088,"kb_used_omap":28,"kb_used_meta":517795,"kb_avail":91752232,"statfs":{"total":95995035648,"available":93954285568,"internally_reserved":0,"allocated":1510490112,"data_stored":1508926609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":530222928},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":8.9990000000000006,"15min":5.2709999999999999},"min":{"1min":0.20200000000000001,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":6.1609999999999996,"5min":209.215,"15min":209.215},"last":0.40500000000000003},{"interface":"front","average":{"1min":3.0499999999999998,"5min":6.8630000000000004,"15min":5.032},"min":{"1min":0.29499999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.547000000000001,"5min":120.934,"15min":120.934},"last":0.89400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.3359999999999999,"15min":3.141},"min":{"1min":0.32900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.025,"5min":17.466999999999999,"15min":118.54000000000001},"last":0.498},{"interface":"front","average":{"1min":2.0609999999999999,"5min":3.2429999999999999,"15min":3.536},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.20300000000000001},"max":{"1min":6.7460000000000004,"5min":20.262,"15min":118.727},"last":0.62}]},{"osd":3,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":3.1400000000000001,"15min":5.2759999999999998},"min":{"1min":0.25700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":9.1669999999999998,"5min":99.709000000000003,"15min":156.488},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.3919999999999999,"5min":3.1720000000000002,"15min":5.3620000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":10.577,"5min":99.560000000000002,"15min":156.50700000000001},"last":0.54100000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":4.899,"5min":2.9969999999999999,"15min":2.7250000000000001},"min":{"1min":0.35399999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":49.840000000000003,"5min":63.375999999999998,"15min":118.633},"last":0.71199999999999997},{"interface":"front","average":{"1min":5.0609999999999999,"5min":3.3290000000000002,"15min":3.0670000000000002},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":49.607999999999997,"5min":49.607999999999997,"15min":118.913},"last":0.80000000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":5.4800000000000004,"15min":4.2960000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.0570000000000004,"5min":156.96100000000001,"15min":182.68899999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":2.0870000000000002,"5min":4.5540000000000003,"15min":3.7749999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.024,"5min":156.97300000000001,"15min":182.66300000000001},"last":0.748}]},{"osd":6,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.528,"15min":2.996},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":7.0709999999999997,"5min":63.468000000000004,"15min":137.36099999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.206,"5min":2.5859999999999999,"15min":3.0329999999999999},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":10.113,"5min":63.447000000000003,"15min":136.88999999999999},"last":0.73699999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.6419999999999999,"15min":3.677},"min":{"1min":0.40000000000000002,"5min":0.375,"15min":0.32100000000000001},"max":{"1min":9.1699999999999999,"5min":63.509,"15min":185.024},"last":2.1800000000000002},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.7770000000000001,"15min":3.3479999999999999},"min":{"1min":0.44700000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":10.077,"5min":63.481999999999999,"15min":184.98599999999999},"last":0.67500000000000004}]}]},{"osd":2,"up_from":94,"seq":403726926034,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2027492,"kb_used_data":1455388,"kb_used_omap":81,"kb_used_meta":572014,"kb_avail":91717660,"statfs":{"total":95995035648,"available":93918883840,"internally_reserved":0,"allocated":1490317312,"data_stored":1488806143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":585742554},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":6.3810000000000002,"15min":3.875},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":5.0780000000000003,"5min":81.385999999999996,"15min":81.385999999999996},"last":17.122},{"interface":"front","average":{"1min":1.8,"5min":5.7300000000000004,"15min":3.899},"min":{"1min":0.22600000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":10.457000000000001,"5min":79.453999999999994,"15min":79.453999999999994},"last":4.0250000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":11.619,"5min":6.9870000000000001,"15min":4.391},"min":{"1min":0.27600000000000002,"5min":0.191,"15min":0.191},"max":{"1min":158.13399999999999,"5min":158.13399999999999,"15min":158.13399999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":11.401999999999999,"5min":7.7629999999999999,"15min":4.6639999999999997},"min":{"1min":0.253,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":161.12,"5min":161.12,"15min":161.12},"last":12.829000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.4700000000000002,"15min":2.8170000000000002},"min":{"1min":0.28000000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.409000000000001,"5min":58.997,"15min":58.997},"last":1.145},{"interface":"front","average":{"1min":2.1499999999999999,"5min":3.5030000000000001,"15min":3.0270000000000001},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.488,"5min":66.135000000000005,"15min":66.135000000000005},"last":0.40000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":2.6139999999999999,"15min":2.1309999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":6.0700000000000003,"5min":58.798000000000002,"15min":58.798000000000002},"last":12.775},{"interface":"front","average":{"1min":1.0509999999999999,"5min":2.7290000000000001,"15min":2.0190000000000001},"min":{"1min":0.32000000000000001,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":3.1659999999999999,"5min":56.616,"15min":56.616},"last":1.1919999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.8620000000000001,"15min":2.327},"min":{"1min":0.28299999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.241,"5min":56.524000000000001,"15min":56.524000000000001},"last":0.375},{"interface":"front","average":{"1min":3.625,"5min":3.157,"15min":2.456},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":51.472000000000001,"5min":58.854999999999997,"15min":58.854999999999997},"last":12.663}]},{"osd":6,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":2.9729999999999999,"15min":2.5249999999999999},"min":{"1min":0.39600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.8650000000000002,"5min":56.561,"15min":114.14},"last":1.214},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.5579999999999998,"15min":2.5859999999999999},"min":{"1min":0.32100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.218,"5min":56.460999999999999,"15min":114.123},"last":0.502}]},{"osd":7,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":3.0350000000000001,"15min":2.7290000000000001},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.8479999999999999,"5min":56.591000000000001,"15min":117.32599999999999},"last":1.294},{"interface":"front","average":{"1min":1.6950000000000001,"5min":2.7429999999999999,"15min":2.7599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.189,"5min":56.497999999999998,"15min":117.28700000000001},"last":0.46999999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762508,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2357440,"kb_used_data":1619192,"kb_used_omap":7,"kb_used_meta":738232,"kb_avail":91387712,"statfs":{"total":95995035648,"available":93581017088,"internally_reserved":0,"allocated":1658052608,"data_stored":1656570670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":755950184},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":3.3730000000000002,"15min":5.4059999999999997},"min":{"1min":0.214,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":6.4169999999999998,"5min":78.771000000000001,"15min":125.765},"last":7.6680000000000001},{"interface":"front","average":{"1min":2.899,"5min":4.2080000000000002,"15min":5.8280000000000003},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.119999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":12.276}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":5.8979999999999997,"15min":5.2619999999999996},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":8.3230000000000004,"5min":103.143,"15min":186.21299999999999},"last":12.792999999999999},{"interface":"front","average":{"1min":1.6060000000000001,"5min":5.7789999999999999,"15min":5.0309999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.193},"max":{"1min":5.9219999999999997,"5min":105.19,"15min":191.78},"last":12.69}]},{"osd":2,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.1660000000000004,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.8760000000000003,"5min":127.599,"15min":127.599},"last":4.9640000000000004},{"interface":"front","average":{"1min":2.359,"5min":6.0670000000000002,"15min":5.0119999999999996},"min":{"1min":0.29799999999999999,"5min":0.221,"15min":0.214},"max":{"1min":10.143000000000001,"5min":129.98599999999999,"15min":129.98599999999999},"last":5.0039999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":2.274,"15min":2.7240000000000002},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.013999999999999,"5min":27.539999999999999,"15min":106.85899999999999},"last":4.0289999999999999},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.2919999999999998,"15min":2.1760000000000002},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.738,"5min":26.663,"15min":37.442},"last":0.309}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.6280000000000001,"15min":2.2010000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":3.7749999999999999,"5min":17.901,"15min":17.901},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.083,"5min":1.766,"15min":1.887},"min":{"1min":0.375,"5min":0.248,"15min":0.248},"max":{"1min":4.6340000000000003,"5min":18.068000000000001,"15min":18.068000000000001},"last":4.101}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.7509999999999999,"15min":2.2610000000000001},"min":{"1min":0.378,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4589999999999996,"5min":14.567,"15min":106.036},"last":4.0620000000000003},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.48,"15min":2.6509999999999998},"min":{"1min":0.42699999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.071,"5min":14.529,"15min":106.741},"last":4.1159999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":2.6720000000000002,"15min":2.786},"min":{"1min":0.39000000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":28.184000000000001,"5min":28.184000000000001,"15min":106.82599999999999},"last":4.0830000000000002},{"interface":"front","average":{"1min":3.1480000000000001,"5min":2.827,"15min":2.7490000000000001},"min":{"1min":0.44800000000000001,"5min":0.40000000000000002,"15min":0.35999999999999999},"max":{"1min":26.648,"5min":26.648,"15min":106.084},"last":0.33300000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986524,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737700,"kb_used_data":1440412,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92007452,"statfs":{"total":95995035648,"available":94215630848,"internally_reserved":0,"allocated":1474981888,"data_stored":1473994090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":10.717000000000001,"5min":10.526,"15min":5.4820000000000002},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":132.631,"5min":154.62100000000001,"15min":154.62100000000001},"last":2.0430000000000001},{"interface":"front","average":{"1min":10.893000000000001,"5min":10.782,"15min":5.5069999999999997},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":132.59100000000001,"5min":154.37299999999999,"15min":154.37299999999999},"last":4.0179999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":7.8129999999999997,"5min":9.4969999999999999,"15min":5.1180000000000003},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":69.706000000000003,"5min":135.25399999999999,"15min":135.25399999999999},"last":1.667},{"interface":"front","average":{"1min":7.3220000000000001,"5min":9.1270000000000007,"15min":5.0060000000000002},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":69.561000000000007,"5min":134.44300000000001,"15min":134.44300000000001},"last":1.724}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.679,"15min":2.0720000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.245},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":123.598},"last":1.5940000000000001},{"interface":"front","average":{"1min":1.708,"5min":1.546,"15min":1.986},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":6.21,"5min":8.4800000000000004,"15min":123.619},"last":1.6040000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.052,"15min":2.29},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.0069999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":2.7829999999999999},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.165,"15min":2.4849999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.7579999999999991,"5min":8.9169999999999998,"15min":127.90000000000001},"last":1.8120000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":3.6099999999999999,"15min":1.6279999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":13.599,"5min":111.809,"15min":111.809},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.532,"5min":3.7200000000000002,"15min":1.7929999999999999},"min":{"1min":0.316,"5min":0.252,"15min":0.25},"max":{"1min":9.1530000000000005,"5min":112.002,"15min":112.002},"last":0.27900000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.395,"15min":1.3680000000000001},"min":{"1min":0.35699999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":6.9470000000000001,"5min":76.307000000000002,"15min":76.307000000000002},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.877,"5min":2.2130000000000001,"15min":1.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9320000000000004,"5min":78.421999999999997,"15min":78.421999999999997},"last":0.36199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:19:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.3049999999999999,"15min":0.95299999999999996},"min":{"1min":0.31900000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.8010000000000002,"5min":34.039000000000001,"15min":34.039000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.5109999999999999,"15min":1.0680000000000001},"min":{"1min":0.375,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.384,"5min":33.433999999999997,"15min":33.433999999999997},"last":0.501}]}]},{"osd":5,"up_from":39,"seq":167503724887,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3074848,"kb_used_data":2476888,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":90670304,"statfs":{"total":95995035648,"available":92846391296,"internally_reserved":0,"allocated":2536333312,"data_stored":2535346020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":4.984,"15min":5.0759999999999996},"min":{"1min":0.223,"5min":0.193,"15min":0.193},"max":{"1min":13.675000000000001,"5min":118.98399999999999,"15min":151.23599999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.4079999999999999,"5min":5.3380000000000001,"15min":5.4470000000000001},"min":{"1min":0.23499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":13.885,"5min":118.846,"15min":151.566},"last":0.79100000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":5.242,"15min":4.0270000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":29.632999999999999,"5min":111.194,"15min":111.194},"last":0.86699999999999999},{"interface":"front","average":{"1min":4.7249999999999996,"5min":5.8289999999999997,"15min":4.4180000000000001},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":29.672000000000001,"5min":112.226,"15min":112.226},"last":2.1110000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.444,"15min":3.1299999999999999},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":8.8599999999999994,"5min":11.324,"15min":106.508},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.718,"15min":2.8879999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":3.8199999999999998,"5min":8.5280000000000005,"15min":106.462},"last":5.444}]},{"osd":3,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.0449999999999999,"15min":2.5939999999999999},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.555999999999999,"5min":11.555999999999999,"15min":91.834999999999994},"last":5.516},{"interface":"front","average":{"1min":2.3260000000000001,"5min":2.2389999999999999,"15min":2.6890000000000001},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3219999999999992,"5min":12.122999999999999,"15min":91.802000000000007},"last":5.5259999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.242,"15min":0.93300000000000005},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.246},"max":{"1min":9.2579999999999991,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.1200000000000001,"15min":0.92900000000000005},"min":{"1min":0.30199999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":3.77,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.33100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.27,"15min":1.3779999999999999},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":9.4090000000000007,"5min":12.068,"15min":123.239},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.615,"5min":1.45,"15min":1.4670000000000001},"min":{"1min":0.313,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":9.7720000000000002,"15min":123.014},"last":0.46500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.202,"15min":0.995},"min":{"1min":0.33100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.3289999999999997,"5min":12.096,"15min":12.096},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.24,"5min":1.446,"15min":1.095},"min":{"1min":0.36699999999999999,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":9.3019999999999996,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.373}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1500905472,"data_stored":1500905472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2979598336,"data_stored":2979594264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1472135168,"data_stored":1472135168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1645608960,"data_stored":1645608960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1468076032,"data_stored":1468071960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2523467776,"data_stored":2523463704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3346993152,"data_stored":3346989080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:20:28.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:28.528 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:28.595 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:28.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:28.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:28.706 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:28.737 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:28.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:28.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:28.834 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:20:28.835 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:20:28.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:28.997 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:29.060 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:29.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:28 smithi163 ceph-mon[221067]: pgmap v958: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 13632/212379 objects misplaced (6.419%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:29.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:28 smithi163 ceph-mon[213841]: pgmap v958: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 13632/212379 objects misplaced (6.419%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:29.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:29.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:29.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:28 smithi181 ceph-mon[196979]: pgmap v958: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 13632/212379 objects misplaced (6.419%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:20:29.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:29.342 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:29.416 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:29.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:29.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:29.548 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:29.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:29.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:29.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:29.747 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:29.795 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:29.880 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:29.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:29.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:29.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:29 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/670455724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:29.926 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:29.970 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:20:29.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:30.028 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:30.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:20:30.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:30.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:30.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:30.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:29 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/670455724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:30.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:29 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/670455724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:30.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:30.299 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:30.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:30.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: pgmap v959: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 13093/210282 objects misplaced (6.226%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: osdmap e357: 8 total, 8 up, 7 in 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: failed to encode map e357 with expected crc 2023-10-09T04:20:30.928 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[213841]: failed to encode map e357 with expected crc 2023-10-09T04:20:30.931 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:20:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:20:30] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:20:30.933 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:31.009 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: pgmap v959: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 13093/210282 objects misplaced (6.226%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: osdmap e357: 8 total, 8 up, 7 in 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: failed to encode map e357 with expected crc 2023-10-09T04:20:31.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:30 smithi163 ceph-mon[221067]: failed to encode map e357 with expected crc 2023-10-09T04:20:31.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: pgmap v959: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 13093/210282 objects misplaced (6.226%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:20:31.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:20:31.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:20:31.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:31.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: osdmap e357: 8 total, 8 up, 7 in 2023-10-09T04:20:31.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: failed to encode map e357 with expected crc 2023-10-09T04:20:31.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:30 smithi181 ceph-mon[196979]: failed to encode map e357 with expected crc 2023-10-09T04:20:31.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:20:31.831 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:31.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:31.945 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:32.043 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:32.055 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:32.191 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:32.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:32.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:32.405 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:32.500 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:32.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:32.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:32.617 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:32.653 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:32.731 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:32.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:32.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:32.880 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:32.884 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:20:32.928 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:20:32.950 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:32.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:32 smithi181 ceph-mon[196979]: pgmap v961: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 12454/200430 objects misplaced (6.214%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:20:32.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:32 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/728885338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:32.965 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:20:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:32.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:33.005 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:20:33.060 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:20:33.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:33.092 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:20:33.113 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:33.126 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:20:33.162 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:20:33.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:33.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:33.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.191 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.192 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:20:33.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:32 smithi163 ceph-mon[213841]: pgmap v961: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 12454/200430 objects misplaced (6.214%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:20:33.220 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:32 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/728885338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:33.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:32 smithi163 ceph-mon[221067]: pgmap v961: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 12454/200430 objects misplaced (6.214%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:20:33.222 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:32 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/728885338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:33.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:33.355 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:33.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:20:33.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:33.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:34.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:34.079 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:34.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:34.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:34.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:34.268 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:34.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:34.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:34.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:34.492 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:34.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:34.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:34.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:34.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:34.701 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:34.769 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:34.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:34.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:34.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:34.925 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:34.958 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:34 smithi181 ceph-mon[196979]: pgmap v962: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 11915/198177 objects misplaced (6.012%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:34.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:34.959 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:35.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:35.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:35.135 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:35.161 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:35.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:35.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:35.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:34 smithi163 ceph-mon[213841]: pgmap v962: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 11915/198177 objects misplaced (6.012%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:35.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:34 smithi163 ceph-mon[221067]: pgmap v962: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 11915/198177 objects misplaced (6.012%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:35.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:35.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:35.370 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:35.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:35.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:35.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:35.642 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:35.742 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:35.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:35.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:35.857 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:35.948 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:35.963 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:20:35.964 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:20:35.964 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:20:35.964 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:20:35.964 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:20:36.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:36.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:36.364 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:36.406 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:36.492 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:36.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:36.601 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:36.651 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:36.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:20:36.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:36.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:36.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:36.925 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:36 smithi181 ceph-mon[196979]: pgmap v963: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 11915/197331 objects misplaced (6.038%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:36 smithi163 ceph-mon[221067]: pgmap v963: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 11915/197331 objects misplaced (6.038%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:37.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:36 smithi163 ceph-mon[213841]: pgmap v963: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 11915/197331 objects misplaced (6.038%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:37.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:20:37.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:37.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:37.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:37.407 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:37.478 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:37.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:37.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:37.600 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:37.614 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:37.625 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:20:37.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:37.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:37.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:37.837 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:37.899 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:37.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:37.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:38.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:38.034 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:38.704 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:38.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:38 smithi163 ceph-mon[213841]: from='client.55705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:38.709 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:38 smithi163 ceph-mon[213841]: pgmap v964: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 11262/187953 objects misplaced (5.992%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:38.709 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:38 smithi163 ceph-mon[221067]: from='client.55705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:38.709 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:38 smithi163 ceph-mon[221067]: pgmap v964: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 11262/187953 objects misplaced (5.992%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:38.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:38.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:38.891 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:38.976 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":963,"stamp":"2023-10-09T04:20:35.842974+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4309795001,"num_objects":65777,"num_object_clones":0,"num_object_copies":197331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11915,"num_objects_unfound":0,"num_objects_dirty":65777,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":131788,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16745,"num_bytes_recovered":1093510881,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96820,"ondisk_log_size":96820,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17090328,"kb_used_data":13181280,"kb_used_omap":126,"kb_used_meta":3908417,"kb_avail":639125736,"statfs":{"total":671965249536,"available":654464753664,"internally_reserved":0,"allocated":13497630720,"data_stored":13488572362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4002219643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-510263296,"num_objects":-7786,"num_object_clones":0,"num_object_copies":-23358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2327,"num_objects_unfound":0,"num_objects_dirty":-7786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7786,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":863,"num_bytes_recovered":56557568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004167"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":481,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567866+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:20:18.567866+0000","last_peered":"2023-10-09T04:20:18.567866+0000","last_clean":"2023-10-09T04:20:18.567866+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:20:18.567866+0000","last_undegraded":"2023-10-09T04:20:18.567866+0000","last_fullsized":"2023-10-09T04:20:18.567866+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":101,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559282+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:20:18.559282+0000","last_peered":"2023-10-09T04:20:18.559282+0000","last_clean":"2023-10-09T04:20:18.559282+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:20:18.559282+0000","last_undegraded":"2023-10-09T04:20:18.559282+0000","last_fullsized":"2023-10-09T04:20:18.559282+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":332,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.564685+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:20:18.564685+0000","last_peered":"2023-10-09T04:20:18.564685+0000","last_clean":"2023-10-09T04:20:18.564685+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:20:18.564685+0000","last_undegraded":"2023-10-09T04:20:18.564685+0000","last_fullsized":"2023-10-09T04:20:18.564685+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":474,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570337+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:20:18.570337+0000","last_peered":"2023-10-09T04:20:18.570337+0000","last_clean":"2023-10-09T04:20:18.570337+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:20:18.570337+0000","last_undegraded":"2023-10-09T04:20:18.570337+0000","last_fullsized":"2023-10-09T04:20:18.570337+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.564704+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:20:18.564704+0000","last_peered":"2023-10-09T04:20:18.564704+0000","last_clean":"2023-10-09T04:20:18.564704+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:20:18.564704+0000","last_undegraded":"2023-10-09T04:20:18.564704+0000","last_fullsized":"2023-10-09T04:20:18.564704+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.589806+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:20:18.589806+0000","last_peered":"2023-10-09T04:20:18.589806+0000","last_clean":"2023-10-09T04:20:18.589806+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:20:18.589806+0000","last_undegraded":"2023-10-09T04:20:18.589806+0000","last_fullsized":"2023-10-09T04:20:18.589806+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":481,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559039+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:20:18.559039+0000","last_peered":"2023-10-09T04:20:18.559039+0000","last_clean":"2023-10-09T04:20:18.559039+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:20:18.559039+0000","last_undegraded":"2023-10-09T04:20:18.559039+0000","last_fullsized":"2023-10-09T04:20:18.559039+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":332,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.568127+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:20:18.568127+0000","last_peered":"2023-10-09T04:20:18.568127+0000","last_clean":"2023-10-09T04:20:18.568127+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:20:18.568127+0000","last_undegraded":"2023-10-09T04:20:18.568127+0000","last_fullsized":"2023-10-09T04:20:18.568127+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":300,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.571150+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:20:18.571150+0000","last_peered":"2023-10-09T04:20:18.571150+0000","last_clean":"2023-10-09T04:20:18.571150+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:20:18.571150+0000","last_undegraded":"2023-10-09T04:20:18.571150+0000","last_fullsized":"2023-10-09T04:20:18.571150+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":756,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553838+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:20:18.553838+0000","last_peered":"2023-10-09T04:20:18.553838+0000","last_clean":"2023-10-09T04:20:18.553838+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:20:18.553838+0000","last_undegraded":"2023-10-09T04:20:18.553838+0000","last_fullsized":"2023-10-09T04:20:18.553838+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":333,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.586670+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:20:18.586670+0000","last_peered":"2023-10-09T04:20:18.586670+0000","last_clean":"2023-10-09T04:20:18.586670+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:20:18.586670+0000","last_undegraded":"2023-10-09T04:20:18.586670+0000","last_fullsized":"2023-10-09T04:20:18.586670+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:29.952676+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:20:29.952676+0000","last_peered":"2023-10-09T04:20:29.952676+0000","last_clean":"2023-10-09T04:20:29.952676+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:20:29.952676+0000","last_undegraded":"2023-10-09T04:20:29.952676+0000","last_fullsized":"2023-10-09T04:20:29.952676+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":483,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:24.392695+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:20:24.392695+0000","last_peered":"2023-10-09T04:20:24.392695+0000","last_clean":"2023-10-09T04:20:24.392695+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:20:24.392695+0000","last_undegraded":"2023-10-09T04:20:24.392695+0000","last_fullsized":"2023-10-09T04:20:24.392695+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":316,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570956+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:20:18.570956+0000","last_peered":"2023-10-09T04:20:18.570956+0000","last_clean":"2023-10-09T04:20:18.570956+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:20:18.570956+0000","last_undegraded":"2023-10-09T04:20:18.570956+0000","last_fullsized":"2023-10-09T04:20:18.570956+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":51,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567722+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:20:18.567722+0000","last_peered":"2023-10-09T04:20:18.567722+0000","last_clean":"2023-10-09T04:20:18.567722+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:20:18.567722+0000","last_undegraded":"2023-10-09T04:20:18.567722+0000","last_fullsized":"2023-10-09T04:20:18.567722+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":491,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.571033+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:20:18.571033+0000","last_peered":"2023-10-09T04:20:18.571033+0000","last_clean":"2023-10-09T04:20:18.571033+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:20:18.571033+0000","last_undegraded":"2023-10-09T04:20:18.571033+0000","last_fullsized":"2023-10-09T04:20:18.571033+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":399,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.554406+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:20:18.554406+0000","last_peered":"2023-10-09T04:20:18.554406+0000","last_clean":"2023-10-09T04:20:18.554406+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:20:18.554406+0000","last_undegraded":"2023-10-09T04:20:18.554406+0000","last_fullsized":"2023-10-09T04:20:18.554406+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":35,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.564930+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:20:18.564930+0000","last_peered":"2023-10-09T04:20:18.564930+0000","last_clean":"2023-10-09T04:20:18.564930+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:20:18.564930+0000","last_undegraded":"2023-10-09T04:20:18.564930+0000","last_fullsized":"2023-10-09T04:20:18.564930+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":499,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:29.961636+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:20:29.961636+0000","last_peered":"2023-10-09T04:20:29.961636+0000","last_clean":"2023-10-09T04:20:29.961636+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:20:29.961636+0000","last_undegraded":"2023-10-09T04:20:29.961636+0000","last_fullsized":"2023-10-09T04:20:29.961636+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":131,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559551+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:20:18.559551+0000","last_peered":"2023-10-09T04:20:18.559551+0000","last_clean":"2023-10-09T04:20:18.559551+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:20:18.559551+0000","last_undegraded":"2023-10-09T04:20:18.559551+0000","last_fullsized":"2023-10-09T04:20:18.559551+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559703+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:20:18.559703+0000","last_peered":"2023-10-09T04:20:18.559703+0000","last_clean":"2023-10-09T04:20:18.559703+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:20:18.559703+0000","last_undegraded":"2023-10-09T04:20:18.559703+0000","last_fullsized":"2023-10-09T04:20:18.559703+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":507,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:21.752583+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:20:21.752583+0000","last_peered":"2023-10-09T04:20:21.752583+0000","last_clean":"2023-10-09T04:20:21.752583+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:20:21.752583+0000","last_undegraded":"2023-10-09T04:20:21.752583+0000","last_fullsized":"2023-10-09T04:20:21.752583+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":350,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553626+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:20:18.553626+0000","last_peered":"2023-10-09T04:20:18.553626+0000","last_clean":"2023-10-09T04:20:18.553626+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:20:18.553626+0000","last_undegraded":"2023-10-09T04:20:18.553626+0000","last_fullsized":"2023-10-09T04:20:18.553626+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":35,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.577058+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:20:18.577058+0000","last_peered":"2023-10-09T04:20:18.577058+0000","last_clean":"2023-10-09T04:20:18.577058+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:20:18.577058+0000","last_undegraded":"2023-10-09T04:20:18.577058+0000","last_fullsized":"2023-10-09T04:20:18.577058+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":501,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567686+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:20:18.567686+0000","last_peered":"2023-10-09T04:20:18.567686+0000","last_clean":"2023-10-09T04:20:18.567686+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:20:18.567686+0000","last_undegraded":"2023-10-09T04:20:18.567686+0000","last_fullsized":"2023-10-09T04:20:18.567686+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":333,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.566194+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:20:18.566194+0000","last_peered":"2023-10-09T04:20:18.566194+0000","last_clean":"2023-10-09T04:20:18.566194+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:20:18.566194+0000","last_undegraded":"2023-10-09T04:20:18.566194+0000","last_fullsized":"2023-10-09T04:20:18.566194+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":335,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570736+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:20:18.570736+0000","last_peered":"2023-10-09T04:20:18.570736+0000","last_clean":"2023-10-09T04:20:18.570736+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:20:18.570736+0000","last_undegraded":"2023-10-09T04:20:18.570736+0000","last_fullsized":"2023-10-09T04:20:18.570736+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":499,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:28.286284+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:20:28.286284+0000","last_peered":"2023-10-09T04:20:28.286284+0000","last_clean":"2023-10-09T04:20:28.286284+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:20:28.286284+0000","last_undegraded":"2023-10-09T04:20:28.286284+0000","last_fullsized":"2023-10-09T04:20:28.286284+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.558936+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:20:18.558936+0000","last_peered":"2023-10-09T04:20:18.558936+0000","last_clean":"2023-10-09T04:20:18.558936+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:20:18.558936+0000","last_undegraded":"2023-10-09T04:20:18.558936+0000","last_fullsized":"2023-10-09T04:20:18.558936+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":333,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.589914+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:20:18.589914+0000","last_peered":"2023-10-09T04:20:18.589914+0000","last_clean":"2023-10-09T04:20:18.589914+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:20:18.589914+0000","last_undegraded":"2023-10-09T04:20:18.589914+0000","last_fullsized":"2023-10-09T04:20:18.589914+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":501,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.568120+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:20:18.568120+0000","last_peered":"2023-10-09T04:20:18.568120+0000","last_clean":"2023-10-09T04:20:18.568120+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:20:18.568120+0000","last_undegraded":"2023-10-09T04:20:18.568120+0000","last_fullsized":"2023-10-09T04:20:18.568120+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":346,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.568008+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:20:18.568008+0000","last_peered":"2023-10-09T04:20:18.568008+0000","last_clean":"2023-10-09T04:20:18.568008+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:20:18.568008+0000","last_undegraded":"2023-10-09T04:20:18.568008+0000","last_fullsized":"2023-10-09T04:20:18.568008+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":19,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.595378+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:20:18.595378+0000","last_peered":"2023-10-09T04:20:18.595378+0000","last_clean":"2023-10-09T04:20:18.595378+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:20:18.595378+0000","last_undegraded":"2023-10-09T04:20:18.595378+0000","last_fullsized":"2023-10-09T04:20:18.595378+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"357'8114","reported_seq":8175,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:31.722675+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:20:31.722675+0000","last_peered":"2023-10-09T04:20:31.722675+0000","last_clean":"2023-10-09T04:20:31.722675+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:20:31.722675+0000","last_undegraded":"2023-10-09T04:20:31.722675+0000","last_fullsized":"2023-10-09T04:20:31.722675+0000","mapping_epoch":306,"log_start":"317'1800","ondisk_log_start":"317'1800","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6314,"log_dups_size":0,"ondisk_log_size":6314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266469376,"num_objects":4066,"num_object_clones":0,"num_object_copies":12198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8114,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":759,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553988+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:20:18.553988+0000","last_peered":"2023-10-09T04:20:18.553988+0000","last_clean":"2023-10-09T04:20:18.553988+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:20:18.553988+0000","last_undegraded":"2023-10-09T04:20:18.553988+0000","last_fullsized":"2023-10-09T04:20:18.553988+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":345,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553947+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:20:18.553947+0000","last_peered":"2023-10-09T04:20:18.553947+0000","last_clean":"2023-10-09T04:20:18.553947+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:20:18.553947+0000","last_undegraded":"2023-10-09T04:20:18.553947+0000","last_fullsized":"2023-10-09T04:20:18.553947+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":308,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567853+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:20:18.567853+0000","last_peered":"2023-10-09T04:20:18.567853+0000","last_clean":"2023-10-09T04:20:18.567853+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:20:18.567853+0000","last_undegraded":"2023-10-09T04:20:18.567853+0000","last_fullsized":"2023-10-09T04:20:18.567853+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"357'8320","reported_seq":8390,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:33.261620+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:20:33.261620+0000","last_peered":"2023-10-09T04:20:33.261620+0000","last_clean":"2023-10-09T04:20:33.261620+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:20:33.261620+0000","last_undegraded":"2023-10-09T04:20:33.261620+0000","last_fullsized":"2023-10-09T04:20:33.261620+0000","mapping_epoch":310,"log_start":"322'2973","ondisk_log_start":"322'2973","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5347,"log_dups_size":0,"ondisk_log_size":5347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267124736,"num_objects":4076,"num_object_clones":0,"num_object_copies":12228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8320,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":524,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:29.957082+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:20:29.957082+0000","last_peered":"2023-10-09T04:20:29.957082+0000","last_clean":"2023-10-09T04:20:29.957082+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:20:29.957082+0000","last_undegraded":"2023-10-09T04:20:29.957082+0000","last_fullsized":"2023-10-09T04:20:29.957082+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":349,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553987+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:20:18.553987+0000","last_peered":"2023-10-09T04:20:18.553987+0000","last_clean":"2023-10-09T04:20:18.553987+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:20:18.553987+0000","last_undegraded":"2023-10-09T04:20:18.553987+0000","last_fullsized":"2023-10-09T04:20:18.553987+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":131,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559021+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:20:18.559021+0000","last_peered":"2023-10-09T04:20:18.559021+0000","last_clean":"2023-10-09T04:20:18.559021+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:20:18.559021+0000","last_undegraded":"2023-10-09T04:20:18.559021+0000","last_fullsized":"2023-10-09T04:20:18.559021+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"357'8368","reported_seq":8438,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:30.832935+0000","last_change":"2023-10-09T04:20:03.101091+0000","last_active":"2023-10-09T04:20:30.832935+0000","last_peered":"2023-10-09T04:20:30.832935+0000","last_clean":"2023-10-09T04:20:00.944119+0000","last_became_active":"2023-10-09T04:20:03.092473+0000","last_became_peered":"2023-10-09T04:20:03.092473+0000","last_unstale":"2023-10-09T04:20:30.832935+0000","last_undegraded":"2023-10-09T04:20:30.832935+0000","last_fullsized":"2023-10-09T04:20:30.832935+0000","mapping_epoch":349,"log_start":"319'2418","ondisk_log_start":"319'2418","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5950,"log_dups_size":0,"ondisk_log_size":5950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:49.431905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4208,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8368,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":4208}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":491,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570955+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:20:18.570955+0000","last_peered":"2023-10-09T04:20:18.570955+0000","last_clean":"2023-10-09T04:20:18.570955+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:20:18.570955+0000","last_undegraded":"2023-10-09T04:20:18.570955+0000","last_fullsized":"2023-10-09T04:20:18.570955+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":314,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.554321+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:20:18.554321+0000","last_peered":"2023-10-09T04:20:18.554321+0000","last_clean":"2023-10-09T04:20:18.554321+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:20:18.554321+0000","last_undegraded":"2023-10-09T04:20:18.554321+0000","last_fullsized":"2023-10-09T04:20:18.554321+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":289,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.565405+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:20:18.565405+0000","last_peered":"2023-10-09T04:20:18.565405+0000","last_clean":"2023-10-09T04:20:18.565405+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:20:18.565405+0000","last_undegraded":"2023-10-09T04:20:18.565405+0000","last_fullsized":"2023-10-09T04:20:18.565405+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"357'8186","reported_seq":8245,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.830457+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:20:30.830457+0000","last_peered":"2023-10-09T04:20:30.830457+0000","last_clean":"2023-10-09T04:20:30.830457+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:20:30.830457+0000","last_undegraded":"2023-10-09T04:20:30.830457+0000","last_fullsized":"2023-10-09T04:20:30.830457+0000","mapping_epoch":306,"log_start":"319'2287","ondisk_log_start":"319'2287","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271187968,"num_objects":4138,"num_object_clones":0,"num_object_copies":12414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8186,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":501,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559993+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:20:18.559993+0000","last_peered":"2023-10-09T04:20:18.559993+0000","last_clean":"2023-10-09T04:20:18.559993+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:20:18.559993+0000","last_undegraded":"2023-10-09T04:20:18.559993+0000","last_fullsized":"2023-10-09T04:20:18.559993+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":346,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570600+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:20:18.570600+0000","last_peered":"2023-10-09T04:20:18.570600+0000","last_clean":"2023-10-09T04:20:18.570600+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:20:18.570600+0000","last_undegraded":"2023-10-09T04:20:18.570600+0000","last_fullsized":"2023-10-09T04:20:18.570600+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":334,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.584500+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:20:18.584500+0000","last_peered":"2023-10-09T04:20:18.584500+0000","last_clean":"2023-10-09T04:20:18.584500+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:20:18.584500+0000","last_undegraded":"2023-10-09T04:20:18.584500+0000","last_fullsized":"2023-10-09T04:20:18.584500+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"357'7856","reported_seq":11288,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.836623+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:20:30.836623+0000","last_peered":"2023-10-09T04:20:30.836623+0000","last_clean":"2023-10-09T04:20:30.836623+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:20:30.836623+0000","last_undegraded":"2023-10-09T04:20:30.836623+0000","last_fullsized":"2023-10-09T04:20:30.836623+0000","mapping_epoch":320,"log_start":"319'2500","ondisk_log_start":"319'2500","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5356,"log_dups_size":0,"ondisk_log_size":5356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263585792,"num_objects":4022,"num_object_clones":0,"num_object_copies":12066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7856,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":780,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553599+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:20:18.553599+0000","last_peered":"2023-10-09T04:20:18.553599+0000","last_clean":"2023-10-09T04:20:18.553599+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:20:18.553599+0000","last_undegraded":"2023-10-09T04:20:18.553599+0000","last_fullsized":"2023-10-09T04:20:18.553599+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":317,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.584597+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:20:18.584597+0000","last_peered":"2023-10-09T04:20:18.584597+0000","last_clean":"2023-10-09T04:20:18.584597+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:20:18.584597+0000","last_undegraded":"2023-10-09T04:20:18.584597+0000","last_fullsized":"2023-10-09T04:20:18.584597+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559633+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:20:18.559633+0000","last_peered":"2023-10-09T04:20:18.559633+0000","last_clean":"2023-10-09T04:20:18.559633+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:20:18.559633+0000","last_undegraded":"2023-10-09T04:20:18.559633+0000","last_fullsized":"2023-10-09T04:20:18.559633+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"357'8226","reported_seq":11796,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:33.297176+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:20:33.297176+0000","last_peered":"2023-10-09T04:20:33.297176+0000","last_clean":"2023-10-09T04:20:33.297176+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:20:33.297176+0000","last_undegraded":"2023-10-09T04:20:33.297176+0000","last_fullsized":"2023-10-09T04:20:33.297176+0000","mapping_epoch":329,"log_start":"322'2960","ondisk_log_start":"322'2960","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5266,"log_dups_size":0,"ondisk_log_size":5266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265420800,"num_objects":4050,"num_object_clones":0,"num_object_copies":12150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8226,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":526,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.565885+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:20:18.565885+0000","last_peered":"2023-10-09T04:20:18.565885+0000","last_clean":"2023-10-09T04:20:18.565885+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:20:18.565885+0000","last_undegraded":"2023-10-09T04:20:18.565885+0000","last_fullsized":"2023-10-09T04:20:18.565885+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559127+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:20:18.559127+0000","last_peered":"2023-10-09T04:20:18.559127+0000","last_clean":"2023-10-09T04:20:18.559127+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:20:18.559127+0000","last_undegraded":"2023-10-09T04:20:18.559127+0000","last_fullsized":"2023-10-09T04:20:18.559127+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559997+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:20:18.559997+0000","last_peered":"2023-10-09T04:20:18.559997+0000","last_clean":"2023-10-09T04:20:18.559997+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:20:18.559997+0000","last_undegraded":"2023-10-09T04:20:18.559997+0000","last_fullsized":"2023-10-09T04:20:18.559997+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"357'8101","reported_seq":8170,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.905158+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:20:30.905158+0000","last_peered":"2023-10-09T04:20:30.905158+0000","last_clean":"2023-10-09T04:20:30.905158+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:20:30.905158+0000","last_undegraded":"2023-10-09T04:20:30.905158+0000","last_fullsized":"2023-10-09T04:20:30.905158+0000","mapping_epoch":310,"log_start":"321'2800","ondisk_log_start":"321'2800","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5301,"log_dups_size":0,"ondisk_log_size":5301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265093120,"num_objects":4045,"num_object_clones":0,"num_object_copies":12135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8101,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":482,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:23.280886+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:20:23.280886+0000","last_peered":"2023-10-09T04:20:23.280886+0000","last_clean":"2023-10-09T04:20:23.280886+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:20:23.280886+0000","last_undegraded":"2023-10-09T04:20:23.280886+0000","last_fullsized":"2023-10-09T04:20:23.280886+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":101,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.558860+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:20:18.558860+0000","last_peered":"2023-10-09T04:20:18.558860+0000","last_clean":"2023-10-09T04:20:18.558860+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:20:18.558860+0000","last_undegraded":"2023-10-09T04:20:18.558860+0000","last_fullsized":"2023-10-09T04:20:18.558860+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.560081+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:20:18.560081+0000","last_peered":"2023-10-09T04:20:18.560081+0000","last_clean":"2023-10-09T04:20:18.560081+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:20:18.560081+0000","last_undegraded":"2023-10-09T04:20:18.560081+0000","last_fullsized":"2023-10-09T04:20:18.560081+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"357'8345","reported_seq":15312,"reported_epoch":357,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:20:33.301494+0000","last_change":"2023-10-09T04:20:05.061955+0000","last_active":"2023-10-09T04:20:33.301494+0000","last_peered":"2023-10-09T04:20:33.301494+0000","last_clean":"2023-10-09T04:20:00.980108+0000","last_became_active":"2023-10-09T04:20:03.079675+0000","last_became_peered":"2023-10-09T04:20:03.079675+0000","last_unstale":"2023-10-09T04:20:33.301494+0000","last_undegraded":"2023-10-09T04:20:33.301494+0000","last_fullsized":"2023-10-09T04:20:33.301494+0000","mapping_epoch":349,"log_start":"324'3081","ondisk_log_start":"324'3081","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:58.283644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267190272,"num_objects":4077,"num_object_clones":0,"num_object_copies":12231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1155,"num_objects_unfound":0,"num_objects_dirty":4077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8345,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3448,"num_bytes_recovered":225968128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4077}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":501,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567854+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:20:18.567854+0000","last_peered":"2023-10-09T04:20:18.567854+0000","last_clean":"2023-10-09T04:20:18.567854+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:20:18.567854+0000","last_undegraded":"2023-10-09T04:20:18.567854+0000","last_fullsized":"2023-10-09T04:20:18.567854+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":19,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.595983+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:20:18.595983+0000","last_peered":"2023-10-09T04:20:18.595983+0000","last_clean":"2023-10-09T04:20:18.595983+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:20:18.595983+0000","last_undegraded":"2023-10-09T04:20:18.595983+0000","last_fullsized":"2023-10-09T04:20:18.595983+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":24,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.595080+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:20:18.595080+0000","last_peered":"2023-10-09T04:20:18.595080+0000","last_clean":"2023-10-09T04:20:18.595080+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:20:18.595080+0000","last_undegraded":"2023-10-09T04:20:18.595080+0000","last_fullsized":"2023-10-09T04:20:18.595080+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"357'8228","reported_seq":8297,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.817311+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:20:30.817311+0000","last_peered":"2023-10-09T04:20:30.817311+0000","last_clean":"2023-10-09T04:20:30.817311+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:20:30.817311+0000","last_undegraded":"2023-10-09T04:20:30.817311+0000","last_fullsized":"2023-10-09T04:20:30.817311+0000","mapping_epoch":310,"log_start":"319'2334","ondisk_log_start":"319'2334","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274071552,"num_objects":4182,"num_object_clones":0,"num_object_copies":12546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8228,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":521,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.571238+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:20:18.571238+0000","last_peered":"2023-10-09T04:20:18.571238+0000","last_clean":"2023-10-09T04:20:18.571238+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:20:18.571238+0000","last_undegraded":"2023-10-09T04:20:18.571238+0000","last_fullsized":"2023-10-09T04:20:18.571238+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":336,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.577507+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:20:18.577507+0000","last_peered":"2023-10-09T04:20:18.577507+0000","last_clean":"2023-10-09T04:20:18.577507+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:20:18.577507+0000","last_undegraded":"2023-10-09T04:20:18.577507+0000","last_fullsized":"2023-10-09T04:20:18.577507+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":353,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570661+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:20:18.570661+0000","last_peered":"2023-10-09T04:20:18.570661+0000","last_clean":"2023-10-09T04:20:18.570661+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:20:18.570661+0000","last_undegraded":"2023-10-09T04:20:18.570661+0000","last_fullsized":"2023-10-09T04:20:18.570661+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.588057+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:20:18.588057+0000","last_peered":"2023-10-09T04:20:18.588057+0000","last_clean":"2023-10-09T04:20:18.588057+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:20:18.588057+0000","last_undegraded":"2023-10-09T04:20:18.588057+0000","last_fullsized":"2023-10-09T04:20:18.588057+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"357'8211","reported_seq":8272,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:31.746595+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:20:31.746595+0000","last_peered":"2023-10-09T04:20:31.746595+0000","last_clean":"2023-10-09T04:20:31.746595+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:20:31.746595+0000","last_undegraded":"2023-10-09T04:20:31.746595+0000","last_fullsized":"2023-10-09T04:20:31.746595+0000","mapping_epoch":306,"log_start":"318'1900","ondisk_log_start":"318'1900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6311,"log_dups_size":0,"ondisk_log_size":6311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268107776,"num_objects":4091,"num_object_clones":0,"num_object_copies":12273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8211,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":474,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567875+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:20:18.567875+0000","last_peered":"2023-10-09T04:20:18.567875+0000","last_clean":"2023-10-09T04:20:18.567875+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:20:18.567875+0000","last_undegraded":"2023-10-09T04:20:18.567875+0000","last_fullsized":"2023-10-09T04:20:18.567875+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":330,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.567991+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:20:18.567991+0000","last_peered":"2023-10-09T04:20:18.567991+0000","last_clean":"2023-10-09T04:20:18.567991+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:20:18.567991+0000","last_undegraded":"2023-10-09T04:20:18.567991+0000","last_fullsized":"2023-10-09T04:20:18.567991+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559624+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:20:18.559624+0000","last_peered":"2023-10-09T04:20:18.559624+0000","last_clean":"2023-10-09T04:20:18.559624+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:20:18.559624+0000","last_undegraded":"2023-10-09T04:20:18.559624+0000","last_fullsized":"2023-10-09T04:20:18.559624+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":364,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.577433+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:20:18.577433+0000","last_peered":"2023-10-09T04:20:18.577433+0000","last_clean":"2023-10-09T04:20:18.577433+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:20:18.577433+0000","last_undegraded":"2023-10-09T04:20:18.577433+0000","last_fullsized":"2023-10-09T04:20:18.577433+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"357'8244","reported_seq":8303,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.857046+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:20:30.857046+0000","last_peered":"2023-10-09T04:20:30.857046+0000","last_clean":"2023-10-09T04:20:30.857046+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:20:30.857046+0000","last_undegraded":"2023-10-09T04:20:30.857046+0000","last_fullsized":"2023-10-09T04:20:30.857046+0000","mapping_epoch":306,"log_start":"319'2268","ondisk_log_start":"319'2268","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5976,"log_dups_size":0,"ondisk_log_size":5976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277610496,"num_objects":4236,"num_object_clones":0,"num_object_copies":12708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8244,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":490,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.566238+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:20:18.566238+0000","last_peered":"2023-10-09T04:20:18.566238+0000","last_clean":"2023-10-09T04:20:18.566238+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:20:18.566238+0000","last_undegraded":"2023-10-09T04:20:18.566238+0000","last_fullsized":"2023-10-09T04:20:18.566238+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":192,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.587124+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:20:18.587124+0000","last_peered":"2023-10-09T04:20:18.587124+0000","last_clean":"2023-10-09T04:20:18.587124+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:20:18.587124+0000","last_undegraded":"2023-10-09T04:20:18.587124+0000","last_fullsized":"2023-10-09T04:20:18.587124+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":339,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.565032+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:20:18.565032+0000","last_peered":"2023-10-09T04:20:18.565032+0000","last_clean":"2023-10-09T04:20:18.565032+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:20:18.565032+0000","last_undegraded":"2023-10-09T04:20:18.565032+0000","last_fullsized":"2023-10-09T04:20:18.565032+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":35,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.594271+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:20:18.594271+0000","last_peered":"2023-10-09T04:20:18.594271+0000","last_clean":"2023-10-09T04:20:18.594271+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:20:18.594271+0000","last_undegraded":"2023-10-09T04:20:18.594271+0000","last_fullsized":"2023-10-09T04:20:18.594271+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"357'8249","reported_seq":13090,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:31.716035+0000","last_change":"2023-10-09T04:20:05.060638+0000","last_active":"2023-10-09T04:20:31.716035+0000","last_peered":"2023-10-09T04:20:31.716035+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:20:02.106154+0000","last_became_peered":"2023-10-09T04:20:02.106154+0000","last_unstale":"2023-10-09T04:20:31.716035+0000","last_undegraded":"2023-10-09T04:20:31.716035+0000","last_fullsized":"2023-10-09T04:20:31.716035+0000","mapping_epoch":348,"log_start":"318'1900","ondisk_log_start":"318'1900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6349,"log_dups_size":0,"ondisk_log_size":6349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:45.805086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271253504,"num_objects":4139,"num_object_clones":0,"num_object_copies":12417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6552,"num_objects_unfound":0,"num_objects_dirty":4139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8249,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2377,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":4139}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":340,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.568151+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:20:18.568151+0000","last_peered":"2023-10-09T04:20:18.568151+0000","last_clean":"2023-10-09T04:20:18.568151+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:20:18.568151+0000","last_undegraded":"2023-10-09T04:20:18.568151+0000","last_fullsized":"2023-10-09T04:20:18.568151+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":751,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553408+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:20:18.553408+0000","last_peered":"2023-10-09T04:20:18.553408+0000","last_clean":"2023-10-09T04:20:18.553408+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:20:18.553408+0000","last_undegraded":"2023-10-09T04:20:18.553408+0000","last_fullsized":"2023-10-09T04:20:18.553408+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":283,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.568031+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:20:18.568031+0000","last_peered":"2023-10-09T04:20:18.568031+0000","last_clean":"2023-10-09T04:20:18.568031+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:20:18.568031+0000","last_undegraded":"2023-10-09T04:20:18.568031+0000","last_fullsized":"2023-10-09T04:20:18.568031+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.560120+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:20:18.560120+0000","last_peered":"2023-10-09T04:20:18.560120+0000","last_clean":"2023-10-09T04:20:18.560120+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:20:18.560120+0000","last_undegraded":"2023-10-09T04:20:18.560120+0000","last_fullsized":"2023-10-09T04:20:18.560120+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"357'8229","reported_seq":11869,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:31.757787+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:20:31.757787+0000","last_peered":"2023-10-09T04:20:31.757787+0000","last_clean":"2023-10-09T04:20:31.757787+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:20:31.757787+0000","last_undegraded":"2023-10-09T04:20:31.757787+0000","last_fullsized":"2023-10-09T04:20:31.757787+0000","mapping_epoch":323,"log_start":"318'1900","ondisk_log_start":"318'1900","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269025280,"num_objects":4105,"num_object_clones":0,"num_object_copies":12315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8229,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":500,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.579652+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:20:18.579652+0000","last_peered":"2023-10-09T04:20:18.579652+0000","last_clean":"2023-10-09T04:20:18.579652+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:20:18.579652+0000","last_undegraded":"2023-10-09T04:20:18.579652+0000","last_fullsized":"2023-10-09T04:20:18.579652+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":337,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553315+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:20:18.553315+0000","last_peered":"2023-10-09T04:20:18.553315+0000","last_clean":"2023-10-09T04:20:18.553315+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:20:18.553315+0000","last_undegraded":"2023-10-09T04:20:18.553315+0000","last_fullsized":"2023-10-09T04:20:18.553315+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":352,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.566525+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:20:18.566525+0000","last_peered":"2023-10-09T04:20:18.566525+0000","last_clean":"2023-10-09T04:20:18.566525+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:20:18.566525+0000","last_undegraded":"2023-10-09T04:20:18.566525+0000","last_fullsized":"2023-10-09T04:20:18.566525+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":349,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.589634+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:20:18.589634+0000","last_peered":"2023-10-09T04:20:18.589634+0000","last_clean":"2023-10-09T04:20:18.589634+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:20:18.589634+0000","last_undegraded":"2023-10-09T04:20:18.589634+0000","last_fullsized":"2023-10-09T04:20:18.589634+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"357'8240","reported_seq":8299,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.900653+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:20:30.900653+0000","last_peered":"2023-10-09T04:20:30.900653+0000","last_clean":"2023-10-09T04:20:30.900653+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:20:30.900653+0000","last_undegraded":"2023-10-09T04:20:30.900653+0000","last_fullsized":"2023-10-09T04:20:30.900653+0000","mapping_epoch":306,"log_start":"322'2900","ondisk_log_start":"322'2900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277479424,"num_objects":4234,"num_object_clones":0,"num_object_copies":12702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":510,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559901+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:20:18.559901+0000","last_peered":"2023-10-09T04:20:18.559901+0000","last_clean":"2023-10-09T04:20:18.559901+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:20:18.559901+0000","last_undegraded":"2023-10-09T04:20:18.559901+0000","last_fullsized":"2023-10-09T04:20:18.559901+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":340,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.554362+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:20:18.554362+0000","last_peered":"2023-10-09T04:20:18.554362+0000","last_clean":"2023-10-09T04:20:18.554362+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:20:18.554362+0000","last_undegraded":"2023-10-09T04:20:18.554362+0000","last_fullsized":"2023-10-09T04:20:18.554362+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":330,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.577588+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:20:18.577588+0000","last_peered":"2023-10-09T04:20:18.577588+0000","last_clean":"2023-10-09T04:20:18.577588+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:20:18.577588+0000","last_undegraded":"2023-10-09T04:20:18.577588+0000","last_fullsized":"2023-10-09T04:20:18.577588+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.568125+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:20:18.568125+0000","last_peered":"2023-10-09T04:20:18.568125+0000","last_clean":"2023-10-09T04:20:18.568125+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:20:18.568125+0000","last_undegraded":"2023-10-09T04:20:18.568125+0000","last_fullsized":"2023-10-09T04:20:18.568125+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"357'8190","reported_seq":19592,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:30.903526+0000","last_change":"2023-10-09T04:20:12.010618+0000","last_active":"2023-10-09T04:20:30.903526+0000","last_peered":"2023-10-09T04:20:30.903526+0000","last_clean":"2023-10-09T04:20:30.903526+0000","last_became_active":"2023-10-09T04:20:12.002110+0000","last_became_peered":"2023-10-09T04:20:12.002110+0000","last_unstale":"2023-10-09T04:20:30.903526+0000","last_undegraded":"2023-10-09T04:20:30.903526+0000","last_fullsized":"2023-10-09T04:20:30.903526+0000","mapping_epoch":354,"log_start":"321'2900","ondisk_log_start":"321'2900","created":306,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5290,"log_dups_size":0,"ondisk_log_size":5290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:12.102225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270073880,"num_objects":4122,"num_object_clones":0,"num_object_copies":12366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4122,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8190,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":501,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559826+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:20:18.559826+0000","last_peered":"2023-10-09T04:20:18.559826+0000","last_clean":"2023-10-09T04:20:18.559826+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:20:18.559826+0000","last_undegraded":"2023-10-09T04:20:18.559826+0000","last_fullsized":"2023-10-09T04:20:18.559826+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":289,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.554180+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:20:18.554180+0000","last_peered":"2023-10-09T04:20:18.554180+0000","last_clean":"2023-10-09T04:20:18.554180+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:20:18.554180+0000","last_undegraded":"2023-10-09T04:20:18.554180+0000","last_fullsized":"2023-10-09T04:20:18.554180+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":324,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.591447+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:20:18.591447+0000","last_peered":"2023-10-09T04:20:18.591447+0000","last_clean":"2023-10-09T04:20:18.591447+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:20:18.591447+0000","last_undegraded":"2023-10-09T04:20:18.591447+0000","last_fullsized":"2023-10-09T04:20:18.591447+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":307,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553853+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:20:18.553853+0000","last_peered":"2023-10-09T04:20:18.553853+0000","last_clean":"2023-10-09T04:20:18.553853+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:20:18.553853+0000","last_undegraded":"2023-10-09T04:20:18.553853+0000","last_fullsized":"2023-10-09T04:20:18.553853+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"357'8588","reported_seq":8657,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-10-09T04:20:35.410798+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:20:35.410798+0000","last_peered":"2023-10-09T04:20:35.410798+0000","last_clean":"2023-10-09T04:20:35.410798+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:20:35.410798+0000","last_undegraded":"2023-10-09T04:20:35.410798+0000","last_fullsized":"2023-10-09T04:20:35.410798+0000","mapping_epoch":310,"log_start":"322'3000","ondisk_log_start":"322'3000","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5588,"log_dups_size":0,"ondisk_log_size":5588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259653632,"num_objects":3962,"num_object_clones":0,"num_object_copies":11886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8588,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":495,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:25.263645+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:20:25.263645+0000","last_peered":"2023-10-09T04:20:25.263645+0000","last_clean":"2023-10-09T04:20:25.263645+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:20:25.263645+0000","last_undegraded":"2023-10-09T04:20:25.263645+0000","last_fullsized":"2023-10-09T04:20:25.263645+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":189,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.595026+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:20:18.595026+0000","last_peered":"2023-10-09T04:20:18.595026+0000","last_clean":"2023-10-09T04:20:18.595026+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:20:18.595026+0000","last_undegraded":"2023-10-09T04:20:18.595026+0000","last_fullsized":"2023-10-09T04:20:18.595026+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":362,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.589731+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:20:18.589731+0000","last_peered":"2023-10-09T04:20:18.589731+0000","last_clean":"2023-10-09T04:20:18.589731+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:20:18.589731+0000","last_undegraded":"2023-10-09T04:20:18.589731+0000","last_fullsized":"2023-10-09T04:20:18.589731+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.558980+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:20:18.558980+0000","last_peered":"2023-10-09T04:20:18.558980+0000","last_clean":"2023-10-09T04:20:18.558980+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:20:18.558980+0000","last_undegraded":"2023-10-09T04:20:18.558980+0000","last_fullsized":"2023-10-09T04:20:18.558980+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":516,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:26.753157+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:20:26.753157+0000","last_peered":"2023-10-09T04:20:26.753157+0000","last_clean":"2023-10-09T04:20:26.753157+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:20:26.753157+0000","last_undegraded":"2023-10-09T04:20:26.753157+0000","last_fullsized":"2023-10-09T04:20:26.753157+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":331,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.584683+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:20:18.584683+0000","last_peered":"2023-10-09T04:20:18.584683+0000","last_clean":"2023-10-09T04:20:18.584683+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:20:18.584683+0000","last_undegraded":"2023-10-09T04:20:18.584683+0000","last_fullsized":"2023-10-09T04:20:18.584683+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.587520+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:20:18.587520+0000","last_peered":"2023-10-09T04:20:18.587520+0000","last_clean":"2023-10-09T04:20:18.587520+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:20:18.587520+0000","last_undegraded":"2023-10-09T04:20:18.587520+0000","last_fullsized":"2023-10-09T04:20:18.587520+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":744,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553900+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:20:18.553900+0000","last_peered":"2023-10-09T04:20:18.553900+0000","last_clean":"2023-10-09T04:20:18.553900+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:20:18.553900+0000","last_undegraded":"2023-10-09T04:20:18.553900+0000","last_fullsized":"2023-10-09T04:20:18.553900+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":346,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570680+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:20:18.570680+0000","last_peered":"2023-10-09T04:20:18.570680+0000","last_clean":"2023-10-09T04:20:18.570680+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:20:18.570680+0000","last_undegraded":"2023-10-09T04:20:18.570680+0000","last_fullsized":"2023-10-09T04:20:18.570680+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":121,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559797+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:20:18.559797+0000","last_peered":"2023-10-09T04:20:18.559797+0000","last_clean":"2023-10-09T04:20:18.559797+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:20:18.559797+0000","last_undegraded":"2023-10-09T04:20:18.559797+0000","last_fullsized":"2023-10-09T04:20:18.559797+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":343,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.570638+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:20:18.570638+0000","last_peered":"2023-10-09T04:20:18.570638+0000","last_clean":"2023-10-09T04:20:18.570638+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:20:18.570638+0000","last_undegraded":"2023-10-09T04:20:18.570638+0000","last_fullsized":"2023-10-09T04:20:18.570638+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":121,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.559200+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:20:18.559200+0000","last_peered":"2023-10-09T04:20:18.559200+0000","last_clean":"2023-10-09T04:20:18.559200+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:20:18.559200+0000","last_undegraded":"2023-10-09T04:20:18.559200+0000","last_fullsized":"2023-10-09T04:20:18.559200+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":779,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.553532+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:20:18.553532+0000","last_peered":"2023-10-09T04:20:18.553532+0000","last_clean":"2023-10-09T04:20:18.553532+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:20:18.553532+0000","last_undegraded":"2023-10-09T04:20:18.553532+0000","last_fullsized":"2023-10-09T04:20:18.553532+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":201,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.587667+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:20:18.587667+0000","last_peered":"2023-10-09T04:20:18.587667+0000","last_clean":"2023-10-09T04:20:18.587667+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:20:18.587667+0000","last_undegraded":"2023-10-09T04:20:18.587667+0000","last_fullsized":"2023-10-09T04:20:18.587667+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:24.392665+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:20:24.392665+0000","last_peered":"2023-10-09T04:20:24.392665+0000","last_clean":"2023-10-09T04:20:24.392665+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:20:24.392665+0000","last_undegraded":"2023-10-09T04:20:24.392665+0000","last_fullsized":"2023-10-09T04:20:24.392665+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":498,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.590295+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:20:18.590295+0000","last_peered":"2023-10-09T04:20:18.590295+0000","last_clean":"2023-10-09T04:20:18.590295+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:20:18.590295+0000","last_undegraded":"2023-10-09T04:20:18.590295+0000","last_fullsized":"2023-10-09T04:20:18.590295+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":350,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.586757+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:20:18.586757+0000","last_peered":"2023-10-09T04:20:18.586757+0000","last_clean":"2023-10-09T04:20:18.586757+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:20:18.586757+0000","last_undegraded":"2023-10-09T04:20:18.586757+0000","last_fullsized":"2023-10-09T04:20:18.586757+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":111,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:20:18.558776+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:20:18.558776+0000","last_peered":"2023-10-09T04:20:18.558776+0000","last_clean":"2023-10-09T04:20:18.558776+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:20:18.558776+0000","last_undegraded":"2023-10-09T04:20:18.558776+0000","last_fullsized":"2023-10-09T04:20:18.558776+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4309123096,"num_objects":65753,"num_object_clones":0,"num_object_copies":197259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11915,"num_objects_unfound":0,"num_objects_dirty":65753,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131695,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16678,"num_bytes_recovered":1092943896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13387841536,"data_stored":13387825248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91774,"ondisk_log_size":91774,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201616,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3543680,"kb_used_data":2949688,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":90201472,"statfs":{"total":95995035648,"available":92366307328,"internally_reserved":0,"allocated":3020480512,"data_stored":3019473356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":2.4089999999999998,"15min":2.9249999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":16.440999999999999,"5min":16.440999999999999,"15min":126.752},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.1869999999999998,"5min":2.2189999999999999,"15min":2.8820000000000001},"min":{"1min":0.27300000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":16.462,"5min":16.462,"15min":128.88399999999999},"last":0.375}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2430000000000003,"5min":4.3739999999999997,"15min":4.21},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":29.792000000000002,"5min":103.324,"15min":245.28700000000001},"last":7.1719999999999997},{"interface":"front","average":{"1min":4.9909999999999997,"5min":4.6210000000000004,"15min":4.1660000000000004},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":29.268000000000001,"5min":104.248,"15min":244.626},"last":6.9569999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.7849999999999999,"15min":2.0979999999999999},"min":{"1min":0.30499999999999999,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.194,"5min":14.076000000000001,"15min":76.774000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.556,"15min":2.794},"min":{"1min":0.35799999999999998,"5min":0.246,"15min":0.246},"max":{"1min":6.1749999999999998,"5min":15.597,"15min":99.372},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":2.6419999999999999,"15min":2.633},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":3.7149999999999999,"5min":85.575000000000003,"15min":99.134},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.1080000000000001,"15min":2.8769999999999998},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":6.1459999999999999,"5min":85.528000000000006,"15min":99.117999999999995},"last":0.41399999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.86899999999999999,"15min":0.82799999999999996},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.8529999999999998,"5min":8.9359999999999999,"15min":16.195},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.6160000000000001,"15min":1.1120000000000001},"min":{"1min":0.27700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":2.5680000000000001,"15min":1.321},"min":{"1min":0.29299999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":6.9390000000000001,"5min":120.066,"15min":120.066},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.85299999999999998,"5min":2.2530000000000001,"15min":1.2789999999999999},"min":{"1min":0.34399999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8899999999999997,"5min":120.09699999999999,"15min":120.09699999999999},"last":0.30599999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.4590000000000001,"15min":1.0249999999999999},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":8.4550000000000001,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.96999999999999997,"15min":0.88300000000000001},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.605,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.43099999999999999}]}]},{"osd":6,"up_from":0,"seq":201863463254,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089562,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3213276,"kb_used_data":2610900,"kb_used_omap":9,"kb_used_meta":602166,"kb_avail":90531876,"statfs":{"total":95995035648,"available":92704641024,"internally_reserved":0,"allocated":2673561600,"data_stored":2672061813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":616618985},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":3.8759999999999999,"15min":4.0629999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":19.617000000000001,"5min":98.257999999999996,"15min":98.257999999999996},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.1949999999999998,"5min":6.3319999999999999,"15min":5.2000000000000002},"min":{"1min":0.27100000000000002,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":9.9960000000000004,"5min":154.654,"15min":154.654},"last":0.61199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":2.1970000000000001,"15min":2.6019999999999999},"min":{"1min":0.26600000000000001,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":19.564,"5min":19.564,"15min":121.54600000000001},"last":0.255},{"interface":"front","average":{"1min":2,"5min":2.476,"15min":2.9239999999999999},"min":{"1min":0.22500000000000001,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":12.382999999999999,"5min":16.114999999999998,"15min":99.899000000000001},"last":2.2400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.8660000000000001,"15min":3.7770000000000001},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":19.640000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.4609999999999999,"5min":3.8170000000000002,"15min":4.6970000000000001},"min":{"1min":0.307,"5min":0.251,"15min":0.251},"max":{"1min":12.353999999999999,"5min":68.563999999999993,"15min":138.49799999999999},"last":0.33800000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.2290000000000001,"15min":2.3370000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":7.5250000000000004,"5min":16.27,"15min":99.989999999999995},"last":5.8079999999999998},{"interface":"front","average":{"1min":1.458,"5min":1.9530000000000001,"15min":2.1829999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":5.282,"5min":13.428000000000001,"15min":100.249},"last":5.4690000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":3.899,"5min":3.2280000000000002,"15min":2.6539999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":25.696999999999999,"5min":38.848999999999997,"15min":99.933999999999997},"last":2.6629999999999998},{"interface":"front","average":{"1min":2.8700000000000001,"5min":3.145,"15min":2.6909999999999998},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":19.977,"5min":39.593000000000004,"15min":100.34099999999999},"last":5.6879999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.2120000000000002,"15min":2.5},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":5.3579999999999997,"5min":13.377000000000001,"15min":100.309},"last":5.5010000000000003},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.9570000000000001,"15min":2.1749999999999998},"min":{"1min":0.41399999999999998,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":100.143},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.7050000000000001,"15min":2.6419999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":5.7430000000000003,"5min":17.460000000000001,"15min":100.214},"last":5.7199999999999998},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.3359999999999999,"15min":2.319},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":19.593,"5min":19.593,"15min":100.113},"last":2.6230000000000002}]}]},{"osd":0,"up_from":84,"seq":360777253088,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1832804,"kb_used_data":1323164,"kb_used_omap":28,"kb_used_meta":509603,"kb_avail":91912348,"statfs":{"total":95995035648,"available":94118244352,"internally_reserved":0,"allocated":1354919936,"data_stored":1353352490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":521834320},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":8.9990000000000006,"15min":5.2709999999999999},"min":{"1min":0.20200000000000001,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":6.1609999999999996,"5min":209.215,"15min":209.215},"last":3.2429999999999999},{"interface":"front","average":{"1min":3.0499999999999998,"5min":6.8630000000000004,"15min":5.032},"min":{"1min":0.29499999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.547000000000001,"5min":120.934,"15min":120.934},"last":5.0609999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.3359999999999999,"15min":3.141},"min":{"1min":0.32900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.025,"5min":17.466999999999999,"15min":118.54000000000001},"last":0.216},{"interface":"front","average":{"1min":2.0609999999999999,"5min":3.2429999999999999,"15min":3.536},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.20300000000000001},"max":{"1min":6.7460000000000004,"5min":20.262,"15min":118.727},"last":0.27900000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":3.1400000000000001,"15min":5.2759999999999998},"min":{"1min":0.25700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":9.1669999999999998,"5min":99.709000000000003,"15min":156.488},"last":5.6020000000000003},{"interface":"front","average":{"1min":2.3919999999999999,"5min":3.1720000000000002,"15min":5.3620000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":10.577,"5min":99.560000000000002,"15min":156.50700000000001},"last":0.27300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":4.899,"5min":2.9969999999999999,"15min":2.7250000000000001},"min":{"1min":0.35399999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":49.840000000000003,"5min":63.375999999999998,"15min":118.633},"last":0.29899999999999999},{"interface":"front","average":{"1min":5.0609999999999999,"5min":3.3290000000000002,"15min":3.0670000000000002},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":49.607999999999997,"5min":49.607999999999997,"15min":118.913},"last":0.34599999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":5.4800000000000004,"15min":4.2960000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.0570000000000004,"5min":156.96100000000001,"15min":182.68899999999999},"last":1.2769999999999999},{"interface":"front","average":{"1min":2.0870000000000002,"5min":4.5540000000000003,"15min":3.7749999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.024,"5min":156.97300000000001,"15min":182.66300000000001},"last":5.5469999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.528,"15min":2.996},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":7.0709999999999997,"5min":63.468000000000004,"15min":137.36099999999999},"last":5.5099999999999998},{"interface":"front","average":{"1min":2.206,"5min":2.5859999999999999,"15min":3.0329999999999999},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":10.113,"5min":63.447000000000003,"15min":136.88999999999999},"last":0.36599999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.6419999999999999,"15min":3.677},"min":{"1min":0.40000000000000002,"5min":0.375,"15min":0.32100000000000001},"max":{"1min":9.1699999999999999,"5min":63.509,"15min":185.024},"last":5.5780000000000003},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.7770000000000001,"15min":3.3479999999999999},"min":{"1min":0.44700000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":10.077,"5min":63.481999999999999,"15min":184.98599999999999},"last":5.0279999999999996}]}]},{"osd":2,"up_from":94,"seq":403726926036,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1865072,"kb_used_data":1297000,"kb_used_omap":81,"kb_used_meta":567918,"kb_avail":91880080,"statfs":{"total":95995035648,"available":94085201920,"internally_reserved":0,"allocated":1328128000,"data_stored":1326612888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":581548250},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":5.3929999999999998,"15min":4.024},"min":{"1min":0.22500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":17.122,"5min":81.385999999999996,"15min":81.385999999999996},"last":21.792000000000002},{"interface":"front","average":{"1min":1.228,"5min":4.617,"15min":3.9540000000000002},"min":{"1min":0.22500000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":4.1070000000000002,"5min":79.453999999999994,"15min":79.453999999999994},"last":21.603000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9340000000000002,"5min":7.2270000000000003,"15min":4.7549999999999999},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.191},"max":{"1min":72.075999999999993,"5min":158.13399999999999,"15min":158.13399999999999},"last":21.556999999999999},{"interface":"front","average":{"1min":6.4489999999999998,"5min":7.883,"15min":5.0590000000000002},"min":{"1min":0.27600000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":69.754999999999995,"5min":161.12,"15min":161.12},"last":4.7190000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.4700000000000002,"15min":2.8170000000000002},"min":{"1min":0.28000000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.409000000000001,"5min":58.997,"15min":58.997},"last":1.653},{"interface":"front","average":{"1min":2.1499999999999999,"5min":3.5030000000000001,"15min":3.0270000000000001},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.488,"5min":66.135000000000005,"15min":66.135000000000005},"last":1.1899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":2.7549999999999999,"15min":2.2650000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":12.775,"5min":58.798000000000002,"15min":58.798000000000002},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.75,"5min":2.5619999999999998,"15min":2.093},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":6.5010000000000003,"5min":56.616,"15min":56.616},"last":1.6859999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":3.9660000000000002,"15min":2.7269999999999999},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":82.030000000000001,"5min":82.030000000000001,"15min":82.030000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":7.4290000000000003,"5min":4.2430000000000003,"15min":2.9119999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":82.016000000000005,"5min":82.016000000000005,"15min":82.016000000000005},"last":0.84399999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.6949999999999998,"15min":2.5579999999999998},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.0129999999999999,"5min":56.561,"15min":114.14},"last":1.7350000000000001},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.629,"15min":2.6469999999999998},"min":{"1min":0.40999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2110000000000003,"5min":56.460999999999999,"15min":114.123},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.0700000000000003,"15min":3.1989999999999998},"min":{"1min":0.42599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":101.675,"5min":101.675,"15min":117.32599999999999},"last":1.7090000000000001},{"interface":"front","average":{"1min":8.2639999999999993,"5min":4.1589999999999998,"15min":3.2639999999999998},"min":{"1min":0.46100000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":101.72499999999999,"5min":101.72499999999999,"15min":117.28700000000001},"last":0.56200000000000006}]}]},{"osd":3,"up_from":99,"seq":425201762510,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2217420,"kb_used_data":1485892,"kb_used_omap":7,"kb_used_meta":731448,"kb_avail":91527732,"statfs":{"total":95995035648,"available":93724397568,"internally_reserved":0,"allocated":1521553408,"data_stored":1520067527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":749003368},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":3.3730000000000002,"15min":5.4059999999999997},"min":{"1min":0.214,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":6.4169999999999998,"5min":78.771000000000001,"15min":125.765},"last":9.0009999999999994},{"interface":"front","average":{"1min":2.899,"5min":4.2080000000000002,"15min":5.8280000000000003},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.119999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":9.0169999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":5.8979999999999997,"15min":5.2619999999999996},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":8.3230000000000004,"5min":103.143,"15min":186.21299999999999},"last":4.625},{"interface":"front","average":{"1min":1.6060000000000001,"5min":5.7789999999999999,"15min":5.0309999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.193},"max":{"1min":5.9219999999999997,"5min":105.19,"15min":191.78},"last":12.617000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.1660000000000004,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.8760000000000003,"5min":127.599,"15min":127.599},"last":5.4500000000000002},{"interface":"front","average":{"1min":2.359,"5min":6.0670000000000002,"15min":5.0119999999999996},"min":{"1min":0.29799999999999999,"5min":0.221,"15min":0.214},"max":{"1min":10.143000000000001,"5min":129.98599999999999,"15min":129.98599999999999},"last":5.375}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":2.274,"15min":2.7240000000000002},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.013999999999999,"5min":27.539999999999999,"15min":106.85899999999999},"last":4.2130000000000001},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.2919999999999998,"15min":2.1760000000000002},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.738,"5min":26.663,"15min":37.442},"last":4.6929999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.6280000000000001,"15min":2.2010000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":3.7749999999999999,"5min":17.901,"15min":17.901},"last":4.6710000000000003},{"interface":"front","average":{"1min":1.083,"5min":1.766,"15min":1.887},"min":{"1min":0.375,"5min":0.248,"15min":0.248},"max":{"1min":4.6340000000000003,"5min":18.068000000000001,"15min":18.068000000000001},"last":5.4909999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.7509999999999999,"15min":2.2610000000000001},"min":{"1min":0.378,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4589999999999996,"5min":14.567,"15min":106.036},"last":5.5300000000000002},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.48,"15min":2.6509999999999998},"min":{"1min":0.42699999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.071,"5min":14.529,"15min":106.741},"last":4.1479999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":2.6720000000000002,"15min":2.786},"min":{"1min":0.39000000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":28.184000000000001,"5min":28.184000000000001,"15min":106.82599999999999},"last":4.1859999999999999},{"interface":"front","average":{"1min":3.1480000000000001,"5min":2.827,"15min":2.7490000000000001},"min":{"1min":0.44800000000000001,"5min":0.40000000000000002,"15min":0.35999999999999999},"max":{"1min":26.648,"5min":26.648,"15min":106.084},"last":4.7300000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986526,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1596272,"kb_used_data":1290792,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92148880,"statfs":{"total":95995035648,"available":94360453120,"internally_reserved":0,"allocated":1321771008,"data_stored":1320779267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":6.7859999999999996,"5min":9.9049999999999994,"15min":5.9029999999999996},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":72.759,"5min":154.62100000000001,"15min":154.62100000000001},"last":8.5329999999999995},{"interface":"front","average":{"1min":6.7279999999999998,"5min":10.082000000000001,"15min":5.9210000000000003},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":72.704999999999998,"5min":154.37299999999999,"15min":154.37299999999999},"last":0.28499999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":7.8129999999999997,"5min":9.4969999999999999,"15min":5.1180000000000003},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":69.706000000000003,"5min":135.25399999999999,"15min":135.25399999999999},"last":3.4329999999999998},{"interface":"front","average":{"1min":7.3220000000000001,"5min":9.1270000000000007,"15min":5.0060000000000002},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":69.561000000000007,"5min":134.44300000000001,"15min":134.44300000000001},"last":8.5749999999999993}]},{"osd":2,"last update":"Mon Oct 9 04:19:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.679,"15min":2.0720000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.245},"max":{"1min":8.7400000000000002,"5min":8.7400000000000002,"15min":123.598},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.708,"5min":1.546,"15min":1.986},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":6.21,"5min":8.4800000000000004,"15min":123.619},"last":8.6039999999999992}]},{"osd":3,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.052,"15min":2.29},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.0069999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":3.391},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.165,"15min":2.4849999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.7579999999999991,"5min":8.9169999999999998,"15min":127.90000000000001},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":3.6099999999999999,"15min":1.6279999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":13.599,"5min":111.809,"15min":111.809},"last":0.317},{"interface":"front","average":{"1min":2.532,"5min":3.7200000000000002,"15min":1.7929999999999999},"min":{"1min":0.316,"5min":0.252,"15min":0.25},"max":{"1min":9.1530000000000005,"5min":112.002,"15min":112.002},"last":3.4620000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.395,"15min":1.3680000000000001},"min":{"1min":0.35699999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":6.9470000000000001,"5min":76.307000000000002,"15min":76.307000000000002},"last":3.4900000000000002},{"interface":"front","average":{"1min":1.877,"5min":2.2130000000000001,"15min":1.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9320000000000004,"5min":78.421999999999997,"15min":78.421999999999997},"last":8.6340000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.3149999999999999,"15min":1.079},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7789999999999999,"5min":8.7789999999999999,"15min":34.039000000000001},"last":6.8650000000000002},{"interface":"front","average":{"1min":2.46,"5min":1.508,"15min":1.194},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.5430000000000001,"5min":10.384,"15min":33.433999999999997},"last":0.433}]}]},{"osd":5,"up_from":39,"seq":167503724889,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2821804,"kb_used_data":2223844,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":90923348,"statfs":{"total":95995035648,"available":93105508352,"internally_reserved":0,"allocated":2277216256,"data_stored":2276225021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":4.984,"15min":5.0759999999999996},"min":{"1min":0.223,"5min":0.193,"15min":0.193},"max":{"1min":13.675000000000001,"5min":118.98399999999999,"15min":151.23599999999999},"last":0.31},{"interface":"front","average":{"1min":2.4079999999999999,"5min":5.3380000000000001,"15min":5.4470000000000001},"min":{"1min":0.23499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":13.885,"5min":118.846,"15min":151.566},"last":0.26700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":5.242,"15min":4.0270000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":29.632999999999999,"5min":111.194,"15min":111.194},"last":0.371},{"interface":"front","average":{"1min":4.7249999999999996,"5min":5.8289999999999997,"15min":4.4180000000000001},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":29.672000000000001,"5min":112.226,"15min":112.226},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.444,"15min":3.1299999999999999},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":8.8599999999999994,"5min":11.324,"15min":106.508},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.718,"15min":2.8879999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":3.8199999999999998,"5min":8.5280000000000005,"15min":106.462},"last":0.28199999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.0449999999999999,"15min":2.5939999999999999},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.555999999999999,"5min":11.555999999999999,"15min":91.834999999999994},"last":7.548},{"interface":"front","average":{"1min":2.3260000000000001,"5min":2.2389999999999999,"15min":2.6890000000000001},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3219999999999992,"5min":12.122999999999999,"15min":91.802000000000007},"last":7.5350000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.242,"15min":0.93300000000000005},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.246},"max":{"1min":9.2579999999999991,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.1200000000000001,"15min":0.92900000000000005},"min":{"1min":0.30199999999999999,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":3.77,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.33000000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.27,"15min":1.3779999999999999},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":9.4090000000000007,"5min":12.068,"15min":123.239},"last":0.435},{"interface":"front","average":{"1min":1.615,"5min":1.45,"15min":1.4670000000000001},"min":{"1min":0.313,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":9.7720000000000002,"15min":123.014},"last":0.35199999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:19:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.202,"15min":0.995},"min":{"1min":0.33100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.3289999999999997,"5min":12.096,"15min":12.096},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.24,"5min":1.446,"15min":1.095},"min":{"1min":0.36699999999999999,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":9.3019999999999996,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.72899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1346109440,"data_stored":1346109440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2651721728,"data_stored":2651717656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1307901952,"data_stored":1307901952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1510670336,"data_stored":1510670336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1314983936,"data_stored":1314979864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2262044672,"data_stored":2262040600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2993950720,"data_stored":2993946648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:20:38.985 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:20:39.006 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:39.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:38 smithi181 ceph-mon[196979]: from='client.55705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:39.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:38 smithi181 ceph-mon[196979]: pgmap v964: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 11262/187953 objects misplaced (5.992%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:20:39.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:39.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:39.109 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:39.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:39.279 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:39.508 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:39.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:39.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:39.617 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:39.690 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:40.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:40.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:40.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:40.397 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:40.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:40 smithi163 ceph-mon[213841]: pgmap v965: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 820 op/s; 10754/185670 objects misplaced (5.792%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:20:40.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3768559883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:40.931 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:40 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:40.931 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:20:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:20:40] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-10-09T04:20:40.942 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:20:41.032 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:41.117 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:41.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:41.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:41.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:41.261 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:40 smithi181 ceph-mon[196979]: pgmap v965: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 820 op/s; 10754/185670 objects misplaced (5.792%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:20:41.261 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3768559883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:41.261 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:40 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:41.266 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:41.308 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:40 smithi163 ceph-mon[221067]: pgmap v965: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 820 op/s; 10754/185670 objects misplaced (5.792%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:20:41.308 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3768559883' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:41.308 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:40 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:20:41.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:41.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:41.468 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:41.546 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:41 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:41 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:20:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:41 smithi181 ceph-mon[196979]: osdmap e358: 8 total, 8 up, 7 in 2023-10-09T04:20:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:41 smithi181 ceph-mon[196979]: failed to encode map e358 with expected crc 2023-10-09T04:20:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:41 smithi181 ceph-mon[196979]: failed to encode map e358 with expected crc 2023-10-09T04:20:42.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:42.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:20:42.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[213841]: osdmap e358: 8 total, 8 up, 7 in 2023-10-09T04:20:42.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[213841]: failed to encode map e358 with expected crc 2023-10-09T04:20:42.272 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[213841]: failed to encode map e358 with expected crc 2023-10-09T04:20:42.272 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:42.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:20:42.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[221067]: osdmap e358: 8 total, 8 up, 7 in 2023-10-09T04:20:42.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[221067]: failed to encode map e358 with expected crc 2023-10-09T04:20:42.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:41 smithi163 ceph-mon[221067]: failed to encode map e358 with expected crc 2023-10-09T04:20:42.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:42.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:42.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:42.514 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:42.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:42.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:42.918 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:42.953 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:20:43.001 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:43.012 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:20:43.048 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:20:43.099 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:20:43.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:42 smithi163 ceph-mon[213841]: pgmap v967: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 10158/176118 objects misplaced (5.768%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:20:43.162 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4103676921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:43.163 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:42 smithi163 ceph-mon[221067]: pgmap v967: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 10158/176118 objects misplaced (5.768%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:20:43.163 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4103676921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:43.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:20:43.196 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:20:43.196 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:20:43.199 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:20:43.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:43.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:43.227 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:20:43.257 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.259 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:20:43.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:42 smithi181 ceph-mon[196979]: pgmap v967: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 10158/176118 objects misplaced (5.768%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:20:43.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4103676921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:43.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:43.352 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:43.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:43.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:43.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:43.569 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:44.014 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:44.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:44.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:44.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:44.150 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:44.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:44.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:44.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:44.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:44.347 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:20:44.413 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:44.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:20:44.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:44.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:44.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:44.882 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:44.950 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:44 smithi163 ceph-mon[213841]: pgmap v968: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 9730/173862 objects misplaced (5.596%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:20:44.977 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:44 smithi181 ceph-mon[196979]: pgmap v968: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 9730/173862 objects misplaced (5.596%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:20:45.280 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:45.289 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:44 smithi163 ceph-mon[221067]: pgmap v968: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 9730/173862 objects misplaced (5.596%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:20:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:45.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:45.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:45.432 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:45.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:45.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:45.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:45.602 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:45.679 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:45.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:45.822 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:45.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:45.865 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:45.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:45.967 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:46.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:46.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:46.151 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:46.193 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:46.258 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:46.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:46.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:46.316 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:46.316 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1210,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":358,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":7,"osd_in_since":1696825107,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":116},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":119,"num_pools":5,"num_objects":57701,"data_bytes":3780526265,"bytes_used":15926972416,"bytes_avail":656038277120,"bytes_total":671965249536,"misplaced_objects":9730,"misplaced_total":173103,"misplaced_ratio":0.056209308908568879,"recovering_objects_per_sec":85,"recovering_bytes_per_sec":5575627,"recovering_keys_per_sec":0,"num_objects_recovered":852,"num_bytes_recovered":55836672,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":806},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T04:19:35.820638+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"cbdf697a-aea6-4b77-8dba-e1904d62d8bd":{"message":"Global Recovery Event (70s)\n [===========================.] (remaining: 1s)","progress":0.97478991746902466,"add_to_ceph_s":true}}} 2023-10-09T04:20:46.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:46.427 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:47.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:46 smithi181 ceph-mon[196979]: pgmap v969: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 9730/173103 objects misplaced (5.621%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:20:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2663697683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:20:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:46 smithi163 ceph-mon[221067]: pgmap v969: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 9730/173103 objects misplaced (5.621%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:20:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2663697683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:20:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:46 smithi163 ceph-mon[213841]: pgmap v969: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 9730/173103 objects misplaced (5.621%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:20:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2663697683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:20:47.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:20:48.000 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:48.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:48.069 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:20:48.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:20:48.108 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:48.219 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:48.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:47 smithi181 ceph-mon[196979]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:47 smithi181 ceph-mon[196979]: osdmap e359: 8 total, 8 up, 7 in 2023-10-09T04:20:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:47 smithi181 ceph-mon[196979]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:47 smithi181 ceph-mon[196979]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.282 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[221067]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.282 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[221067]: osdmap e359: 8 total, 8 up, 7 in 2023-10-09T04:20:48.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[221067]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[221067]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[213841]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[213841]: osdmap e359: 8 total, 8 up, 7 in 2023-10-09T04:20:48.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[213841]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:47 smithi163 ceph-mon[213841]: failed to encode map e359 with expected crc 2023-10-09T04:20:48.749 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:48.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:48.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:48.943 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[213841]: pgmap v971: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 893 op/s; 9082/164193 objects misplaced (5.531%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[213841]: osdmap e360: 8 total, 8 up, 7 in 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[213841]: failed to encode map e360 with expected crc 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[213841]: failed to encode map e360 with expected crc 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[213841]: failed to encode map e360 with expected crc 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[221067]: pgmap v971: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 893 op/s; 9082/164193 objects misplaced (5.531%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:20:48.989 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[221067]: osdmap e360: 8 total, 8 up, 7 in 2023-10-09T04:20:48.990 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[221067]: failed to encode map e360 with expected crc 2023-10-09T04:20:48.990 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[221067]: failed to encode map e360 with expected crc 2023-10-09T04:20:48.990 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:48 smithi163 ceph-mon[221067]: failed to encode map e360 with expected crc 2023-10-09T04:20:49.188 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:49.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:49.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:48 smithi181 ceph-mon[196979]: pgmap v971: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 893 op/s; 9082/164193 objects misplaced (5.531%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:20:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:48 smithi181 ceph-mon[196979]: osdmap e360: 8 total, 8 up, 7 in 2023-10-09T04:20:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:48 smithi181 ceph-mon[196979]: failed to encode map e360 with expected crc 2023-10-09T04:20:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:48 smithi181 ceph-mon[196979]: failed to encode map e360 with expected crc 2023-10-09T04:20:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:48 smithi181 ceph-mon[196979]: failed to encode map e360 with expected crc 2023-10-09T04:20:49.291 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:49.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:49.357 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:49.414 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:20:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:49.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:20:49.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:49.615 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:49.873 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:49.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:49.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:49.892 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:20:49.892 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:20:49.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:50.046 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:50.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:50.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:50.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/866687083' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:50.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:50.355 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:20:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[213841]: pgmap v973: 119 pgs: 2 active+remapped+backfill_wait, 117 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8863/162267 objects misplaced (5.462%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:20:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/866687083' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:50.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:50 smithi163 ceph-mon[221067]: pgmap v973: 119 pgs: 2 active+remapped+backfill_wait, 117 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8863/162267 objects misplaced (5.462%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:20:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:20:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/866687083' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:20:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:50 smithi181 ceph-mon[196979]: pgmap v973: 119 pgs: 2 active+remapped+backfill_wait, 117 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8863/162267 objects misplaced (5.462%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:20:51.032 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:20:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:20:50] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-10-09T04:20:51.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:51.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:51.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:51.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:51.446 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:51.523 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:51.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:51.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:51.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:51.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[213841]: osdmap e361: 8 total, 8 up, 7 in 2023-10-09T04:20:51.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[213841]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[213841]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[213841]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:51.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:51.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[221067]: osdmap e361: 8 total, 8 up, 7 in 2023-10-09T04:20:51.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[221067]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[221067]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.580 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:51 smithi163 ceph-mon[221067]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:20:51.635 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:51.714 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:51.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:51 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:20:51.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:20:51.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:51 smithi181 ceph-mon[196979]: osdmap e361: 8 total, 8 up, 7 in 2023-10-09T04:20:51.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:51 smithi181 ceph-mon[196979]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:51 smithi181 ceph-mon[196979]: failed to encode map e361 with expected crc 2023-10-09T04:20:51.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:51 smithi181 ceph-mon[196979]: failed to encode map e361 with expected crc 2023-10-09T04:20:52.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:52.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:52.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:52.233 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:52.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:52.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:52.345 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:52 smithi163 ceph-mon[213841]: pgmap v975: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 7448/153603 objects misplaced (4.849%); 11 MiB/s, 175 objects/s recovering 2023-10-09T04:20:52.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:52.490 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:52 smithi181 ceph-mon[196979]: pgmap v975: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 7448/153603 objects misplaced (4.849%); 11 MiB/s, 175 objects/s recovering 2023-10-09T04:20:52.495 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:52.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:52.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:52.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:52.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:52.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:52 smithi163 ceph-mon[221067]: pgmap v975: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 7448/153603 objects misplaced (4.849%); 11 MiB/s, 175 objects/s recovering 2023-10-09T04:20:52.710 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:52.781 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:52.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:52.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:52.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:52.950 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:53.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:53.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:53.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:53.243 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:53.345 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:53.381 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:53.405 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:53.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:53.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:20:53.544 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:53.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:20:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:53.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:20:53.731 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:53.850 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:53.866 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:20:53.878 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:20:53.888 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4015765736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:53.913 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:20:53.926 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:20:53.970 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:20:54.011 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:20:54.053 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:20:54.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4015765736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:54.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4015765736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:20:54.090 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:20:54.125 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:20:54.174 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:20:54.175 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.175 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.176 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.176 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.176 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.176 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.176 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.176 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:20:54.182 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:54.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:54.296 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:54.335 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:54.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:20:55.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:54 smithi163 ceph-mon[213841]: from='client.55726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:55.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:54 smithi163 ceph-mon[213841]: pgmap v976: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 7448/151104 objects misplaced (4.929%); 9.6 MiB/s, 152 objects/s recovering 2023-10-09T04:20:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:54 smithi163 ceph-mon[221067]: from='client.55726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:55.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:54 smithi163 ceph-mon[221067]: pgmap v976: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 7448/151104 objects misplaced (4.929%); 9.6 MiB/s, 152 objects/s recovering 2023-10-09T04:20:55.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:54 smithi181 ceph-mon[196979]: from='client.55726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:20:55.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:54 smithi181 ceph-mon[196979]: pgmap v976: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 7448/151104 objects misplaced (4.929%); 9.6 MiB/s, 152 objects/s recovering 2023-10-09T04:20:55.405 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:55.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:55.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:55.563 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:55.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:55.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:55.746 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:20:55.788 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:55.859 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:20:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:55.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:20:55.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:55.990 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:56.214 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":975,"stamp":"2023-10-09T04:20:51.858740+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3354542265,"num_objects":51201,"num_object_clones":0,"num_object_copies":153603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7448,"num_objects_unfound":0,"num_objects_dirty":51201,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":146364,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18652,"num_bytes_recovered":1218488033,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96596,"ondisk_log_size":96596,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14478528,"kb_used_data":10569928,"kb_used_omap":126,"kb_used_meta":3908417,"kb_avail":641737536,"statfs":{"total":671965249536,"available":657139236864,"internally_reserved":0,"allocated":10823606272,"data_stored":10814438534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4002219643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-425984000,"num_objects":-6500,"num_object_clones":0,"num_object_copies":-19500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2282,"num_objects_unfound":0,"num_objects_dirty":-6500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6500,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1055,"num_bytes_recovered":69140480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001345"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":485,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.988887+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:20:47.988887+0000","last_peered":"2023-10-09T04:20:47.988887+0000","last_clean":"2023-10-09T04:20:47.988887+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:20:47.988887+0000","last_undegraded":"2023-10-09T04:20:47.988887+0000","last_fullsized":"2023-10-09T04:20:47.988887+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":105,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026127+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:20:48.026127+0000","last_peered":"2023-10-09T04:20:48.026127+0000","last_clean":"2023-10-09T04:20:48.026127+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:20:48.026127+0000","last_undegraded":"2023-10-09T04:20:48.026127+0000","last_fullsized":"2023-10-09T04:20:48.026127+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":335,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.972612+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:20:46.972612+0000","last_peered":"2023-10-09T04:20:46.972612+0000","last_clean":"2023-10-09T04:20:46.972612+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:20:46.972612+0000","last_undegraded":"2023-10-09T04:20:46.972612+0000","last_fullsized":"2023-10-09T04:20:46.972612+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":478,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989224+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:20:47.989224+0000","last_peered":"2023-10-09T04:20:47.989224+0000","last_clean":"2023-10-09T04:20:47.989224+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:20:47.989224+0000","last_undegraded":"2023-10-09T04:20:47.989224+0000","last_fullsized":"2023-10-09T04:20:47.989224+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":350,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.972470+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:20:46.972470+0000","last_peered":"2023-10-09T04:20:46.972470+0000","last_clean":"2023-10-09T04:20:46.972470+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:20:46.972470+0000","last_undegraded":"2023-10-09T04:20:46.972470+0000","last_fullsized":"2023-10-09T04:20:46.972470+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":339,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.063789+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:20:48.063789+0000","last_peered":"2023-10-09T04:20:48.063789+0000","last_clean":"2023-10-09T04:20:48.063789+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:20:48.063789+0000","last_undegraded":"2023-10-09T04:20:48.063789+0000","last_fullsized":"2023-10-09T04:20:48.063789+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":485,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026021+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:20:48.026021+0000","last_peered":"2023-10-09T04:20:48.026021+0000","last_clean":"2023-10-09T04:20:48.026021+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:20:48.026021+0000","last_undegraded":"2023-10-09T04:20:48.026021+0000","last_fullsized":"2023-10-09T04:20:48.026021+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":336,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.066260+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:20:48.066260+0000","last_peered":"2023-10-09T04:20:48.066260+0000","last_clean":"2023-10-09T04:20:48.066260+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:20:48.066260+0000","last_undegraded":"2023-10-09T04:20:48.066260+0000","last_fullsized":"2023-10-09T04:20:48.066260+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":304,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989725+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:20:47.989725+0000","last_peered":"2023-10-09T04:20:47.989725+0000","last_clean":"2023-10-09T04:20:47.989725+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:20:47.989725+0000","last_undegraded":"2023-10-09T04:20:47.989725+0000","last_fullsized":"2023-10-09T04:20:47.989725+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":760,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023937+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:20:48.023937+0000","last_peered":"2023-10-09T04:20:48.023937+0000","last_clean":"2023-10-09T04:20:48.023937+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:20:48.023937+0000","last_undegraded":"2023-10-09T04:20:48.023937+0000","last_fullsized":"2023-10-09T04:20:48.023937+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":337,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.067722+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:20:48.067722+0000","last_peered":"2023-10-09T04:20:48.067722+0000","last_clean":"2023-10-09T04:20:48.067722+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:20:48.067722+0000","last_undegraded":"2023-10-09T04:20:48.067722+0000","last_fullsized":"2023-10-09T04:20:48.067722+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":70,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.976615+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:20:46.976615+0000","last_peered":"2023-10-09T04:20:46.976615+0000","last_clean":"2023-10-09T04:20:46.976615+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:20:46.976615+0000","last_undegraded":"2023-10-09T04:20:46.976615+0000","last_fullsized":"2023-10-09T04:20:46.976615+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":486,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.986619+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:20:46.986619+0000","last_peered":"2023-10-09T04:20:46.986619+0000","last_clean":"2023-10-09T04:20:46.986619+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:20:46.986619+0000","last_undegraded":"2023-10-09T04:20:46.986619+0000","last_fullsized":"2023-10-09T04:20:46.986619+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":320,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989534+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:20:47.989534+0000","last_peered":"2023-10-09T04:20:47.989534+0000","last_clean":"2023-10-09T04:20:47.989534+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:20:47.989534+0000","last_undegraded":"2023-10-09T04:20:47.989534+0000","last_fullsized":"2023-10-09T04:20:47.989534+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":55,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.988881+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:20:47.988881+0000","last_peered":"2023-10-09T04:20:47.988881+0000","last_clean":"2023-10-09T04:20:47.988881+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:20:47.988881+0000","last_undegraded":"2023-10-09T04:20:47.988881+0000","last_fullsized":"2023-10-09T04:20:47.988881+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":495,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989607+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:20:47.989607+0000","last_peered":"2023-10-09T04:20:47.989607+0000","last_clean":"2023-10-09T04:20:47.989607+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:20:47.989607+0000","last_undegraded":"2023-10-09T04:20:47.989607+0000","last_fullsized":"2023-10-09T04:20:47.989607+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":403,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.024205+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:20:48.024205+0000","last_peered":"2023-10-09T04:20:48.024205+0000","last_clean":"2023-10-09T04:20:48.024205+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:20:48.024205+0000","last_undegraded":"2023-10-09T04:20:48.024205+0000","last_fullsized":"2023-10-09T04:20:48.024205+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":38,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.972405+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:20:46.972405+0000","last_peered":"2023-10-09T04:20:46.972405+0000","last_clean":"2023-10-09T04:20:46.972405+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:20:46.972405+0000","last_undegraded":"2023-10-09T04:20:46.972405+0000","last_fullsized":"2023-10-09T04:20:46.972405+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":502,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.975965+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:20:46.975965+0000","last_peered":"2023-10-09T04:20:46.975965+0000","last_clean":"2023-10-09T04:20:46.975965+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:20:46.975965+0000","last_undegraded":"2023-10-09T04:20:46.975965+0000","last_fullsized":"2023-10-09T04:20:46.975965+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":135,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026308+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:20:48.026308+0000","last_peered":"2023-10-09T04:20:48.026308+0000","last_clean":"2023-10-09T04:20:48.026308+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:20:48.026308+0000","last_undegraded":"2023-10-09T04:20:48.026308+0000","last_fullsized":"2023-10-09T04:20:48.026308+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026370+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:20:48.026370+0000","last_peered":"2023-10-09T04:20:48.026370+0000","last_clean":"2023-10-09T04:20:48.026370+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:20:48.026370+0000","last_undegraded":"2023-10-09T04:20:48.026370+0000","last_fullsized":"2023-10-09T04:20:48.026370+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":511,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.068193+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:20:48.068193+0000","last_peered":"2023-10-09T04:20:48.068193+0000","last_clean":"2023-10-09T04:20:48.068193+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:20:48.068193+0000","last_undegraded":"2023-10-09T04:20:48.068193+0000","last_fullsized":"2023-10-09T04:20:48.068193+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":354,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.024675+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:20:48.024675+0000","last_peered":"2023-10-09T04:20:48.024675+0000","last_clean":"2023-10-09T04:20:48.024675+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:20:48.024675+0000","last_undegraded":"2023-10-09T04:20:48.024675+0000","last_fullsized":"2023-10-09T04:20:48.024675+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":38,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.973184+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:20:46.973184+0000","last_peered":"2023-10-09T04:20:46.973184+0000","last_clean":"2023-10-09T04:20:46.973184+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:20:46.973184+0000","last_undegraded":"2023-10-09T04:20:46.973184+0000","last_fullsized":"2023-10-09T04:20:46.973184+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":505,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.988858+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:20:47.988858+0000","last_peered":"2023-10-09T04:20:47.988858+0000","last_clean":"2023-10-09T04:20:47.988858+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:20:47.988858+0000","last_undegraded":"2023-10-09T04:20:47.988858+0000","last_fullsized":"2023-10-09T04:20:47.988858+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":338,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.063444+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:20:48.063444+0000","last_peered":"2023-10-09T04:20:48.063444+0000","last_clean":"2023-10-09T04:20:48.063444+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:20:48.063444+0000","last_undegraded":"2023-10-09T04:20:48.063444+0000","last_fullsized":"2023-10-09T04:20:48.063444+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":339,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989655+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:20:47.989655+0000","last_peered":"2023-10-09T04:20:47.989655+0000","last_clean":"2023-10-09T04:20:47.989655+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:20:47.989655+0000","last_undegraded":"2023-10-09T04:20:47.989655+0000","last_fullsized":"2023-10-09T04:20:47.989655+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":502,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.975479+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:20:46.975479+0000","last_peered":"2023-10-09T04:20:46.975479+0000","last_clean":"2023-10-09T04:20:46.975479+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:20:46.975479+0000","last_undegraded":"2023-10-09T04:20:46.975479+0000","last_fullsized":"2023-10-09T04:20:46.975479+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026016+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:20:48.026016+0000","last_peered":"2023-10-09T04:20:48.026016+0000","last_clean":"2023-10-09T04:20:48.026016+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:20:48.026016+0000","last_undegraded":"2023-10-09T04:20:48.026016+0000","last_fullsized":"2023-10-09T04:20:48.026016+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":337,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.063861+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:20:48.063861+0000","last_peered":"2023-10-09T04:20:48.063861+0000","last_clean":"2023-10-09T04:20:48.063861+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:20:48.063861+0000","last_undegraded":"2023-10-09T04:20:48.063861+0000","last_fullsized":"2023-10-09T04:20:48.063861+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":505,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989212+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:20:47.989212+0000","last_peered":"2023-10-09T04:20:47.989212+0000","last_clean":"2023-10-09T04:20:47.989212+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:20:47.989212+0000","last_undegraded":"2023-10-09T04:20:47.989212+0000","last_fullsized":"2023-10-09T04:20:47.989212+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":350,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.066143+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:20:48.066143+0000","last_peered":"2023-10-09T04:20:48.066143+0000","last_clean":"2023-10-09T04:20:48.066143+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:20:48.066143+0000","last_undegraded":"2023-10-09T04:20:48.066143+0000","last_fullsized":"2023-10-09T04:20:48.066143+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.976492+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:20:46.976492+0000","last_peered":"2023-10-09T04:20:46.976492+0000","last_clean":"2023-10-09T04:20:46.976492+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:20:46.976492+0000","last_undegraded":"2023-10-09T04:20:46.976492+0000","last_fullsized":"2023-10-09T04:20:46.976492+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"361'9059","reported_seq":9124,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:51.695921+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:20:51.695921+0000","last_peered":"2023-10-09T04:20:51.695921+0000","last_clean":"2023-10-09T04:20:51.695921+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:20:51.695921+0000","last_undegraded":"2023-10-09T04:20:51.695921+0000","last_fullsized":"2023-10-09T04:20:51.695921+0000","mapping_epoch":306,"log_start":"321'2800","ondisk_log_start":"321'2800","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204537856,"num_objects":3121,"num_object_clones":0,"num_object_copies":9363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9059,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":763,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.024023+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:20:48.024023+0000","last_peered":"2023-10-09T04:20:48.024023+0000","last_clean":"2023-10-09T04:20:48.024023+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:20:48.024023+0000","last_undegraded":"2023-10-09T04:20:48.024023+0000","last_fullsized":"2023-10-09T04:20:48.024023+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":349,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023179+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:20:48.023179+0000","last_peered":"2023-10-09T04:20:48.023179+0000","last_clean":"2023-10-09T04:20:48.023179+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:20:48.023179+0000","last_undegraded":"2023-10-09T04:20:48.023179+0000","last_fullsized":"2023-10-09T04:20:48.023179+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":312,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.988982+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:20:47.988982+0000","last_peered":"2023-10-09T04:20:47.988982+0000","last_clean":"2023-10-09T04:20:47.988982+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:20:47.988982+0000","last_undegraded":"2023-10-09T04:20:47.988982+0000","last_fullsized":"2023-10-09T04:20:47.988982+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"360'9048","reported_seq":9121,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.319516+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:20:48.319516+0000","last_peered":"2023-10-09T04:20:48.319516+0000","last_clean":"2023-10-09T04:20:48.319516+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:20:48.319516+0000","last_undegraded":"2023-10-09T04:20:48.319516+0000","last_fullsized":"2023-10-09T04:20:48.319516+0000","mapping_epoch":310,"log_start":"325'3773","ondisk_log_start":"325'3773","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5275,"log_dups_size":0,"ondisk_log_size":5275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219414528,"num_objects":3348,"num_object_clones":0,"num_object_copies":10044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9048,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":527,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.979198+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:20:46.979198+0000","last_peered":"2023-10-09T04:20:46.979198+0000","last_clean":"2023-10-09T04:20:46.979198+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:20:46.979198+0000","last_undegraded":"2023-10-09T04:20:46.979198+0000","last_fullsized":"2023-10-09T04:20:46.979198+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":353,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023321+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:20:48.023321+0000","last_peered":"2023-10-09T04:20:48.023321+0000","last_clean":"2023-10-09T04:20:48.023321+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:20:48.023321+0000","last_undegraded":"2023-10-09T04:20:48.023321+0000","last_fullsized":"2023-10-09T04:20:48.023321+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":135,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026107+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:20:48.026107+0000","last_peered":"2023-10-09T04:20:48.026107+0000","last_clean":"2023-10-09T04:20:48.026107+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:20:48.026107+0000","last_undegraded":"2023-10-09T04:20:48.026107+0000","last_fullsized":"2023-10-09T04:20:48.026107+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"361'9337","reported_seq":11098,"reported_epoch":361,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:20:50.858459+0000","last_change":"2023-10-09T04:20:46.011306+0000","last_active":"2023-10-09T04:20:50.858459+0000","last_peered":"2023-10-09T04:20:50.858459+0000","last_clean":"2023-10-09T04:20:00.944119+0000","last_became_active":"2023-10-09T04:20:03.092473+0000","last_became_peered":"2023-10-09T04:20:03.092473+0000","last_unstale":"2023-10-09T04:20:50.858459+0000","last_undegraded":"2023-10-09T04:20:50.858459+0000","last_fullsized":"2023-10-09T04:20:50.858459+0000","mapping_epoch":349,"log_start":"324'3418","ondisk_log_start":"324'3418","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5919,"log_dups_size":0,"ondisk_log_size":5919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:49.431905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212271104,"num_objects":3239,"num_object_clones":0,"num_object_copies":9717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2431,"num_objects_unfound":0,"num_objects_dirty":3239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9337,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":843,"num_bytes_recovered":55246848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":3239}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":495,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989561+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:20:47.989561+0000","last_peered":"2023-10-09T04:20:47.989561+0000","last_clean":"2023-10-09T04:20:47.989561+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:20:47.989561+0000","last_undegraded":"2023-10-09T04:20:47.989561+0000","last_fullsized":"2023-10-09T04:20:47.989561+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":318,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.024119+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:20:48.024119+0000","last_peered":"2023-10-09T04:20:48.024119+0000","last_clean":"2023-10-09T04:20:48.024119+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:20:48.024119+0000","last_undegraded":"2023-10-09T04:20:48.024119+0000","last_fullsized":"2023-10-09T04:20:48.024119+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":292,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.974829+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:20:46.974829+0000","last_peered":"2023-10-09T04:20:46.974829+0000","last_clean":"2023-10-09T04:20:46.974829+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:20:46.974829+0000","last_undegraded":"2023-10-09T04:20:46.974829+0000","last_fullsized":"2023-10-09T04:20:46.974829+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"361'9169","reported_seq":9232,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.843926+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:20:50.843926+0000","last_peered":"2023-10-09T04:20:50.843926+0000","last_clean":"2023-10-09T04:20:50.843926+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:20:50.843926+0000","last_undegraded":"2023-10-09T04:20:50.843926+0000","last_fullsized":"2023-10-09T04:20:50.843926+0000","mapping_epoch":306,"log_start":"324'3187","ondisk_log_start":"324'3187","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5982,"log_dups_size":0,"ondisk_log_size":5982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206766080,"num_objects":3155,"num_object_clones":0,"num_object_copies":9465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9169,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":505,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026648+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:20:48.026648+0000","last_peered":"2023-10-09T04:20:48.026648+0000","last_clean":"2023-10-09T04:20:48.026648+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:20:48.026648+0000","last_undegraded":"2023-10-09T04:20:48.026648+0000","last_fullsized":"2023-10-09T04:20:48.026648+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":350,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989405+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:20:47.989405+0000","last_peered":"2023-10-09T04:20:47.989405+0000","last_clean":"2023-10-09T04:20:47.989405+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:20:47.989405+0000","last_undegraded":"2023-10-09T04:20:47.989405+0000","last_fullsized":"2023-10-09T04:20:47.989405+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":338,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.064757+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:20:48.064757+0000","last_peered":"2023-10-09T04:20:48.064757+0000","last_clean":"2023-10-09T04:20:48.064757+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:20:48.064757+0000","last_undegraded":"2023-10-09T04:20:48.064757+0000","last_fullsized":"2023-10-09T04:20:48.064757+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"361'8797","reported_seq":12233,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.885447+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:20:50.885447+0000","last_peered":"2023-10-09T04:20:50.885447+0000","last_clean":"2023-10-09T04:20:50.885447+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:20:50.885447+0000","last_undegraded":"2023-10-09T04:20:50.885447+0000","last_fullsized":"2023-10-09T04:20:50.885447+0000","mapping_epoch":320,"log_start":"324'3500","ondisk_log_start":"324'3500","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5297,"log_dups_size":0,"ondisk_log_size":5297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201916416,"num_objects":3081,"num_object_clones":0,"num_object_copies":9243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8797,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":784,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023710+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:20:48.023710+0000","last_peered":"2023-10-09T04:20:48.023710+0000","last_clean":"2023-10-09T04:20:48.023710+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:20:48.023710+0000","last_undegraded":"2023-10-09T04:20:48.023710+0000","last_fullsized":"2023-10-09T04:20:48.023710+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":321,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.064834+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:20:48.064834+0000","last_peered":"2023-10-09T04:20:48.064834+0000","last_clean":"2023-10-09T04:20:48.064834+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:20:48.064834+0000","last_undegraded":"2023-10-09T04:20:48.064834+0000","last_fullsized":"2023-10-09T04:20:48.064834+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":135,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026302+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:20:48.026302+0000","last_peered":"2023-10-09T04:20:48.026302+0000","last_clean":"2023-10-09T04:20:48.026302+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:20:48.026302+0000","last_undegraded":"2023-10-09T04:20:48.026302+0000","last_fullsized":"2023-10-09T04:20:48.026302+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"360'8941","reported_seq":12516,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.304032+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:20:48.304032+0000","last_peered":"2023-10-09T04:20:48.304032+0000","last_clean":"2023-10-09T04:20:48.304032+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:20:48.304032+0000","last_undegraded":"2023-10-09T04:20:48.304032+0000","last_fullsized":"2023-10-09T04:20:48.304032+0000","mapping_epoch":329,"log_start":"324'3660","ondisk_log_start":"324'3660","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5281,"log_dups_size":0,"ondisk_log_size":5281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218562560,"num_objects":3335,"num_object_clones":0,"num_object_copies":10005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8941,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":530,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.988802+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:20:47.988802+0000","last_peered":"2023-10-09T04:20:47.988802+0000","last_clean":"2023-10-09T04:20:47.988802+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:20:47.988802+0000","last_undegraded":"2023-10-09T04:20:47.988802+0000","last_fullsized":"2023-10-09T04:20:47.988802+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026062+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:20:48.026062+0000","last_peered":"2023-10-09T04:20:48.026062+0000","last_clean":"2023-10-09T04:20:48.026062+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:20:48.026062+0000","last_undegraded":"2023-10-09T04:20:48.026062+0000","last_fullsized":"2023-10-09T04:20:48.026062+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026596+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:20:48.026596+0000","last_peered":"2023-10-09T04:20:48.026596+0000","last_clean":"2023-10-09T04:20:48.026596+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:20:48.026596+0000","last_undegraded":"2023-10-09T04:20:48.026596+0000","last_fullsized":"2023-10-09T04:20:48.026596+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"361'9094","reported_seq":9167,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.889280+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:20:50.889280+0000","last_peered":"2023-10-09T04:20:50.889280+0000","last_clean":"2023-10-09T04:20:50.889280+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:20:50.889280+0000","last_undegraded":"2023-10-09T04:20:50.889280+0000","last_fullsized":"2023-10-09T04:20:50.889280+0000","mapping_epoch":310,"log_start":"325'3800","ondisk_log_start":"325'3800","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5294,"log_dups_size":0,"ondisk_log_size":5294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200015872,"num_objects":3052,"num_object_clones":0,"num_object_copies":9156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9094,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":485,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.973151+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:20:46.973151+0000","last_peered":"2023-10-09T04:20:46.973151+0000","last_clean":"2023-10-09T04:20:46.973151+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:20:46.973151+0000","last_undegraded":"2023-10-09T04:20:46.973151+0000","last_fullsized":"2023-10-09T04:20:46.973151+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":105,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.025947+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:20:48.025947+0000","last_peered":"2023-10-09T04:20:48.025947+0000","last_clean":"2023-10-09T04:20:48.025947+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:20:48.025947+0000","last_undegraded":"2023-10-09T04:20:48.025947+0000","last_fullsized":"2023-10-09T04:20:48.025947+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":155,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026735+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:20:48.026735+0000","last_peered":"2023-10-09T04:20:48.026735+0000","last_clean":"2023-10-09T04:20:48.026735+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:20:48.026735+0000","last_undegraded":"2023-10-09T04:20:48.026735+0000","last_fullsized":"2023-10-09T04:20:48.026735+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"360'9057","reported_seq":18167,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.250489+0000","last_change":"2023-10-09T04:20:48.014336+0000","last_active":"2023-10-09T04:20:48.250489+0000","last_peered":"2023-10-09T04:20:48.250489+0000","last_clean":"2023-10-09T04:20:48.250489+0000","last_became_active":"2023-10-09T04:20:47.989792+0000","last_became_peered":"2023-10-09T04:20:47.989792+0000","last_unstale":"2023-10-09T04:20:48.250489+0000","last_undegraded":"2023-10-09T04:20:48.250489+0000","last_fullsized":"2023-10-09T04:20:48.250489+0000","mapping_epoch":359,"log_start":"325'3781","ondisk_log_start":"325'3781","created":306,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5276,"log_dups_size":0,"ondisk_log_size":5276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:26.170681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220528640,"num_objects":3365,"num_object_clones":0,"num_object_copies":10095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9057,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4512,"num_bytes_recovered":295698432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":505,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989009+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:20:47.989009+0000","last_peered":"2023-10-09T04:20:47.989009+0000","last_clean":"2023-10-09T04:20:47.989009+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:20:47.989009+0000","last_undegraded":"2023-10-09T04:20:47.989009+0000","last_fullsized":"2023-10-09T04:20:47.989009+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":22,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.976290+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:20:46.976290+0000","last_peered":"2023-10-09T04:20:46.976290+0000","last_clean":"2023-10-09T04:20:46.976290+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:20:46.976290+0000","last_undegraded":"2023-10-09T04:20:46.976290+0000","last_fullsized":"2023-10-09T04:20:46.976290+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":27,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.986730+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:20:46.986730+0000","last_peered":"2023-10-09T04:20:46.986730+0000","last_clean":"2023-10-09T04:20:46.986730+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:20:46.986730+0000","last_undegraded":"2023-10-09T04:20:46.986730+0000","last_fullsized":"2023-10-09T04:20:46.986730+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"361'9222","reported_seq":9295,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.731725+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:20:50.731725+0000","last_peered":"2023-10-09T04:20:50.731725+0000","last_clean":"2023-10-09T04:20:50.731725+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:20:50.731725+0000","last_undegraded":"2023-10-09T04:20:50.731725+0000","last_fullsized":"2023-10-09T04:20:50.731725+0000","mapping_epoch":310,"log_start":"324'3334","ondisk_log_start":"324'3334","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208928768,"num_objects":3188,"num_object_clones":0,"num_object_copies":9564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9222,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":525,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989760+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:20:47.989760+0000","last_peered":"2023-10-09T04:20:47.989760+0000","last_clean":"2023-10-09T04:20:47.989760+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:20:47.989760+0000","last_undegraded":"2023-10-09T04:20:47.989760+0000","last_fullsized":"2023-10-09T04:20:47.989760+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":339,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.974974+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:20:46.974974+0000","last_peered":"2023-10-09T04:20:46.974974+0000","last_clean":"2023-10-09T04:20:46.974974+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:20:46.974974+0000","last_undegraded":"2023-10-09T04:20:46.974974+0000","last_fullsized":"2023-10-09T04:20:46.974974+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":357,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989571+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:20:47.989571+0000","last_peered":"2023-10-09T04:20:47.989571+0000","last_clean":"2023-10-09T04:20:47.989571+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:20:47.989571+0000","last_undegraded":"2023-10-09T04:20:47.989571+0000","last_fullsized":"2023-10-09T04:20:47.989571+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":27,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.980858+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:20:46.980858+0000","last_peered":"2023-10-09T04:20:46.980858+0000","last_clean":"2023-10-09T04:20:46.980858+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:20:46.980858+0000","last_undegraded":"2023-10-09T04:20:46.980858+0000","last_fullsized":"2023-10-09T04:20:46.980858+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"361'9208","reported_seq":9273,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:51.704928+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:20:51.704928+0000","last_peered":"2023-10-09T04:20:51.704928+0000","last_clean":"2023-10-09T04:20:51.704928+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:20:51.704928+0000","last_undegraded":"2023-10-09T04:20:51.704928+0000","last_fullsized":"2023-10-09T04:20:51.704928+0000","mapping_epoch":306,"log_start":"321'2900","ondisk_log_start":"321'2900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202768384,"num_objects":3094,"num_object_clones":0,"num_object_copies":9282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9208,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":478,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989042+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:20:47.989042+0000","last_peered":"2023-10-09T04:20:47.989042+0000","last_clean":"2023-10-09T04:20:47.989042+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:20:47.989042+0000","last_undegraded":"2023-10-09T04:20:47.989042+0000","last_fullsized":"2023-10-09T04:20:47.989042+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":334,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989024+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:20:47.989024+0000","last_peered":"2023-10-09T04:20:47.989024+0000","last_clean":"2023-10-09T04:20:47.989024+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:20:47.989024+0000","last_undegraded":"2023-10-09T04:20:47.989024+0000","last_fullsized":"2023-10-09T04:20:47.989024+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026365+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:20:48.026365+0000","last_peered":"2023-10-09T04:20:48.026365+0000","last_clean":"2023-10-09T04:20:48.026365+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:20:48.026365+0000","last_undegraded":"2023-10-09T04:20:48.026365+0000","last_fullsized":"2023-10-09T04:20:48.026365+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":367,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.974887+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:20:46.974887+0000","last_peered":"2023-10-09T04:20:46.974887+0000","last_clean":"2023-10-09T04:20:46.974887+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:20:46.974887+0000","last_undegraded":"2023-10-09T04:20:46.974887+0000","last_fullsized":"2023-10-09T04:20:46.974887+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"361'9263","reported_seq":9326,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.681637+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:20:50.681637+0000","last_peered":"2023-10-09T04:20:50.681637+0000","last_clean":"2023-10-09T04:20:50.681637+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:20:50.681637+0000","last_undegraded":"2023-10-09T04:20:50.681637+0000","last_fullsized":"2023-10-09T04:20:50.681637+0000","mapping_epoch":306,"log_start":"324'3368","ondisk_log_start":"324'3368","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210829312,"num_objects":3217,"num_object_clones":0,"num_object_copies":9651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9263,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":494,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.063527+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:20:48.063527+0000","last_peered":"2023-10-09T04:20:48.063527+0000","last_clean":"2023-10-09T04:20:48.063527+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:20:48.063527+0000","last_undegraded":"2023-10-09T04:20:48.063527+0000","last_fullsized":"2023-10-09T04:20:48.063527+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":195,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.980974+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:20:46.980974+0000","last_peered":"2023-10-09T04:20:46.980974+0000","last_clean":"2023-10-09T04:20:46.980974+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:20:46.980974+0000","last_undegraded":"2023-10-09T04:20:46.980974+0000","last_fullsized":"2023-10-09T04:20:46.980974+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":343,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.975398+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:20:46.975398+0000","last_peered":"2023-10-09T04:20:46.975398+0000","last_clean":"2023-10-09T04:20:46.975398+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:20:46.975398+0000","last_undegraded":"2023-10-09T04:20:46.975398+0000","last_fullsized":"2023-10-09T04:20:46.975398+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":38,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.972994+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:20:46.972994+0000","last_peered":"2023-10-09T04:20:46.972994+0000","last_clean":"2023-10-09T04:20:46.972994+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:20:46.972994+0000","last_undegraded":"2023-10-09T04:20:46.972994+0000","last_fullsized":"2023-10-09T04:20:46.972994+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"361'9229","reported_seq":14072,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:20:51.738777+0000","last_change":"2023-10-09T04:20:05.060638+0000","last_active":"2023-10-09T04:20:51.738777+0000","last_peered":"2023-10-09T04:20:51.738777+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:20:02.106154+0000","last_became_peered":"2023-10-09T04:20:02.106154+0000","last_unstale":"2023-10-09T04:20:51.738777+0000","last_undegraded":"2023-10-09T04:20:51.738777+0000","last_fullsized":"2023-10-09T04:20:51.738777+0000","mapping_epoch":348,"log_start":"321'2900","ondisk_log_start":"321'2900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:45.805086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207028224,"num_objects":3159,"num_object_clones":0,"num_object_copies":9477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5017,"num_objects_unfound":0,"num_objects_dirty":3159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9229,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2377,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":3159}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":344,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989125+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:20:47.989125+0000","last_peered":"2023-10-09T04:20:47.989125+0000","last_clean":"2023-10-09T04:20:47.989125+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:20:47.989125+0000","last_undegraded":"2023-10-09T04:20:47.989125+0000","last_fullsized":"2023-10-09T04:20:47.989125+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":755,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.022945+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:20:48.022945+0000","last_peered":"2023-10-09T04:20:48.022945+0000","last_clean":"2023-10-09T04:20:48.022945+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:20:48.022945+0000","last_undegraded":"2023-10-09T04:20:48.022945+0000","last_fullsized":"2023-10-09T04:20:48.022945+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":287,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989108+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:20:47.989108+0000","last_peered":"2023-10-09T04:20:47.989108+0000","last_clean":"2023-10-09T04:20:47.989108+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:20:47.989108+0000","last_undegraded":"2023-10-09T04:20:47.989108+0000","last_fullsized":"2023-10-09T04:20:47.989108+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026654+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:20:48.026654+0000","last_peered":"2023-10-09T04:20:48.026654+0000","last_clean":"2023-10-09T04:20:48.026654+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:20:48.026654+0000","last_undegraded":"2023-10-09T04:20:48.026654+0000","last_fullsized":"2023-10-09T04:20:48.026654+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"361'9149","reported_seq":12793,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:51.668671+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:20:51.668671+0000","last_peered":"2023-10-09T04:20:51.668671+0000","last_clean":"2023-10-09T04:20:51.668671+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:20:51.668671+0000","last_undegraded":"2023-10-09T04:20:51.668671+0000","last_fullsized":"2023-10-09T04:20:51.668671+0000","mapping_epoch":323,"log_start":"321'2800","ondisk_log_start":"321'2800","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6349,"log_dups_size":0,"ondisk_log_size":6349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208732160,"num_objects":3185,"num_object_clones":0,"num_object_copies":9555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9149,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":503,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.975212+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:20:46.975212+0000","last_peered":"2023-10-09T04:20:46.975212+0000","last_clean":"2023-10-09T04:20:46.975212+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:20:46.975212+0000","last_undegraded":"2023-10-09T04:20:46.975212+0000","last_fullsized":"2023-10-09T04:20:46.975212+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":341,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.022846+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:20:48.022846+0000","last_peered":"2023-10-09T04:20:48.022846+0000","last_clean":"2023-10-09T04:20:48.022846+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:20:48.022846+0000","last_undegraded":"2023-10-09T04:20:48.022846+0000","last_fullsized":"2023-10-09T04:20:48.022846+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":357,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.068528+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:20:48.068528+0000","last_peered":"2023-10-09T04:20:48.068528+0000","last_clean":"2023-10-09T04:20:48.068528+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:20:48.068528+0000","last_undegraded":"2023-10-09T04:20:48.068528+0000","last_fullsized":"2023-10-09T04:20:48.068528+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":353,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.063622+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:20:48.063622+0000","last_peered":"2023-10-09T04:20:48.063622+0000","last_clean":"2023-10-09T04:20:48.063622+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:20:48.063622+0000","last_undegraded":"2023-10-09T04:20:48.063622+0000","last_fullsized":"2023-10-09T04:20:48.063622+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"361'9195","reported_seq":9258,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.893201+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:20:50.893201+0000","last_peered":"2023-10-09T04:20:50.893201+0000","last_clean":"2023-10-09T04:20:50.893201+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:20:50.893201+0000","last_undegraded":"2023-10-09T04:20:50.893201+0000","last_fullsized":"2023-10-09T04:20:50.893201+0000","mapping_epoch":306,"log_start":"325'3900","ondisk_log_start":"325'3900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5295,"log_dups_size":0,"ondisk_log_size":5295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214892544,"num_objects":3279,"num_object_clones":0,"num_object_copies":9837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9195,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":514,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026565+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:20:48.026565+0000","last_peered":"2023-10-09T04:20:48.026565+0000","last_clean":"2023-10-09T04:20:48.026565+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:20:48.026565+0000","last_undegraded":"2023-10-09T04:20:48.026565+0000","last_fullsized":"2023-10-09T04:20:48.026565+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":344,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.024159+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:20:48.024159+0000","last_peered":"2023-10-09T04:20:48.024159+0000","last_clean":"2023-10-09T04:20:48.024159+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:20:48.024159+0000","last_undegraded":"2023-10-09T04:20:48.024159+0000","last_fullsized":"2023-10-09T04:20:48.024159+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":333,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.975052+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:20:46.975052+0000","last_peered":"2023-10-09T04:20:46.975052+0000","last_clean":"2023-10-09T04:20:46.975052+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:20:46.975052+0000","last_undegraded":"2023-10-09T04:20:46.975052+0000","last_fullsized":"2023-10-09T04:20:46.975052+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":60,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989120+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:20:47.989120+0000","last_peered":"2023-10-09T04:20:47.989120+0000","last_clean":"2023-10-09T04:20:47.989120+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:20:47.989120+0000","last_undegraded":"2023-10-09T04:20:47.989120+0000","last_fullsized":"2023-10-09T04:20:47.989120+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"361'9192","reported_seq":20597,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.910769+0000","last_change":"2023-10-09T04:20:12.010618+0000","last_active":"2023-10-09T04:20:50.910769+0000","last_peered":"2023-10-09T04:20:50.910769+0000","last_clean":"2023-10-09T04:20:50.910769+0000","last_became_active":"2023-10-09T04:20:12.002110+0000","last_became_peered":"2023-10-09T04:20:12.002110+0000","last_unstale":"2023-10-09T04:20:50.910769+0000","last_undegraded":"2023-10-09T04:20:50.910769+0000","last_fullsized":"2023-10-09T04:20:50.910769+0000","mapping_epoch":354,"log_start":"325'3900","ondisk_log_start":"325'3900","created":306,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5292,"log_dups_size":0,"ondisk_log_size":5292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:12.102225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204406808,"num_objects":3120,"num_object_clones":0,"num_object_copies":9360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3120,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9192,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":505,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026477+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:20:48.026477+0000","last_peered":"2023-10-09T04:20:48.026477+0000","last_clean":"2023-10-09T04:20:48.026477+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:20:48.026477+0000","last_undegraded":"2023-10-09T04:20:48.026477+0000","last_fullsized":"2023-10-09T04:20:48.026477+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":293,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023420+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:20:48.023420+0000","last_peered":"2023-10-09T04:20:48.023420+0000","last_clean":"2023-10-09T04:20:48.023420+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:20:48.023420+0000","last_undegraded":"2023-10-09T04:20:48.023420+0000","last_fullsized":"2023-10-09T04:20:48.023420+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":327,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.976090+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:20:46.976090+0000","last_peered":"2023-10-09T04:20:46.976090+0000","last_clean":"2023-10-09T04:20:46.976090+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:20:46.976090+0000","last_undegraded":"2023-10-09T04:20:46.976090+0000","last_fullsized":"2023-10-09T04:20:46.976090+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":311,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023053+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:20:48.023053+0000","last_peered":"2023-10-09T04:20:48.023053+0000","last_clean":"2023-10-09T04:20:48.023053+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:20:48.023053+0000","last_undegraded":"2023-10-09T04:20:48.023053+0000","last_fullsized":"2023-10-09T04:20:48.023053+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"361'9311","reported_seq":9384,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.385729+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:20:50.385729+0000","last_peered":"2023-10-09T04:20:50.385729+0000","last_clean":"2023-10-09T04:20:50.385729+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:20:50.385729+0000","last_undegraded":"2023-10-09T04:20:50.385729+0000","last_fullsized":"2023-10-09T04:20:50.385729+0000","mapping_epoch":310,"log_start":"324'3700","ondisk_log_start":"324'3700","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212271104,"num_objects":3239,"num_object_clones":0,"num_object_copies":9717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9311,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":498,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.979050+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:20:46.979050+0000","last_peered":"2023-10-09T04:20:46.979050+0000","last_clean":"2023-10-09T04:20:46.979050+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:20:46.979050+0000","last_undegraded":"2023-10-09T04:20:46.979050+0000","last_fullsized":"2023-10-09T04:20:46.979050+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":192,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.986656+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:20:46.986656+0000","last_peered":"2023-10-09T04:20:46.986656+0000","last_clean":"2023-10-09T04:20:46.986656+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:20:46.986656+0000","last_undegraded":"2023-10-09T04:20:46.986656+0000","last_fullsized":"2023-10-09T04:20:46.986656+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":366,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.063711+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:20:48.063711+0000","last_peered":"2023-10-09T04:20:48.063711+0000","last_clean":"2023-10-09T04:20:48.063711+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:20:48.063711+0000","last_undegraded":"2023-10-09T04:20:48.063711+0000","last_fullsized":"2023-10-09T04:20:48.063711+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026033+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:20:48.026033+0000","last_peered":"2023-10-09T04:20:48.026033+0000","last_clean":"2023-10-09T04:20:48.026033+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:20:48.026033+0000","last_undegraded":"2023-10-09T04:20:48.026033+0000","last_fullsized":"2023-10-09T04:20:48.026033+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":520,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.068114+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:20:48.068114+0000","last_peered":"2023-10-09T04:20:48.068114+0000","last_clean":"2023-10-09T04:20:48.068114+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:20:48.068114+0000","last_undegraded":"2023-10-09T04:20:48.068114+0000","last_fullsized":"2023-10-09T04:20:48.068114+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":335,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.064909+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:20:48.064909+0000","last_peered":"2023-10-09T04:20:48.064909+0000","last_clean":"2023-10-09T04:20:48.064909+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:20:48.064909+0000","last_undegraded":"2023-10-09T04:20:48.064909+0000","last_fullsized":"2023-10-09T04:20:48.064909+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":195,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.986819+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:20:46.986819+0000","last_peered":"2023-10-09T04:20:46.986819+0000","last_clean":"2023-10-09T04:20:46.986819+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:20:46.986819+0000","last_undegraded":"2023-10-09T04:20:46.986819+0000","last_fullsized":"2023-10-09T04:20:46.986819+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":748,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023233+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:20:48.023233+0000","last_peered":"2023-10-09T04:20:48.023233+0000","last_clean":"2023-10-09T04:20:48.023233+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:20:48.023233+0000","last_undegraded":"2023-10-09T04:20:48.023233+0000","last_fullsized":"2023-10-09T04:20:48.023233+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":350,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989462+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:20:47.989462+0000","last_peered":"2023-10-09T04:20:47.989462+0000","last_clean":"2023-10-09T04:20:47.989462+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:20:47.989462+0000","last_undegraded":"2023-10-09T04:20:47.989462+0000","last_fullsized":"2023-10-09T04:20:47.989462+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026446+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:20:48.026446+0000","last_peered":"2023-10-09T04:20:48.026446+0000","last_clean":"2023-10-09T04:20:48.026446+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:20:48.026446+0000","last_undegraded":"2023-10-09T04:20:48.026446+0000","last_fullsized":"2023-10-09T04:20:48.026446+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":347,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:47.989570+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:20:47.989570+0000","last_peered":"2023-10-09T04:20:47.989570+0000","last_clean":"2023-10-09T04:20:47.989570+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:20:47.989570+0000","last_undegraded":"2023-10-09T04:20:47.989570+0000","last_fullsized":"2023-10-09T04:20:47.989570+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.026103+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:20:48.026103+0000","last_peered":"2023-10-09T04:20:48.026103+0000","last_clean":"2023-10-09T04:20:48.026103+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:20:48.026103+0000","last_undegraded":"2023-10-09T04:20:48.026103+0000","last_fullsized":"2023-10-09T04:20:48.026103+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":783,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.023783+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:20:48.023783+0000","last_peered":"2023-10-09T04:20:48.023783+0000","last_clean":"2023-10-09T04:20:48.023783+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:20:48.023783+0000","last_undegraded":"2023-10-09T04:20:48.023783+0000","last_fullsized":"2023-10-09T04:20:48.023783+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":205,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.987646+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:20:46.987646+0000","last_peered":"2023-10-09T04:20:46.987646+0000","last_clean":"2023-10-09T04:20:46.987646+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:20:46.987646+0000","last_undegraded":"2023-10-09T04:20:46.987646+0000","last_fullsized":"2023-10-09T04:20:46.987646+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":195,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.981044+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:20:46.981044+0000","last_peered":"2023-10-09T04:20:46.981044+0000","last_clean":"2023-10-09T04:20:46.981044+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:20:46.981044+0000","last_undegraded":"2023-10-09T04:20:46.981044+0000","last_fullsized":"2023-10-09T04:20:46.981044+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":502,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:20:46.972790+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:20:46.972790+0000","last_peered":"2023-10-09T04:20:46.972790+0000","last_clean":"2023-10-09T04:20:46.972790+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:20:46.972790+0000","last_undegraded":"2023-10-09T04:20:46.972790+0000","last_fullsized":"2023-10-09T04:20:46.972790+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":355,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.064395+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:20:48.064395+0000","last_peered":"2023-10-09T04:20:48.064395+0000","last_clean":"2023-10-09T04:20:48.064395+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:20:48.064395+0000","last_undegraded":"2023-10-09T04:20:48.064395+0000","last_fullsized":"2023-10-09T04:20:48.064395+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":115,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T04:20:48.025878+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:20:48.025878+0000","last_peered":"2023-10-09T04:20:48.025878+0000","last_clean":"2023-10-09T04:20:48.025878+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:20:48.025878+0000","last_undegraded":"2023-10-09T04:20:48.025878+0000","last_fullsized":"2023-10-09T04:20:48.025878+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3353870360,"num_objects":51177,"num_object_clones":0,"num_object_copies":153531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7448,"num_objects_unfound":0,"num_objects_dirty":51177,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146271,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18585,"num_bytes_recovered":1217921048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10732978176,"data_stored":10732961888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91550,"ondisk_log_size":91550,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201620,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2843696,"kb_used_data":2258024,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":90901456,"statfs":{"total":95995035648,"available":93083090944,"internally_reserved":0,"allocated":2312216576,"data_stored":2311193549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":9.6059999999999999,"5min":3.8919999999999999,"15min":3.5350000000000001},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":148.65899999999999,"5min":148.65899999999999,"15min":148.65899999999999},"last":1.681},{"interface":"front","average":{"1min":10.054,"5min":3.8660000000000001,"15min":3.5179999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":148.642,"5min":148.642,"15min":148.642},"last":2.302}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2430000000000003,"5min":4.3739999999999997,"15min":4.21},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":29.792000000000002,"5min":103.324,"15min":245.28700000000001},"last":2.5680000000000001},{"interface":"front","average":{"1min":4.9909999999999997,"5min":4.6210000000000004,"15min":4.1660000000000004},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":29.268000000000001,"5min":104.248,"15min":244.626},"last":0.40200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.698,"15min":2.1240000000000001},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.2270000000000003,"5min":14.076000000000001,"15min":76.774000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.5,"5min":2.528,"15min":2.8540000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":6.5460000000000003,"5min":15.597,"15min":99.372},"last":0.45000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":2.6419999999999999,"15min":2.633},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":3.7149999999999999,"5min":85.575000000000003,"15min":99.134},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.1080000000000001,"15min":2.8769999999999998},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":6.1459999999999999,"5min":85.528000000000006,"15min":99.117999999999995},"last":0.47799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.96499999999999997,"15min":0.85799999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.1799999999999997,"5min":8.9359999999999999,"15min":16.195},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.236,"5min":1.714,"15min":1.1539999999999999},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26000000000000001},"max":{"1min":4.4500000000000002,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":1.883,"15min":1.4810000000000001},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":13.122999999999999,"5min":13.122999999999999,"15min":120.066},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.3700000000000001,"15min":1.359},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8360000000000003,"5min":9,"15min":120.09699999999999},"last":0.38900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.667,"15min":1.1140000000000001},"min":{"1min":0.34300000000000003,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":7.3310000000000004,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.438},{"interface":"front","average":{"1min":1.149,"5min":1.069,"15min":0.92000000000000004},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.7060000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.58799999999999997}]}]},{"osd":6,"up_from":0,"seq":201863463257,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089565,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2746872,"kb_used_data":2140592,"kb_used_omap":9,"kb_used_meta":606262,"kb_avail":90998280,"statfs":{"total":95995035648,"available":93182238720,"internally_reserved":0,"allocated":2191966208,"data_stored":2190454882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":620813289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":3.8759999999999999,"15min":4.0629999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":19.617000000000001,"5min":98.257999999999996,"15min":98.257999999999996},"last":3.7240000000000002},{"interface":"front","average":{"1min":2.1949999999999998,"5min":6.3319999999999999,"15min":5.2000000000000002},"min":{"1min":0.27100000000000002,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":9.9960000000000004,"5min":154.654,"15min":154.654},"last":0.41499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.1699999999999999,"15min":2.7170000000000001},"min":{"1min":0.221,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":12.791,"5min":19.564,"15min":121.54600000000001},"last":3.887},{"interface":"front","average":{"1min":3.702,"5min":2.657,"15min":3.1400000000000001},"min":{"1min":0.28100000000000003,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":14.503,"5min":16.114999999999998,"15min":99.899000000000001},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.8660000000000001,"15min":3.7770000000000001},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":19.640000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":2.9409999999999998},{"interface":"front","average":{"1min":2.4609999999999999,"5min":3.8170000000000002,"15min":4.6970000000000001},"min":{"1min":0.307,"5min":0.251,"15min":0.251},"max":{"1min":12.353999999999999,"5min":68.563999999999993,"15min":138.49799999999999},"last":1.363}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.2290000000000001,"15min":2.3370000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":7.5250000000000004,"5min":16.27,"15min":99.989999999999995},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.458,"5min":1.9530000000000001,"15min":2.1829999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":5.282,"5min":13.428000000000001,"15min":100.249},"last":0.63}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":3.899,"5min":3.2280000000000002,"15min":2.6539999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":25.696999999999999,"5min":38.848999999999997,"15min":99.933999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.8700000000000001,"5min":3.145,"15min":2.6909999999999998},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":19.977,"5min":39.593000000000004,"15min":100.34099999999999},"last":0.66200000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.2120000000000002,"15min":2.5},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":5.3579999999999997,"5min":13.377000000000001,"15min":100.309},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.9570000000000001,"15min":2.1749999999999998},"min":{"1min":0.41399999999999998,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":100.143},"last":2.875}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.7050000000000001,"15min":2.6419999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":5.7430000000000003,"5min":17.460000000000001,"15min":100.214},"last":3.7490000000000001},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.3359999999999999,"15min":2.319},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":19.593,"5min":19.593,"15min":100.113},"last":2.915}]}]},{"osd":0,"up_from":84,"seq":360777253092,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1520920,"kb_used_data":1011280,"kb_used_omap":28,"kb_used_meta":509603,"kb_avail":92224232,"statfs":{"total":95995035648,"available":94437613568,"internally_reserved":0,"allocated":1035550720,"data_stored":1033963762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":521834320},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":8.9990000000000006,"15min":5.2709999999999999},"min":{"1min":0.20200000000000001,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":6.1609999999999996,"5min":209.215,"15min":209.215},"last":0.318},{"interface":"front","average":{"1min":3.0499999999999998,"5min":6.8630000000000004,"15min":5.032},"min":{"1min":0.29499999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.547000000000001,"5min":120.934,"15min":120.934},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:20:47 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":2.149,"15min":3.2570000000000001},"min":{"1min":0.216,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.131,"5min":17.466999999999999,"15min":118.54000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.7530000000000001,"5min":3.1480000000000001,"15min":3.6899999999999999},"min":{"1min":0.224,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":11.050000000000001,"5min":20.262,"15min":118.727},"last":0.35299999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":3.1400000000000001,"15min":5.2759999999999998},"min":{"1min":0.25700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":9.1669999999999998,"5min":99.709000000000003,"15min":156.488},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.3919999999999999,"5min":3.1720000000000002,"15min":5.3620000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":10.577,"5min":99.560000000000002,"15min":156.50700000000001},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":4.899,"5min":2.9969999999999999,"15min":2.7250000000000001},"min":{"1min":0.35399999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":49.840000000000003,"5min":63.375999999999998,"15min":118.633},"last":0.47099999999999997},{"interface":"front","average":{"1min":5.0609999999999999,"5min":3.3290000000000002,"15min":3.0670000000000002},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":49.607999999999997,"5min":49.607999999999997,"15min":118.913},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":5.4800000000000004,"15min":4.2960000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29499999999999998},"max":{"1min":7.0570000000000004,"5min":156.96100000000001,"15min":182.68899999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.0870000000000002,"5min":4.5540000000000003,"15min":3.7749999999999999},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.024,"5min":156.97300000000001,"15min":182.66300000000001},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.528,"15min":2.996},"min":{"1min":0.35499999999999998,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":7.0709999999999997,"5min":63.468000000000004,"15min":137.36099999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.206,"5min":2.5859999999999999,"15min":3.0329999999999999},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":10.113,"5min":63.447000000000003,"15min":136.88999999999999},"last":0.63600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.6419999999999999,"15min":3.677},"min":{"1min":0.40000000000000002,"5min":0.375,"15min":0.32100000000000001},"max":{"1min":9.1699999999999999,"5min":63.509,"15min":185.024},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.5979999999999999,"5min":2.7770000000000001,"15min":3.3479999999999999},"min":{"1min":0.44700000000000001,"5min":0.36699999999999999,"15min":0.32700000000000001},"max":{"1min":10.077,"5min":63.481999999999999,"15min":184.98599999999999},"last":0.59899999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926039,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1631004,"kb_used_data":1067092,"kb_used_omap":81,"kb_used_meta":563822,"kb_avail":92114148,"statfs":{"total":95995035648,"available":94324887552,"internally_reserved":0,"allocated":1092702208,"data_stored":1091167940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":577353946},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":5.3929999999999998,"15min":4.024},"min":{"1min":0.22500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":17.122,"5min":81.385999999999996,"15min":81.385999999999996},"last":0.17699999999999999},{"interface":"front","average":{"1min":1.228,"5min":4.617,"15min":3.9540000000000002},"min":{"1min":0.22500000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":4.1070000000000002,"5min":79.453999999999994,"15min":79.453999999999994},"last":1.4690000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9340000000000002,"5min":7.2270000000000003,"15min":4.7549999999999999},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.191},"max":{"1min":72.075999999999993,"5min":158.13399999999999,"15min":158.13399999999999},"last":0.19700000000000001},{"interface":"front","average":{"1min":6.4489999999999998,"5min":7.883,"15min":5.0590000000000002},"min":{"1min":0.27600000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":69.754999999999995,"5min":161.12,"15min":161.12},"last":0.316}]},{"osd":3,"last update":"Mon Oct 9 04:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":3.4700000000000002,"15min":2.8170000000000002},"min":{"1min":0.28000000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.409000000000001,"5min":58.997,"15min":58.997},"last":1.927},{"interface":"front","average":{"1min":2.1499999999999999,"5min":3.5030000000000001,"15min":3.0270000000000001},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.488,"5min":66.135000000000005,"15min":66.135000000000005},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":2.7549999999999999,"15min":2.2650000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":12.775,"5min":58.798000000000002,"15min":58.798000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.75,"5min":2.5619999999999998,"15min":2.093},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":6.5010000000000003,"5min":56.616,"15min":56.616},"last":0.49199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":3.9660000000000002,"15min":2.7269999999999999},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":82.030000000000001,"5min":82.030000000000001,"15min":82.030000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":7.4290000000000003,"5min":4.2430000000000003,"15min":2.9119999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":82.016000000000005,"5min":82.016000000000005,"15min":82.016000000000005},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.6949999999999998,"15min":2.5579999999999998},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.0129999999999999,"5min":56.561,"15min":114.14},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.629,"15min":2.6469999999999998},"min":{"1min":0.40999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2110000000000003,"5min":56.460999999999999,"15min":114.123},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.0700000000000003,"15min":3.1989999999999998},"min":{"1min":0.42599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":101.675,"5min":101.675,"15min":117.32599999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":8.2639999999999993,"5min":4.1589999999999998,"15min":3.2639999999999998},"min":{"1min":0.46100000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":101.72499999999999,"5min":101.72499999999999,"15min":117.28700000000001},"last":0.53900000000000003}]}]},{"osd":3,"up_from":99,"seq":425201762513,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2050420,"kb_used_data":1306668,"kb_used_omap":7,"kb_used_meta":743736,"kb_avail":91694732,"statfs":{"total":95995035648,"available":93895405568,"internally_reserved":0,"allocated":1338028032,"data_stored":1336526615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":761586280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":3.3730000000000002,"15min":5.4059999999999997},"min":{"1min":0.214,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":6.4169999999999998,"5min":78.771000000000001,"15min":125.765},"last":2.0950000000000002},{"interface":"front","average":{"1min":2.899,"5min":4.2080000000000002,"15min":5.8280000000000003},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.119999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":22.446999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":5.8979999999999997,"15min":5.2619999999999996},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":8.3230000000000004,"5min":103.143,"15min":186.21299999999999},"last":8.3960000000000008},{"interface":"front","average":{"1min":1.6060000000000001,"5min":5.7789999999999999,"15min":5.0309999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.193},"max":{"1min":5.9219999999999997,"5min":105.19,"15min":191.78},"last":2.8809999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.1660000000000004,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.8760000000000003,"5min":127.599,"15min":127.599},"last":1.5629999999999999},{"interface":"front","average":{"1min":2.359,"5min":6.0670000000000002,"15min":5.0119999999999996},"min":{"1min":0.29799999999999999,"5min":0.221,"15min":0.214},"max":{"1min":10.143000000000001,"5min":129.98599999999999,"15min":129.98599999999999},"last":6.7480000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":2.274,"15min":2.7240000000000002},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.013999999999999,"5min":27.539999999999999,"15min":106.85899999999999},"last":6.798},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.2919999999999998,"15min":2.1760000000000002},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.738,"5min":26.663,"15min":37.442},"last":3.6789999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.6280000000000001,"15min":2.2010000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":3.7749999999999999,"5min":17.901,"15min":17.901},"last":3.633},{"interface":"front","average":{"1min":1.083,"5min":1.766,"15min":1.887},"min":{"1min":0.375,"5min":0.248,"15min":0.248},"max":{"1min":4.6340000000000003,"5min":18.068000000000001,"15min":18.068000000000001},"last":2.2320000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.7509999999999999,"15min":2.2610000000000001},"min":{"1min":0.378,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4589999999999996,"5min":14.567,"15min":106.036},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.48,"15min":2.6509999999999998},"min":{"1min":0.42699999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.071,"5min":14.529,"15min":106.741},"last":6.8479999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":2.6720000000000002,"15min":2.786},"min":{"1min":0.39000000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":28.184000000000001,"5min":28.184000000000001,"15min":106.82599999999999},"last":6.8239999999999998},{"interface":"front","average":{"1min":3.1480000000000001,"5min":2.827,"15min":2.7490000000000001},"min":{"1min":0.44800000000000001,"5min":0.40000000000000002,"15min":0.35999999999999999},"max":{"1min":26.648,"5min":26.648,"15min":106.084},"last":3.7029999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986529,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1385172,"kb_used_data":1083788,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92359980,"statfs":{"total":95995035648,"available":94576619520,"internally_reserved":0,"allocated":1109798912,"data_stored":1108795276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":6.7859999999999996,"5min":9.9049999999999994,"15min":5.9029999999999996},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":72.759,"5min":154.62100000000001,"15min":154.62100000000001},"last":5.7809999999999997},{"interface":"front","average":{"1min":6.7279999999999998,"5min":10.082000000000001,"15min":5.9210000000000003},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":72.704999999999998,"5min":154.37299999999999,"15min":154.37299999999999},"last":2.8039999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":7.8129999999999997,"5min":9.4969999999999999,"15min":5.1180000000000003},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":69.706000000000003,"5min":135.25399999999999,"15min":135.25399999999999},"last":1.004},{"interface":"front","average":{"1min":7.3220000000000001,"5min":9.1270000000000007,"15min":5.0060000000000002},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":69.561000000000007,"5min":134.44300000000001,"15min":134.44300000000001},"last":5.1710000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.8180000000000001,"15min":2.1560000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":8.202,"5min":8.7400000000000002,"15min":123.598},"last":4.5199999999999996},{"interface":"front","average":{"1min":2.5299999999999998,"5min":1.8440000000000001,"15min":2.1160000000000001},"min":{"1min":0.33400000000000002,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":11.064,"5min":11.064,"15min":123.619},"last":5.4610000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.052,"15min":2.29},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.0069999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.165,"15min":2.4849999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.7579999999999991,"5min":8.9169999999999998,"15min":127.90000000000001},"last":6.21}]},{"osd":5,"last update":"Mon Oct 9 04:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4769999999999999,"5min":3.6099999999999999,"15min":1.6279999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":13.599,"5min":111.809,"15min":111.809},"last":2.9489999999999998},{"interface":"front","average":{"1min":2.532,"5min":3.7200000000000002,"15min":1.7929999999999999},"min":{"1min":0.316,"5min":0.252,"15min":0.25},"max":{"1min":9.1530000000000005,"5min":112.002,"15min":112.002},"last":6.4800000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.395,"15min":1.3680000000000001},"min":{"1min":0.35699999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":6.9470000000000001,"5min":76.307000000000002,"15min":76.307000000000002},"last":6.4539999999999997},{"interface":"front","average":{"1min":1.877,"5min":2.2130000000000001,"15min":1.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9320000000000004,"5min":78.421999999999997,"15min":78.421999999999997},"last":5.2110000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.3149999999999999,"15min":1.079},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7789999999999999,"5min":8.7789999999999999,"15min":34.039000000000001},"last":5.3609999999999998},{"interface":"front","average":{"1min":2.46,"5min":1.508,"15min":1.194},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.5430000000000001,"5min":10.384,"15min":33.433999999999997},"last":6.4210000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724893,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2300444,"kb_used_data":1702484,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":91444708,"statfs":{"total":95995035648,"available":93639380992,"internally_reserved":0,"allocated":1743343616,"data_stored":1742336510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":3.25,"15min":5.1600000000000001},"min":{"1min":0.31,"5min":0.193,"15min":0.193},"max":{"1min":9.3529999999999998,"5min":111.312,"15min":151.23599999999999},"last":2.8119999999999998},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.552,"15min":5.5739999999999998},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":5.8890000000000002,"5min":111.096,"15min":151.566},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":5.242,"15min":4.0270000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":29.632999999999999,"5min":111.194,"15min":111.194},"last":1.974},{"interface":"front","average":{"1min":4.7249999999999996,"5min":5.8289999999999997,"15min":4.4180000000000001},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":29.672000000000001,"5min":112.226,"15min":112.226},"last":2.5870000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.4079999999999999,"15min":3.2759999999999998},"min":{"1min":0.28899999999999998,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":5.8579999999999997,"5min":11.324,"15min":106.508},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.8979999999999999,"15min":3.0179999999999998},"min":{"1min":0.28199999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":5.9340000000000002,"5min":8.5280000000000005,"15min":106.462},"last":1.1060000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.0449999999999999,"15min":2.5939999999999999},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.555999999999999,"5min":11.555999999999999,"15min":91.834999999999994},"last":3.3980000000000001},{"interface":"front","average":{"1min":2.3260000000000001,"5min":2.2389999999999999,"15min":2.6890000000000001},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3219999999999992,"5min":12.122999999999999,"15min":91.802000000000007},"last":3.2890000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.27,"15min":1.0029999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.4370000000000003,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.897,"5min":1.2829999999999999,"15min":1.0189999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.5890000000000004,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.36099999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.27,"15min":1.3779999999999999},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":9.4090000000000007,"5min":12.068,"15min":123.239},"last":1.931},{"interface":"front","average":{"1min":1.615,"5min":1.45,"15min":1.4670000000000001},"min":{"1min":0.313,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":9.7720000000000002,"15min":123.014},"last":0.56499999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":1.3899999999999999,"15min":1.109},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3800000000000008,"5min":12.096,"15min":12.096},"last":1.956},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.5740000000000001,"15min":1.194},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":5.7309999999999999,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.58899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1028915200,"data_stored":1028915200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2177765376,"data_stored":2177761304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1075642368,"data_stored":1075642368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1327562752,"data_stored":1327562752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1097732096,"data_stored":1097728024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1729761280,"data_stored":1729757208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2295140352,"data_stored":2295136280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:20:56.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:56.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:56.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:20:56.653 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:56.797 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:56.830 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:20:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:56.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:20:56.935 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:20:56.959 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:20:56.959 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:20:56.959 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:20:56.959 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:20:56.959 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:20:56.985 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:57.062 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:20:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:57.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:20:57.125 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:56 smithi163 ceph-mon[221067]: pgmap v977: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 7448/150240 objects misplaced (4.957%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T04:20:57.133 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:56 smithi163 ceph-mon[213841]: pgmap v977: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 7448/150240 objects misplaced (4.957%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T04:20:57.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:20:57.252 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:56 smithi181 ceph-mon[196979]: pgmap v977: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 7448/150240 objects misplaced (4.957%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T04:20:58.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:20:58.649 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:20:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:58.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:20:58.680 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:20:58.760 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:58.830 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:58 smithi163 ceph-mon[213841]: pgmap v978: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 6297/140886 objects misplaced (4.470%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:20:58.830 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:58 smithi163 ceph-mon[221067]: pgmap v978: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 6297/140886 objects misplaced (4.470%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:20:58.840 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:58 smithi181 ceph-mon[196979]: pgmap v978: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 786 op/s; 6297/140886 objects misplaced (4.470%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:20:59.049 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:59.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:59.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:20:59.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:20:59.226 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:59.366 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:20:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:59.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:20:59.476 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:59.479 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:20:59.543 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:20:59.579 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:20:59.817 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:20:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:20:59.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:20:59.870 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:20:59.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:20:59.973 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:00.036 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:00.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:00.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2656555991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:00.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:20:59 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:00.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2656555991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:00.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:20:59 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:00.143 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:00.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2656555991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:00.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:20:59 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:00.193 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:00.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:00.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:00.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:00.423 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2656555991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:21:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: osdmap e362: 8 total, 8 up, 7 in 2023-10-09T04:21:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: pgmap v980: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 6297/138756 objects misplaced (4.538%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:21:01.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[221067]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:21:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:21:00] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2656555991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: osdmap e362: 8 total, 8 up, 7 in 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: pgmap v980: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 6297/138756 objects misplaced (4.538%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:21:01.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:00 smithi163 ceph-mon[213841]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2656555991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: osdmap e362: 8 total, 8 up, 7 in 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: pgmap v980: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 6297/138756 objects misplaced (4.538%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:00 smithi181 ceph-mon[196979]: failed to encode map e362 with expected crc 2023-10-09T04:21:01.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:01.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:01.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:01.639 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:01.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:01.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:01.816 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:01.865 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:02.279 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:02.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:02.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:02.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:02.486 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:02.682 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:02.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:02.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:02.837 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:02.882 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:02 smithi163 ceph-mon[213841]: pgmap v981: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 5188/129438 objects misplaced (4.008%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:02.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:02.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:03.015 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:03.050 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:03.129 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:03.140 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:02 smithi181 ceph-mon[196979]: pgmap v981: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 5188/129438 objects misplaced (4.008%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:03.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:03.242 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:03.319 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:02 smithi163 ceph-mon[221067]: pgmap v981: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 5188/129438 objects misplaced (4.008%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:03.323 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:03.720 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:21:03.720 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131742 8.b 8:d953a743:::benchmark_data_smithi163_404719_object41158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 279.88836393999998, 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2377822549999999, 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.721 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131742 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516190+0000", 2023-10-09T04:21:03.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2799999999999999e-06 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516199+0000", 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2289999999999992e-06 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516205+0000", 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5300000000000004e-06 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516277+0000", 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2401e-05 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516293+0000", 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5812e-05 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.647614+0000", 2023-10-09T04:21:03.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1313209529999999 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.647795+0000", 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018145100000000001 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.647961+0000", 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165184 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659048+0000", 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011087575000000001 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659058+0000", 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0154e-05 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.666851+0000", 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0077926530000000001 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.750590+0000", 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.083738669000000002 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.750667+0000", 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7825999999999997e-05 2023-10-09T04:21:03.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.753969+0000", 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033019490000000002 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131768 8.b 8:d94053f1:::benchmark_data_smithi163_404719_object41184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:21:03.729 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 279.83234492999998, 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1915944220000001, 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131768 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:21:03.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572209+0000", 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2880000000000001e-06 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572218+0000", 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3899999999999999e-06 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572225+0000", 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2330000000000003e-06 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603410+0000", 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031185429000000001 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603422+0000", 2023-10-09T04:21:03.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2248e-05 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654444+0000", 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0510221209999999 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654564+0000", 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012014 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654689+0000", 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012443999999999999 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.666895+0000", 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012206318000000001 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679418+0000", 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012522881 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679478+0000", 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0134999999999999e-05 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.760738+0000", 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.081259972999999999 2023-10-09T04:21:03.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.760812+0000", 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3843000000000004e-05 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.763801+0000", 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029885020000000001 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131772 8.b 8:ddae5715:::benchmark_data_smithi163_404719_object41188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 279.83231480299997, 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1946515559999999, 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131772 2023-10-09T04:21:03.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572239+0000", 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7470000000000001e-06 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572248+0000", 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0780000000000002e-06 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572253+0000", 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3360000000000002e-06 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603454+0000", 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031201550000000002 2023-10-09T04:21:03.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603462+0000", 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5059999999999998e-06 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659085+0000", 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.055622584 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659249+0000", 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016422999999999999 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659397+0000", 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014811800000000001 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679489+0000", 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020092334999999999 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679557+0000", 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8110999999999999e-05 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.684554+0000", 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049966020000000002 2023-10-09T04:21:03.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.763867+0000", 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.079313215000000006 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.763938+0000", 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0433999999999997e-05 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.766888+0000", 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029501950000000001 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131788 8.b 8:d2de4d69:::benchmark_data_smithi163_404719_object41204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.619087+0000", 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 279.785464386, 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1507276369999999, 2023-10-09T04:21:03.746 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131788 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619087+0000", 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619087+0000", 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619085+0000", 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619093+0000", 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3429999999999996e-06 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619112+0000", 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9116000000000001e-05 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619122+0000", 2023-10-09T04:21:03.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1449999999999997e-06 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619183+0000", 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1407000000000003e-05 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619193+0000", 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8390000000000002e-06 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.662664+0000", 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.043471458 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.662836+0000", 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017205899999999999 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:21:03.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.663011+0000", 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017457400000000001 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679498+0000", 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016486912999999999 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679604+0000", 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106127 2023-10-09T04:21:03.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.689086+0000", 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0094817489999999994 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.766965+0000", 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.077878942000000007 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.767018+0000", 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3208000000000003e-05 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.769814+0000", 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027964800000000001 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:146202 8.1 8:846b80f0:::benchmark_data_smithi163_404719_object55618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:49.150472+0000", 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 254.25407954299999, 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24566232299999999, 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 146202 2023-10-09T04:21:03.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150472+0000", 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150472+0000", 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150471+0000", 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150477+0000", 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9240000000000004e-06 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150487+0000", 2023-10-09T04:21:03.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0796e-05 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150495+0000", 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0139999999999998e-06 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150546+0000", 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1007000000000001e-05 2023-10-09T04:21:03.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150681+0000", 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013445500000000001 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.150796+0000", 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011525 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.155091+0000", 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042952049999999999 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.161451+0000", 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0063598179999999997 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.396058+0000", 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23460672399999999 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.396100+0000", 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2549999999999997e-05 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.396134+0000", 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3479999999999998e-05 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:146214 8.b 8:dc9d1a1e:::benchmark_data_smithi163_404719_object55630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:49.171463+0000", 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 254.233088119, 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22361091899999999, 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.763 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 146214 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171463+0000", 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171463+0000", 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171463+0000", 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171465+0000", 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2979999999999999e-06 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171469+0000", 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8619999999999996e-06 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171472+0000", 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0929999999999999e-06 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171524+0000", 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2080000000000003e-05 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171684+0000", 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000160089 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.171850+0000", 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016631400000000001 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.391167+0000", 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.219316869 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.391544+0000", 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00037676499999999999 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.392893+0000", 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013493209999999999 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.392925+0000", 2023-10-09T04:21:03.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1782000000000002e-05 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.392977+0000", 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2337999999999999e-05 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.395074+0000", 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020964310000000002 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.770 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:146212 8.3 e283/277 8:c8a67332:::benchmark_data_smithi163_404719_object55628:head v 283'9041, mlcod=283'9039)", 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:49.175377+0000", 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 254.22917400700001, 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.224449379, 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175377+0000", 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175377+0000", 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175375+0000", 2023-10-09T04:21:03.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175390+0000", 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4188999999999999e-05 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175406+0000", 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6782e-05 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175417+0000", 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0229e-05 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175464+0000", 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7027000000000002e-05 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.175497+0000", 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3266000000000003e-05 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.399791+0000", 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.22429390799999999 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:49.399826+0000", 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5648000000000002e-05 2023-10-09T04:21:03.775 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:170514 8.1 8:8f7d6c46:::benchmark_data_smithi163_404719_object79930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.53603705099999, 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27164164200000002, 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.776 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 170514 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:21:03.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868512+0000", 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868530+0000", 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8127000000000001e-05 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868545+0000", 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4941e-05 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868556+0000", 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1137e-05 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.871810+0000", 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032543070000000001 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.871939+0000", 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012900100000000001 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.872059+0000", 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120219 2023-10-09T04:21:03.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.878504+0000", 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064444419999999999 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.883814+0000", 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053105230000000002 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140056+0000", 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25624188199999998 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140112+0000", 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5835000000000003e-05 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140156+0000", 2023-10-09T04:21:03.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3615999999999997e-05 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:170526 8.7 8:e7199b63:::benchmark_data_smithi163_404719_object79942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.887351+0000", 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.517200148, 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25287778700000002, 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.783 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 170526 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887351+0000", 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887351+0000", 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887349+0000", 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887357+0000", 2023-10-09T04:21:03.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4419999999999998e-06 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887371+0000", 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4087000000000001e-05 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887382+0000", 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.047e-05 2023-10-09T04:21:03.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887436+0000", 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3989000000000001e-05 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887567+0000", 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013151800000000001 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.887688+0000", 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012005799999999999 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140079+0000", 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25239145899999998 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140114+0000", 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4969999999999999e-05 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140145+0000", 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1421999999999998e-05 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140187+0000", 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1986000000000001e-05 2023-10-09T04:21:03.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140229+0000", 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1357999999999999e-05 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35475.0:170514 8.1 e290/241 ondisk, result = 0)", 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.889386+0000", 2023-10-09T04:21:03.790 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.51516502199999, 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25077627499999999, 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.889386+0000", 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.889386+0000", 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.889386+0000", 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.889388+0000", 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5909999999999998e-06 2023-10-09T04:21:03.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.889394+0000", 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9000000000000003e-06 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.889399+0000", 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6260000000000003e-06 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140037+0000", 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25063849399999999 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140049+0000", 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1409e-05 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140162+0000", 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113578 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPushReply(8.b 290/276 [PushReplyOp(8:d7c08e6e:::benchmark_data_smithi163_404719_object79984:head)])", 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.890454+0000", 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.514097142, 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25253847499999998, 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.795 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.890454+0000", 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.890454+0000", 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.890453+0000", 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.890460+0000", 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8889999999999996e-06 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140016+0000", 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24955581399999999 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140032+0000", 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6413999999999999e-05 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140098+0000", 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5988999999999994e-05 2023-10-09T04:21:03.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.142992+0000", 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028941119999999999 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.35475.0:170526 8.7 e290/241 ondisk, result = 0)", 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.892565+0000", 2023-10-09T04:21:03.799 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.51198648499999, 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24755307800000001, 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892565+0000", 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892565+0000", 2023-10-09T04:21:03.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892564+0000", 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892570+0000", 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5039999999999999e-06 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140025+0000", 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.247454652 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140032+0000", 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3089999999999998e-06 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140100+0000", 2023-10-09T04:21:03.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8076000000000003e-05 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140108+0000", 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7249999999999997e-06 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140118+0000", 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5670000000000001e-06 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.803 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.35475.0:170530 8.3 e290/277 8:c4654280:::benchmark_data_smithi163_404719_object79946:head v 290'10552, mlcod=290'10550)", 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.892623+0000", 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 211.511928477, 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26320082500000003, 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892623+0000", 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892623+0000", 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892622+0000", 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.892629+0000", 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5050000000000004e-06 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140010+0000", 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24738184399999999 2023-10-09T04:21:03.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140027+0000", 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6413e-05 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140071+0000", 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4471999999999997e-05 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140100+0000", 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9108999999999999e-05 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.155794+0000", 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015693537 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.155823+0000", 2023-10-09T04:21:03.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9476e-05 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:42844 9.2 9:4b626253:::benchmark_data_smithi163_464650_object42843:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:40.874569+0000", 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 142.529982128, 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24442297800000001, 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.809 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 42844 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.874569+0000", 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.874569+0000", 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.874568+0000", 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.874587+0000", 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.931e-05 2023-10-09T04:21:03.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105826+0000", 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.231238529 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105843+0000", 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7669e-05 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105893+0000", 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9685999999999998e-05 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.106060+0000", 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016693200000000001 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.106205+0000", 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000145322 2023-10-09T04:21:03.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.108725+0000", 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025200719999999999 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.112614+0000", 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038885259999999998 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.118909+0000", 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062948739999999998 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.118955+0000", 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5831000000000002e-05 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.118992+0000", 2023-10-09T04:21:03.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7276e-05 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:42846 9.f e319/310 9:f5669cfb:::benchmark_data_smithi163_464650_object42845:head v 319'2709, mlcod=319'2707)", 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:40.877158+0000", 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 142.52739272400001, 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23187658899999999, 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.816 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.877158+0000", 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.877158+0000", 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.877151+0000", 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999924 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:40.877237+0000", 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6454999999999998e-05 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105826+0000", 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.228588703 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105841+0000", 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5636000000000001e-05 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105894+0000", 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2506000000000002e-05 2023-10-09T04:21:03.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.105926+0000", 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1894999999999999e-05 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.108980+0000", 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030542820000000002 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:41.109035+0000", 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4880999999999999e-05 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:63536 9.0 9:0aceaddd:::benchmark_data_smithi163_464650_object63535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 123.035941573, 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28875457799999998, 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.821 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 63536 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368608+0000", 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368630+0000", 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2226e-05 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368640+0000", 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8500000000000006e-06 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368647+0000", 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2179999999999996e-06 2023-10-09T04:21:03.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534087+0000", 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16544025500000001 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534242+0000", 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000155162 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534420+0000", 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017757 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.543991+0000", 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095711679999999997 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544001+0000", 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0030000000000001e-05 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.656468+0000", 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112466861 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.656522+0000", 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4855000000000002e-05 2023-10-09T04:21:03.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657364+0000", 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00084163200000000004 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:63537 9.0 9:03499599:::benchmark_data_smithi163_464650_object63536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2023-10-09T04:21:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 123.034119465, 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28755178100000001, 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 63537 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370430+0000", 2023-10-09T04:21:03.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370480+0000", 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9536000000000001e-05 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370503+0000", 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3405e-05 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370518+0000", 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4299e-05 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536564+0000", 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16604591899999999 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536746+0000", 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018275999999999999 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536962+0000", 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021562400000000001 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544049+0000", 2023-10-09T04:21:03.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070868019999999997 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544171+0000", 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012230300000000001 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657426+0000", 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.113255047 2023-10-09T04:21:03.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657481+0000", 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4827999999999998e-05 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657983+0000", 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00050239800000000004 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.835 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:108768 9.2 9:40bf3da0:::benchmark_data_smithi163_464650_object10042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 65.407918871000007, 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26903813999999998, 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 108768 2023-10-09T04:21:03.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996634+0000", 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3300000000000001e-06 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996640+0000", 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9089999999999997e-06 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996645+0000", 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7729999999999999e-06 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996907+0000", 2023-10-09T04:21:03.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026236199999999998 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.997011+0000", 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010401800000000001 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.997115+0000", 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103387 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.003248+0000", 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006133416 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.008657+0000", 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054094549999999996 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265580+0000", 2023-10-09T04:21:03.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25692238499999998 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265624+0000", 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3671999999999998e-05 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265670+0000", 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6819e-05 2023-10-09T04:21:03.842 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:108785 9.2 9:4881f18b:::benchmark_data_smithi163_464650_object10059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:58.013131+0000", 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 65.391420514999993, 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26342125799999999, 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:21:03.843 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 108785 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013131+0000", 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013131+0000", 2023-10-09T04:21:03.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013129+0000", 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013136+0000", 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7360000000000004e-06 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013150+0000", 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4099e-05 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013157+0000", 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8889999999999996e-06 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013210+0000", 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2927000000000003e-05 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013360+0000", 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015013599999999999 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.013454+0000", 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4395999999999995e-05 2023-10-09T04:21:03.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.015907+0000", 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024525520000000002 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.016502+0000", 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059492499999999995 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.848 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.276470+0000", 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.259967699 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.276516+0000", 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6008999999999999e-05 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.276552+0000", 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6173000000000001e-05 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:123803 9.2 9:44df357e:::benchmark_data_smithi163_464650_object25077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:21.672388+0000", 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 41.732162584999998, 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.223587117, 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:03.850 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 123803 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.672388+0000", 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.672388+0000", 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.672387+0000", 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:03.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.672393+0000", 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.353e-06 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.672410+0000", 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6180000000000001e-05 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.672417+0000", 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5870000000000004e-06 2023-10-09T04:21:03.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.675874+0000", 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034566000000000002 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.676003+0000", 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129493 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,7", 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.676133+0000", 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012990000000000001 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.679476+0000", 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003343169 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.688665+0000", 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091886169999999996 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.688728+0000", 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3472000000000005e-05 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.895782+0000", 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20705331599999999 2023-10-09T04:21:03.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.895840+0000", 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8202999999999998e-05 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:21.895976+0000", 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000135687 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.857 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.858 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:03.858 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:03.858 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:21:03.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:03.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:03.949 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:21:03.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:04.003 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:04.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:04.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:04.088 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:04.140 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:04.164 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:04.173 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:04.186 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:04.204 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:04.220 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:04.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:04.268 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:04.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:04.307 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:04.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:04.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:04.378 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.378 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.378 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.379 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.379 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.379 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.379 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.379 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:21:04.473 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:05.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:04 smithi163 ceph-mon[213841]: pgmap v982: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 5188/127173 objects misplaced (4.079%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1259811801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:05.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:05.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:04 smithi163 ceph-mon[221067]: pgmap v982: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 5188/127173 objects misplaced (4.079%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:05.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1259811801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:05.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:05.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:21:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:04 smithi181 ceph-mon[196979]: pgmap v982: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 5188/127173 objects misplaced (4.079%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1259811801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:05.369 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:05.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:05.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:05.513 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:05.576 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:05.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:05.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:05.684 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:05.686 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:05.778 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:05.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:05.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:05.985 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:06.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:06.074 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:06.179 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:21:06.185 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:21:06.216 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:06.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:06.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:06.369 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:06.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:06.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:06.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:06.577 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:06.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:06.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:06.778 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:06.863 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:07.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:06 smithi181 ceph-mon[196979]: pgmap v983: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 5188/126384 objects misplaced (4.105%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:07.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:06 smithi163 ceph-mon[213841]: pgmap v983: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 5188/126384 objects misplaced (4.105%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:07.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:06 smithi163 ceph-mon[221067]: pgmap v983: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 794 op/s; 5188/126384 objects misplaced (4.105%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:07.426 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:21:08.121 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:08.121 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:07 smithi163 ceph-mon[221067]: from='client.45929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:08.122 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:07 smithi163 ceph-mon[213841]: from='client.45929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:08.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:08.197 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":983,"stamp":"2023-10-09T04:21:05.868762+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2759934137,"num_objects":42128,"num_object_clones":0,"num_object_copies":126384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5188,"num_objects_unfound":0,"num_objects_dirty":42128,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":155437,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19825,"num_bytes_recovered":1295361761,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96469,"ondisk_log_size":96469,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12623152,"kb_used_data":8661880,"kb_used_omap":126,"kb_used_meta":3960897,"kb_avail":643592912,"statfs":{"total":671965249536,"available":659039141888,"internally_reserved":0,"allocated":8869765120,"data_stored":8860520486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4055959163},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,6,3,4,5,5,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-521142272,"num_objects":-7952,"num_object_clones":0,"num_object_copies":-23856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2260,"num_objects_unfound":0,"num_objects_dirty":-7952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7952,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1173,"num_bytes_recovered":76873728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004890"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":486,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.345925+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:20:50.345925+0000","last_peered":"2023-10-09T04:20:50.345925+0000","last_clean":"2023-10-09T04:20:50.345925+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:20:50.345925+0000","last_undegraded":"2023-10-09T04:20:50.345925+0000","last_fullsized":"2023-10-09T04:20:50.345925+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":106,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353135+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:20:50.353135+0000","last_peered":"2023-10-09T04:20:50.353135+0000","last_clean":"2023-10-09T04:20:50.353135+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:20:50.353135+0000","last_undegraded":"2023-10-09T04:20:50.353135+0000","last_fullsized":"2023-10-09T04:20:50.353135+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":337,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.331731+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:20:50.331731+0000","last_peered":"2023-10-09T04:20:50.331731+0000","last_clean":"2023-10-09T04:20:50.331731+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:20:50.331731+0000","last_undegraded":"2023-10-09T04:20:50.331731+0000","last_fullsized":"2023-10-09T04:20:50.331731+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":479,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346426+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:20:50.346426+0000","last_peered":"2023-10-09T04:20:50.346426+0000","last_clean":"2023-10-09T04:20:50.346426+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:20:50.346426+0000","last_undegraded":"2023-10-09T04:20:50.346426+0000","last_fullsized":"2023-10-09T04:20:50.346426+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":352,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329021+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:20:50.329021+0000","last_peered":"2023-10-09T04:20:50.329021+0000","last_clean":"2023-10-09T04:20:50.329021+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:20:50.329021+0000","last_undegraded":"2023-10-09T04:20:50.329021+0000","last_fullsized":"2023-10-09T04:20:50.329021+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":340,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351588+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:20:50.351588+0000","last_peered":"2023-10-09T04:20:50.351588+0000","last_clean":"2023-10-09T04:20:50.351588+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:20:50.351588+0000","last_undegraded":"2023-10-09T04:20:50.351588+0000","last_fullsized":"2023-10-09T04:20:50.351588+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":486,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353023+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:20:50.353023+0000","last_peered":"2023-10-09T04:20:50.353023+0000","last_clean":"2023-10-09T04:20:50.353023+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:20:50.353023+0000","last_undegraded":"2023-10-09T04:20:50.353023+0000","last_fullsized":"2023-10-09T04:20:50.353023+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":337,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.355624+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:20:50.355624+0000","last_peered":"2023-10-09T04:20:50.355624+0000","last_clean":"2023-10-09T04:20:50.355624+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:20:50.355624+0000","last_undegraded":"2023-10-09T04:20:50.355624+0000","last_fullsized":"2023-10-09T04:20:50.355624+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":305,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346574+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:20:50.346574+0000","last_peered":"2023-10-09T04:20:50.346574+0000","last_clean":"2023-10-09T04:20:50.346574+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:20:50.346574+0000","last_undegraded":"2023-10-09T04:20:50.346574+0000","last_fullsized":"2023-10-09T04:20:50.346574+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":761,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342249+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:20:50.342249+0000","last_peered":"2023-10-09T04:20:50.342249+0000","last_clean":"2023-10-09T04:20:50.342249+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:20:50.342249+0000","last_undegraded":"2023-10-09T04:20:50.342249+0000","last_fullsized":"2023-10-09T04:20:50.342249+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":338,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351947+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:20:50.351947+0000","last_peered":"2023-10-09T04:20:50.351947+0000","last_clean":"2023-10-09T04:20:50.351947+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:20:50.351947+0000","last_undegraded":"2023-10-09T04:20:50.351947+0000","last_fullsized":"2023-10-09T04:20:50.351947+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.344657+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:20:50.344657+0000","last_peered":"2023-10-09T04:20:50.344657+0000","last_clean":"2023-10-09T04:20:50.344657+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:20:50.344657+0000","last_undegraded":"2023-10-09T04:20:50.344657+0000","last_fullsized":"2023-10-09T04:20:50.344657+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":488,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.350607+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:20:50.350607+0000","last_peered":"2023-10-09T04:20:50.350607+0000","last_clean":"2023-10-09T04:20:50.350607+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:20:50.350607+0000","last_undegraded":"2023-10-09T04:20:50.350607+0000","last_fullsized":"2023-10-09T04:20:50.350607+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":321,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346335+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:20:50.346335+0000","last_peered":"2023-10-09T04:20:50.346335+0000","last_clean":"2023-10-09T04:20:50.346335+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:20:50.346335+0000","last_undegraded":"2023-10-09T04:20:50.346335+0000","last_fullsized":"2023-10-09T04:20:50.346335+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.345863+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:20:50.345863+0000","last_peered":"2023-10-09T04:20:50.345863+0000","last_clean":"2023-10-09T04:20:50.345863+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:20:50.345863+0000","last_undegraded":"2023-10-09T04:20:50.345863+0000","last_fullsized":"2023-10-09T04:20:50.345863+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":496,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346439+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:20:50.346439+0000","last_peered":"2023-10-09T04:20:50.346439+0000","last_clean":"2023-10-09T04:20:50.346439+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:20:50.346439+0000","last_undegraded":"2023-10-09T04:20:50.346439+0000","last_fullsized":"2023-10-09T04:20:50.346439+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":404,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342746+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:20:50.342746+0000","last_peered":"2023-10-09T04:20:50.342746+0000","last_clean":"2023-10-09T04:20:50.342746+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:20:50.342746+0000","last_undegraded":"2023-10-09T04:20:50.342746+0000","last_fullsized":"2023-10-09T04:20:50.342746+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329007+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:20:50.329007+0000","last_peered":"2023-10-09T04:20:50.329007+0000","last_clean":"2023-10-09T04:20:50.329007+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:20:50.329007+0000","last_undegraded":"2023-10-09T04:20:50.329007+0000","last_fullsized":"2023-10-09T04:20:50.329007+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":504,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.331765+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:20:50.331765+0000","last_peered":"2023-10-09T04:20:50.331765+0000","last_clean":"2023-10-09T04:20:50.331765+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:20:50.331765+0000","last_undegraded":"2023-10-09T04:20:50.331765+0000","last_fullsized":"2023-10-09T04:20:50.331765+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":136,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353045+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:20:50.353045+0000","last_peered":"2023-10-09T04:20:50.353045+0000","last_clean":"2023-10-09T04:20:50.353045+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:20:50.353045+0000","last_undegraded":"2023-10-09T04:20:50.353045+0000","last_fullsized":"2023-10-09T04:20:50.353045+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353359+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:20:50.353359+0000","last_peered":"2023-10-09T04:20:50.353359+0000","last_clean":"2023-10-09T04:20:50.353359+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:20:50.353359+0000","last_undegraded":"2023-10-09T04:20:50.353359+0000","last_fullsized":"2023-10-09T04:20:50.353359+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":512,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.354655+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:20:50.354655+0000","last_peered":"2023-10-09T04:20:50.354655+0000","last_clean":"2023-10-09T04:20:50.354655+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:20:50.354655+0000","last_undegraded":"2023-10-09T04:20:50.354655+0000","last_fullsized":"2023-10-09T04:20:50.354655+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":355,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.341878+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:20:50.341878+0000","last_peered":"2023-10-09T04:20:50.341878+0000","last_clean":"2023-10-09T04:20:50.341878+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:20:50.341878+0000","last_undegraded":"2023-10-09T04:20:50.341878+0000","last_fullsized":"2023-10-09T04:20:50.341878+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":40,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329228+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:20:50.329228+0000","last_peered":"2023-10-09T04:20:50.329228+0000","last_clean":"2023-10-09T04:20:50.329228+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:20:50.329228+0000","last_undegraded":"2023-10-09T04:20:50.329228+0000","last_fullsized":"2023-10-09T04:20:50.329228+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":506,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.345918+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:20:50.345918+0000","last_peered":"2023-10-09T04:20:50.345918+0000","last_clean":"2023-10-09T04:20:50.345918+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:20:50.345918+0000","last_undegraded":"2023-10-09T04:20:50.345918+0000","last_fullsized":"2023-10-09T04:20:50.345918+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":339,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.352221+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:20:50.352221+0000","last_peered":"2023-10-09T04:20:50.352221+0000","last_clean":"2023-10-09T04:20:50.352221+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:20:50.352221+0000","last_undegraded":"2023-10-09T04:20:50.352221+0000","last_fullsized":"2023-10-09T04:20:50.352221+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":340,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.348473+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:20:50.348473+0000","last_peered":"2023-10-09T04:20:50.348473+0000","last_clean":"2023-10-09T04:20:50.348473+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:20:50.348473+0000","last_undegraded":"2023-10-09T04:20:50.348473+0000","last_fullsized":"2023-10-09T04:20:50.348473+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":504,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329547+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:20:50.329547+0000","last_peered":"2023-10-09T04:20:50.329547+0000","last_clean":"2023-10-09T04:20:50.329547+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:20:50.329547+0000","last_undegraded":"2023-10-09T04:20:50.329547+0000","last_fullsized":"2023-10-09T04:20:50.329547+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353009+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:20:50.353009+0000","last_peered":"2023-10-09T04:20:50.353009+0000","last_clean":"2023-10-09T04:20:50.353009+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:20:50.353009+0000","last_undegraded":"2023-10-09T04:20:50.353009+0000","last_fullsized":"2023-10-09T04:20:50.353009+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":338,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351655+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:20:50.351655+0000","last_peered":"2023-10-09T04:20:50.351655+0000","last_clean":"2023-10-09T04:20:50.351655+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:20:50.351655+0000","last_undegraded":"2023-10-09T04:20:50.351655+0000","last_fullsized":"2023-10-09T04:20:50.351655+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":506,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346387+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:20:50.346387+0000","last_peered":"2023-10-09T04:20:50.346387+0000","last_clean":"2023-10-09T04:20:50.346387+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:20:50.346387+0000","last_undegraded":"2023-10-09T04:20:50.346387+0000","last_fullsized":"2023-10-09T04:20:50.346387+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":352,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:56.773130+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:20:56.773130+0000","last_peered":"2023-10-09T04:20:56.773130+0000","last_clean":"2023-10-09T04:20:56.773130+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:20:56.773130+0000","last_undegraded":"2023-10-09T04:20:56.773130+0000","last_fullsized":"2023-10-09T04:20:56.773130+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346697+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:20:50.346697+0000","last_peered":"2023-10-09T04:20:50.346697+0000","last_clean":"2023-10-09T04:20:50.346697+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:20:50.346697+0000","last_undegraded":"2023-10-09T04:20:50.346697+0000","last_fullsized":"2023-10-09T04:20:50.346697+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"362'9558","reported_seq":9624,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:01.649970+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:21:01.649970+0000","last_peered":"2023-10-09T04:21:01.649970+0000","last_clean":"2023-10-09T04:21:01.649970+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:21:01.649970+0000","last_undegraded":"2023-10-09T04:21:01.649970+0000","last_fullsized":"2023-10-09T04:21:01.649970+0000","mapping_epoch":306,"log_start":"324'3300","ondisk_log_start":"324'3300","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171835392,"num_objects":2622,"num_object_clones":0,"num_object_copies":7866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9558,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":764,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342402+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:20:50.342402+0000","last_peered":"2023-10-09T04:20:50.342402+0000","last_clean":"2023-10-09T04:20:50.342402+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:20:50.342402+0000","last_undegraded":"2023-10-09T04:20:50.342402+0000","last_fullsized":"2023-10-09T04:20:50.342402+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":350,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342393+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:20:50.342393+0000","last_peered":"2023-10-09T04:20:50.342393+0000","last_clean":"2023-10-09T04:20:50.342393+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:20:50.342393+0000","last_undegraded":"2023-10-09T04:20:50.342393+0000","last_fullsized":"2023-10-09T04:20:50.342393+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":313,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346080+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:20:50.346080+0000","last_peered":"2023-10-09T04:20:50.346080+0000","last_clean":"2023-10-09T04:20:50.346080+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:20:50.346080+0000","last_undegraded":"2023-10-09T04:20:50.346080+0000","last_fullsized":"2023-10-09T04:20:50.346080+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"362'9798","reported_seq":9873,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:03.269085+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:21:03.269085+0000","last_peered":"2023-10-09T04:21:03.269085+0000","last_clean":"2023-10-09T04:21:03.269085+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:21:03.269085+0000","last_undegraded":"2023-10-09T04:21:03.269085+0000","last_fullsized":"2023-10-09T04:21:03.269085+0000","mapping_epoch":310,"log_start":"327'4473","ondisk_log_start":"327'4473","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5325,"log_dups_size":0,"ondisk_log_size":5325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170262528,"num_objects":2598,"num_object_clones":0,"num_object_copies":7794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9798,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":529,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.345781+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:20:50.345781+0000","last_peered":"2023-10-09T04:20:50.345781+0000","last_clean":"2023-10-09T04:20:50.345781+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:20:50.345781+0000","last_undegraded":"2023-10-09T04:20:50.345781+0000","last_fullsized":"2023-10-09T04:20:50.345781+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":354,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.344419+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:20:50.344419+0000","last_peered":"2023-10-09T04:20:50.344419+0000","last_clean":"2023-10-09T04:20:50.344419+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:20:50.344419+0000","last_undegraded":"2023-10-09T04:20:50.344419+0000","last_fullsized":"2023-10-09T04:20:50.344419+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":136,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353094+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:20:50.353094+0000","last_peered":"2023-10-09T04:20:50.353094+0000","last_clean":"2023-10-09T04:20:50.353094+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:20:50.353094+0000","last_undegraded":"2023-10-09T04:20:50.353094+0000","last_fullsized":"2023-10-09T04:20:50.353094+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"362'9812","reported_seq":13920,"reported_epoch":362,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:21:00.874645+0000","last_change":"2023-10-09T04:20:46.011306+0000","last_active":"2023-10-09T04:21:00.874645+0000","last_peered":"2023-10-09T04:21:00.874645+0000","last_clean":"2023-10-09T04:20:00.944119+0000","last_became_active":"2023-10-09T04:20:03.092473+0000","last_became_peered":"2023-10-09T04:20:03.092473+0000","last_unstale":"2023-10-09T04:21:00.874645+0000","last_undegraded":"2023-10-09T04:21:00.874645+0000","last_fullsized":"2023-10-09T04:21:00.874645+0000","mapping_epoch":349,"log_start":"325'3918","ondisk_log_start":"325'3918","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:49.431905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181141504,"num_objects":2764,"num_object_clones":0,"num_object_copies":8292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":979,"num_objects_unfound":0,"num_objects_dirty":2764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9812,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2016,"num_bytes_recovered":132120576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":2764}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":496,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346358+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:20:50.346358+0000","last_peered":"2023-10-09T04:20:50.346358+0000","last_clean":"2023-10-09T04:20:50.346358+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:20:50.346358+0000","last_undegraded":"2023-10-09T04:20:50.346358+0000","last_fullsized":"2023-10-09T04:20:50.346358+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":319,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342595+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:20:50.342595+0000","last_peered":"2023-10-09T04:20:50.342595+0000","last_clean":"2023-10-09T04:20:50.342595+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:20:50.342595+0000","last_undegraded":"2023-10-09T04:20:50.342595+0000","last_fullsized":"2023-10-09T04:20:50.342595+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":294,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329422+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:20:50.329422+0000","last_peered":"2023-10-09T04:20:50.329422+0000","last_clean":"2023-10-09T04:20:50.329422+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:20:50.329422+0000","last_undegraded":"2023-10-09T04:20:50.329422+0000","last_fullsized":"2023-10-09T04:20:50.329422+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"362'9672","reported_seq":9736,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.860248+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:21:00.860248+0000","last_peered":"2023-10-09T04:21:00.860248+0000","last_clean":"2023-10-09T04:21:00.860248+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:21:00.860248+0000","last_undegraded":"2023-10-09T04:21:00.860248+0000","last_fullsized":"2023-10-09T04:21:00.860248+0000","mapping_epoch":306,"log_start":"325'3787","ondisk_log_start":"325'3787","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173801472,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9672,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":506,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.354075+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:20:50.354075+0000","last_peered":"2023-10-09T04:20:50.354075+0000","last_clean":"2023-10-09T04:20:50.354075+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:20:50.354075+0000","last_undegraded":"2023-10-09T04:20:50.354075+0000","last_fullsized":"2023-10-09T04:20:50.354075+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":351,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346242+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:20:50.346242+0000","last_peered":"2023-10-09T04:20:50.346242+0000","last_clean":"2023-10-09T04:20:50.346242+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:20:50.346242+0000","last_undegraded":"2023-10-09T04:20:50.346242+0000","last_fullsized":"2023-10-09T04:20:50.346242+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":339,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351319+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:20:50.351319+0000","last_peered":"2023-10-09T04:20:50.351319+0000","last_clean":"2023-10-09T04:20:50.351319+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:20:50.351319+0000","last_undegraded":"2023-10-09T04:20:50.351319+0000","last_fullsized":"2023-10-09T04:20:50.351319+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"362'9293","reported_seq":12730,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.917589+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:21:00.917589+0000","last_peered":"2023-10-09T04:21:00.917589+0000","last_clean":"2023-10-09T04:21:00.917589+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:21:00.917589+0000","last_undegraded":"2023-10-09T04:21:00.917589+0000","last_fullsized":"2023-10-09T04:21:00.917589+0000","mapping_epoch":320,"log_start":"325'4000","ondisk_log_start":"325'4000","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5293,"log_dups_size":0,"ondisk_log_size":5293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9293,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":785,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.341918+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:20:50.341918+0000","last_peered":"2023-10-09T04:20:50.341918+0000","last_clean":"2023-10-09T04:20:50.341918+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:20:50.341918+0000","last_undegraded":"2023-10-09T04:20:50.341918+0000","last_fullsized":"2023-10-09T04:20:50.341918+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":322,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351411+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:20:50.351411+0000","last_peered":"2023-10-09T04:20:50.351411+0000","last_clean":"2023-10-09T04:20:50.351411+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:20:50.351411+0000","last_undegraded":"2023-10-09T04:20:50.351411+0000","last_fullsized":"2023-10-09T04:20:50.351411+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353185+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:20:50.353185+0000","last_peered":"2023-10-09T04:20:50.353185+0000","last_clean":"2023-10-09T04:20:50.353185+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:20:50.353185+0000","last_undegraded":"2023-10-09T04:20:50.353185+0000","last_fullsized":"2023-10-09T04:20:50.353185+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"362'9695","reported_seq":13272,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:03.261745+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:21:03.261745+0000","last_peered":"2023-10-09T04:21:03.261745+0000","last_clean":"2023-10-09T04:21:03.261745+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:21:03.261745+0000","last_undegraded":"2023-10-09T04:21:03.261745+0000","last_fullsized":"2023-10-09T04:21:03.261745+0000","mapping_epoch":329,"log_start":"327'4360","ondisk_log_start":"327'4360","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9695,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":531,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.345825+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:20:50.345825+0000","last_peered":"2023-10-09T04:20:50.345825+0000","last_clean":"2023-10-09T04:20:50.345825+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:20:50.345825+0000","last_undegraded":"2023-10-09T04:20:50.345825+0000","last_fullsized":"2023-10-09T04:20:50.345825+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353043+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:20:50.353043+0000","last_peered":"2023-10-09T04:20:50.353043+0000","last_clean":"2023-10-09T04:20:50.353043+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:20:50.353043+0000","last_undegraded":"2023-10-09T04:20:50.353043+0000","last_fullsized":"2023-10-09T04:20:50.353043+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":106,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353752+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:20:50.353752+0000","last_peered":"2023-10-09T04:20:50.353752+0000","last_clean":"2023-10-09T04:20:50.353752+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:20:50.353752+0000","last_undegraded":"2023-10-09T04:20:50.353752+0000","last_fullsized":"2023-10-09T04:20:50.353752+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"362'9603","reported_seq":9677,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.910375+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:21:00.910375+0000","last_peered":"2023-10-09T04:21:00.910375+0000","last_clean":"2023-10-09T04:21:00.910375+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:21:00.910375+0000","last_undegraded":"2023-10-09T04:21:00.910375+0000","last_fullsized":"2023-10-09T04:21:00.910375+0000","mapping_epoch":310,"log_start":"325'4300","ondisk_log_start":"325'4300","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5303,"log_dups_size":0,"ondisk_log_size":5303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166658048,"num_objects":2543,"num_object_clones":0,"num_object_copies":7629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9603,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":487,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329453+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:20:50.329453+0000","last_peered":"2023-10-09T04:20:50.329453+0000","last_clean":"2023-10-09T04:20:50.329453+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:20:50.329453+0000","last_undegraded":"2023-10-09T04:20:50.329453+0000","last_fullsized":"2023-10-09T04:20:50.329453+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":106,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.352925+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:20:50.352925+0000","last_peered":"2023-10-09T04:20:50.352925+0000","last_clean":"2023-10-09T04:20:50.352925+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:20:50.352925+0000","last_undegraded":"2023-10-09T04:20:50.352925+0000","last_fullsized":"2023-10-09T04:20:50.352925+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":156,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.354265+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:20:50.354265+0000","last_peered":"2023-10-09T04:20:50.354265+0000","last_clean":"2023-10-09T04:20:50.354265+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:20:50.354265+0000","last_undegraded":"2023-10-09T04:20:50.354265+0000","last_fullsized":"2023-10-09T04:20:50.354265+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"362'9851","reported_seq":18963,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:03.483008+0000","last_change":"2023-10-09T04:20:48.014336+0000","last_active":"2023-10-09T04:21:03.483008+0000","last_peered":"2023-10-09T04:21:03.483008+0000","last_clean":"2023-10-09T04:21:03.483008+0000","last_became_active":"2023-10-09T04:20:47.989792+0000","last_became_peered":"2023-10-09T04:20:47.989792+0000","last_unstale":"2023-10-09T04:21:03.483008+0000","last_undegraded":"2023-10-09T04:21:03.483008+0000","last_fullsized":"2023-10-09T04:21:03.483008+0000","mapping_epoch":359,"log_start":"328'4581","ondisk_log_start":"328'4581","created":306,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5270,"log_dups_size":0,"ondisk_log_size":5270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:26.170681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168493056,"num_objects":2571,"num_object_clones":0,"num_object_copies":7713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9851,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4512,"num_bytes_recovered":295698432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":506,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346102+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:20:50.346102+0000","last_peered":"2023-10-09T04:20:50.346102+0000","last_clean":"2023-10-09T04:20:50.346102+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:20:50.346102+0000","last_undegraded":"2023-10-09T04:20:50.346102+0000","last_fullsized":"2023-10-09T04:20:50.346102+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.344550+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:20:50.344550+0000","last_peered":"2023-10-09T04:20:50.344550+0000","last_clean":"2023-10-09T04:20:50.344550+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:20:50.344550+0000","last_undegraded":"2023-10-09T04:20:50.344550+0000","last_fullsized":"2023-10-09T04:20:50.344550+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":29,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351086+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:20:50.351086+0000","last_peered":"2023-10-09T04:20:50.351086+0000","last_clean":"2023-10-09T04:20:50.351086+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:20:50.351086+0000","last_undegraded":"2023-10-09T04:20:50.351086+0000","last_fullsized":"2023-10-09T04:20:50.351086+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"362'9732","reported_seq":9806,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.809027+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:21:00.809027+0000","last_peered":"2023-10-09T04:21:00.809027+0000","last_clean":"2023-10-09T04:21:00.809027+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:21:00.809027+0000","last_undegraded":"2023-10-09T04:21:00.809027+0000","last_fullsized":"2023-10-09T04:21:00.809027+0000","mapping_epoch":310,"log_start":"325'3834","ondisk_log_start":"325'3834","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505408,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9732,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":526,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346706+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:20:50.346706+0000","last_peered":"2023-10-09T04:20:50.346706+0000","last_clean":"2023-10-09T04:20:50.346706+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:20:50.346706+0000","last_undegraded":"2023-10-09T04:20:50.346706+0000","last_fullsized":"2023-10-09T04:20:50.346706+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":341,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329675+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:20:50.329675+0000","last_peered":"2023-10-09T04:20:50.329675+0000","last_clean":"2023-10-09T04:20:50.329675+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:20:50.329675+0000","last_undegraded":"2023-10-09T04:20:50.329675+0000","last_fullsized":"2023-10-09T04:20:50.329675+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":358,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.348360+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:20:50.348360+0000","last_peered":"2023-10-09T04:20:50.348360+0000","last_clean":"2023-10-09T04:20:50.348360+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:20:50.348360+0000","last_undegraded":"2023-10-09T04:20:50.348360+0000","last_fullsized":"2023-10-09T04:20:50.348360+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":29,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.350520+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:20:50.350520+0000","last_peered":"2023-10-09T04:20:50.350520+0000","last_clean":"2023-10-09T04:20:50.350520+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:20:50.350520+0000","last_undegraded":"2023-10-09T04:20:50.350520+0000","last_fullsized":"2023-10-09T04:20:50.350520+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"362'9695","reported_seq":9761,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:01.834955+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:21:01.834955+0000","last_peered":"2023-10-09T04:21:01.834955+0000","last_clean":"2023-10-09T04:21:01.834955+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:21:01.834955+0000","last_undegraded":"2023-10-09T04:21:01.834955+0000","last_fullsized":"2023-10-09T04:21:01.834955+0000","mapping_epoch":306,"log_start":"324'3400","ondisk_log_start":"324'3400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6295,"log_dups_size":0,"ondisk_log_size":6295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9695,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":479,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346049+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:20:50.346049+0000","last_peered":"2023-10-09T04:20:50.346049+0000","last_clean":"2023-10-09T04:20:50.346049+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:20:50.346049+0000","last_undegraded":"2023-10-09T04:20:50.346049+0000","last_fullsized":"2023-10-09T04:20:50.346049+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":335,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346179+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:20:50.346179+0000","last_peered":"2023-10-09T04:20:50.346179+0000","last_clean":"2023-10-09T04:20:50.346179+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:20:50.346179+0000","last_undegraded":"2023-10-09T04:20:50.346179+0000","last_fullsized":"2023-10-09T04:20:50.346179+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353319+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:20:50.353319+0000","last_peered":"2023-10-09T04:20:50.353319+0000","last_clean":"2023-10-09T04:20:50.353319+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:20:50.353319+0000","last_undegraded":"2023-10-09T04:20:50.353319+0000","last_fullsized":"2023-10-09T04:20:50.353319+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":369,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329607+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:20:50.329607+0000","last_peered":"2023-10-09T04:20:50.329607+0000","last_clean":"2023-10-09T04:20:50.329607+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:20:50.329607+0000","last_undegraded":"2023-10-09T04:20:50.329607+0000","last_fullsized":"2023-10-09T04:20:50.329607+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"362'9766","reported_seq":9830,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.852831+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:21:00.852831+0000","last_peered":"2023-10-09T04:21:00.852831+0000","last_clean":"2023-10-09T04:21:00.852831+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:21:00.852831+0000","last_undegraded":"2023-10-09T04:21:00.852831+0000","last_fullsized":"2023-10-09T04:21:00.852831+0000","mapping_epoch":306,"log_start":"325'3868","ondisk_log_start":"325'3868","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177864704,"num_objects":2714,"num_object_clones":0,"num_object_copies":8142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9766,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":495,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.352294+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:20:50.352294+0000","last_peered":"2023-10-09T04:20:50.352294+0000","last_clean":"2023-10-09T04:20:50.352294+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:20:50.352294+0000","last_undegraded":"2023-10-09T04:20:50.352294+0000","last_fullsized":"2023-10-09T04:20:50.352294+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":198,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:20:59.856243+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:20:59.856243+0000","last_peered":"2023-10-09T04:20:59.856243+0000","last_clean":"2023-10-09T04:20:59.856243+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:20:59.856243+0000","last_undegraded":"2023-10-09T04:20:59.856243+0000","last_fullsized":"2023-10-09T04:20:59.856243+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.338482+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:20:50.338482+0000","last_peered":"2023-10-09T04:20:50.338482+0000","last_clean":"2023-10-09T04:20:50.338482+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:20:50.338482+0000","last_undegraded":"2023-10-09T04:20:50.338482+0000","last_fullsized":"2023-10-09T04:20:50.338482+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":40,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.331464+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:20:50.331464+0000","last_peered":"2023-10-09T04:20:50.331464+0000","last_clean":"2023-10-09T04:20:50.331464+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:20:50.331464+0000","last_undegraded":"2023-10-09T04:20:50.331464+0000","last_fullsized":"2023-10-09T04:20:50.331464+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"362'9728","reported_seq":14571,"reported_epoch":362,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:21:01.660534+0000","last_change":"2023-10-09T04:20:05.060638+0000","last_active":"2023-10-09T04:21:01.660534+0000","last_peered":"2023-10-09T04:21:01.660534+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:20:02.106154+0000","last_became_peered":"2023-10-09T04:20:02.106154+0000","last_unstale":"2023-10-09T04:21:01.660534+0000","last_undegraded":"2023-10-09T04:21:01.660534+0000","last_fullsized":"2023-10-09T04:21:01.660534+0000","mapping_epoch":348,"log_start":"324'3400","ondisk_log_start":"324'3400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:45.805086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4209,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9728,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2377,"num_bytes_recovered":155779072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2660}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346340+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:20:50.346340+0000","last_peered":"2023-10-09T04:20:50.346340+0000","last_clean":"2023-10-09T04:20:50.346340+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:20:50.346340+0000","last_undegraded":"2023-10-09T04:20:50.346340+0000","last_fullsized":"2023-10-09T04:20:50.346340+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":756,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.344047+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:20:50.344047+0000","last_peered":"2023-10-09T04:20:50.344047+0000","last_clean":"2023-10-09T04:20:50.344047+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:20:50.344047+0000","last_undegraded":"2023-10-09T04:20:50.344047+0000","last_fullsized":"2023-10-09T04:20:50.344047+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":288,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346255+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:20:50.346255+0000","last_peered":"2023-10-09T04:20:50.346255+0000","last_clean":"2023-10-09T04:20:50.346255+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:20:50.346255+0000","last_undegraded":"2023-10-09T04:20:50.346255+0000","last_fullsized":"2023-10-09T04:20:50.346255+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353784+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:20:50.353784+0000","last_peered":"2023-10-09T04:20:50.353784+0000","last_clean":"2023-10-09T04:20:50.353784+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:20:50.353784+0000","last_undegraded":"2023-10-09T04:20:50.353784+0000","last_fullsized":"2023-10-09T04:20:50.353784+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"362'9662","reported_seq":13307,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:01.674984+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:21:01.674984+0000","last_peered":"2023-10-09T04:21:01.674984+0000","last_clean":"2023-10-09T04:21:01.674984+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:21:01.674984+0000","last_undegraded":"2023-10-09T04:21:01.674984+0000","last_fullsized":"2023-10-09T04:21:01.674984+0000","mapping_epoch":323,"log_start":"324'3400","ondisk_log_start":"324'3400","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6262,"log_dups_size":0,"ondisk_log_size":6262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175112192,"num_objects":2672,"num_object_clones":0,"num_object_copies":8016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9662,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":505,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329656+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:20:50.329656+0000","last_peered":"2023-10-09T04:20:50.329656+0000","last_clean":"2023-10-09T04:20:50.329656+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:20:50.329656+0000","last_undegraded":"2023-10-09T04:20:50.329656+0000","last_fullsized":"2023-10-09T04:20:50.329656+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":342,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.341747+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:20:50.341747+0000","last_peered":"2023-10-09T04:20:50.341747+0000","last_clean":"2023-10-09T04:20:50.341747+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:20:50.341747+0000","last_undegraded":"2023-10-09T04:20:50.341747+0000","last_fullsized":"2023-10-09T04:20:50.341747+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":358,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.354931+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:20:50.354931+0000","last_peered":"2023-10-09T04:20:50.354931+0000","last_clean":"2023-10-09T04:20:50.354931+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:20:50.354931+0000","last_undegraded":"2023-10-09T04:20:50.354931+0000","last_fullsized":"2023-10-09T04:20:50.354931+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":354,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351436+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:20:50.351436+0000","last_peered":"2023-10-09T04:20:50.351436+0000","last_clean":"2023-10-09T04:20:50.351436+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:20:50.351436+0000","last_undegraded":"2023-10-09T04:20:50.351436+0000","last_fullsized":"2023-10-09T04:20:50.351436+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"362'9684","reported_seq":9748,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.895809+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:21:00.895809+0000","last_peered":"2023-10-09T04:21:00.895809+0000","last_clean":"2023-10-09T04:21:00.895809+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:21:00.895809+0000","last_undegraded":"2023-10-09T04:21:00.895809+0000","last_fullsized":"2023-10-09T04:21:00.895809+0000","mapping_epoch":306,"log_start":"327'4400","ondisk_log_start":"327'4400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5284,"log_dups_size":0,"ondisk_log_size":5284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182845440,"num_objects":2790,"num_object_clones":0,"num_object_copies":8370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9684,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":515,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353941+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:20:50.353941+0000","last_peered":"2023-10-09T04:20:50.353941+0000","last_clean":"2023-10-09T04:20:50.353941+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:20:50.353941+0000","last_undegraded":"2023-10-09T04:20:50.353941+0000","last_fullsized":"2023-10-09T04:20:50.353941+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":345,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342667+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:20:50.342667+0000","last_peered":"2023-10-09T04:20:50.342667+0000","last_clean":"2023-10-09T04:20:50.342667+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:20:50.342667+0000","last_undegraded":"2023-10-09T04:20:50.342667+0000","last_fullsized":"2023-10-09T04:20:50.342667+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":335,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.329746+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:20:50.329746+0000","last_peered":"2023-10-09T04:20:50.329746+0000","last_clean":"2023-10-09T04:20:50.329746+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:20:50.329746+0000","last_undegraded":"2023-10-09T04:20:50.329746+0000","last_fullsized":"2023-10-09T04:20:50.329746+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":61,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346262+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:20:50.346262+0000","last_peered":"2023-10-09T04:20:50.346262+0000","last_clean":"2023-10-09T04:20:50.346262+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:20:50.346262+0000","last_undegraded":"2023-10-09T04:20:50.346262+0000","last_fullsized":"2023-10-09T04:20:50.346262+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"362'9700","reported_seq":21107,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:00.907555+0000","last_change":"2023-10-09T04:20:12.010618+0000","last_active":"2023-10-09T04:21:00.907555+0000","last_peered":"2023-10-09T04:21:00.907555+0000","last_clean":"2023-10-09T04:21:00.907555+0000","last_became_active":"2023-10-09T04:20:12.002110+0000","last_became_peered":"2023-10-09T04:20:12.002110+0000","last_unstale":"2023-10-09T04:21:00.907555+0000","last_undegraded":"2023-10-09T04:21:00.907555+0000","last_fullsized":"2023-10-09T04:21:00.907555+0000","mapping_epoch":354,"log_start":"327'4400","ondisk_log_start":"327'4400","created":306,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:12.102225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171114520,"num_objects":2612,"num_object_clones":0,"num_object_copies":7836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2612,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9700,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":506,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353671+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:20:50.353671+0000","last_peered":"2023-10-09T04:20:50.353671+0000","last_clean":"2023-10-09T04:20:50.353671+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:20:50.353671+0000","last_undegraded":"2023-10-09T04:20:50.353671+0000","last_fullsized":"2023-10-09T04:20:50.353671+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":294,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342552+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:20:50.342552+0000","last_peered":"2023-10-09T04:20:50.342552+0000","last_clean":"2023-10-09T04:20:50.342552+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:20:50.342552+0000","last_undegraded":"2023-10-09T04:20:50.342552+0000","last_fullsized":"2023-10-09T04:20:50.342552+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":329,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.331883+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:20:50.331883+0000","last_peered":"2023-10-09T04:20:50.331883+0000","last_clean":"2023-10-09T04:20:50.331883+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:20:50.331883+0000","last_undegraded":"2023-10-09T04:20:50.331883+0000","last_fullsized":"2023-10-09T04:20:50.331883+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":312,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.344123+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:20:50.344123+0000","last_peered":"2023-10-09T04:20:50.344123+0000","last_clean":"2023-10-09T04:20:50.344123+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:20:50.344123+0000","last_undegraded":"2023-10-09T04:20:50.344123+0000","last_fullsized":"2023-10-09T04:20:50.344123+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"362'10095","reported_seq":10169,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:21:05.445625+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:21:05.445625+0000","last_peered":"2023-10-09T04:21:05.445625+0000","last_clean":"2023-10-09T04:21:05.445625+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:21:05.445625+0000","last_undegraded":"2023-10-09T04:21:05.445625+0000","last_fullsized":"2023-10-09T04:21:05.445625+0000","mapping_epoch":310,"log_start":"327'4500","ondisk_log_start":"327'4500","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160890880,"num_objects":2455,"num_object_clones":0,"num_object_copies":7365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10095,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":500,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.345648+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:20:50.345648+0000","last_peered":"2023-10-09T04:20:50.345648+0000","last_clean":"2023-10-09T04:20:50.345648+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:20:50.345648+0000","last_undegraded":"2023-10-09T04:20:50.345648+0000","last_fullsized":"2023-10-09T04:20:50.345648+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":194,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351020+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:20:50.351020+0000","last_peered":"2023-10-09T04:20:50.351020+0000","last_clean":"2023-10-09T04:20:50.351020+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:20:50.351020+0000","last_undegraded":"2023-10-09T04:20:50.351020+0000","last_fullsized":"2023-10-09T04:20:50.351020+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":367,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351517+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:20:50.351517+0000","last_peered":"2023-10-09T04:20:50.351517+0000","last_clean":"2023-10-09T04:20:50.351517+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:20:50.351517+0000","last_undegraded":"2023-10-09T04:20:50.351517+0000","last_fullsized":"2023-10-09T04:20:50.351517+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353114+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:20:50.353114+0000","last_peered":"2023-10-09T04:20:50.353114+0000","last_clean":"2023-10-09T04:20:50.353114+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:20:50.353114+0000","last_undegraded":"2023-10-09T04:20:50.353114+0000","last_fullsized":"2023-10-09T04:20:50.353114+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":521,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.354563+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:20:50.354563+0000","last_peered":"2023-10-09T04:20:50.354563+0000","last_clean":"2023-10-09T04:20:50.354563+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:20:50.354563+0000","last_undegraded":"2023-10-09T04:20:50.354563+0000","last_fullsized":"2023-10-09T04:20:50.354563+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":336,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.351490+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:20:50.351490+0000","last_peered":"2023-10-09T04:20:50.351490+0000","last_clean":"2023-10-09T04:20:50.351490+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:20:50.351490+0000","last_undegraded":"2023-10-09T04:20:50.351490+0000","last_fullsized":"2023-10-09T04:20:50.351490+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":197,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.350871+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:20:50.350871+0000","last_peered":"2023-10-09T04:20:50.350871+0000","last_clean":"2023-10-09T04:20:50.350871+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:20:50.350871+0000","last_undegraded":"2023-10-09T04:20:50.350871+0000","last_fullsized":"2023-10-09T04:20:50.350871+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":749,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.344329+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:20:50.344329+0000","last_peered":"2023-10-09T04:20:50.344329+0000","last_clean":"2023-10-09T04:20:50.344329+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:20:50.344329+0000","last_undegraded":"2023-10-09T04:20:50.344329+0000","last_fullsized":"2023-10-09T04:20:50.344329+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":351,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.346326+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:20:50.346326+0000","last_peered":"2023-10-09T04:20:50.346326+0000","last_clean":"2023-10-09T04:20:50.346326+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:20:50.346326+0000","last_undegraded":"2023-10-09T04:20:50.346326+0000","last_fullsized":"2023-10-09T04:20:50.346326+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":126,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353398+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:20:50.353398+0000","last_peered":"2023-10-09T04:20:50.353398+0000","last_clean":"2023-10-09T04:20:50.353398+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:20:50.353398+0000","last_undegraded":"2023-10-09T04:20:50.353398+0000","last_fullsized":"2023-10-09T04:20:50.353398+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":348,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.348277+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:20:50.348277+0000","last_peered":"2023-10-09T04:20:50.348277+0000","last_clean":"2023-10-09T04:20:50.348277+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:20:50.348277+0000","last_undegraded":"2023-10-09T04:20:50.348277+0000","last_fullsized":"2023-10-09T04:20:50.348277+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":126,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353119+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:20:50.353119+0000","last_peered":"2023-10-09T04:20:50.353119+0000","last_clean":"2023-10-09T04:20:50.353119+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:20:50.353119+0000","last_undegraded":"2023-10-09T04:20:50.353119+0000","last_fullsized":"2023-10-09T04:20:50.353119+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":784,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.342091+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:20:50.342091+0000","last_peered":"2023-10-09T04:20:50.342091+0000","last_clean":"2023-10-09T04:20:50.342091+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:20:50.342091+0000","last_undegraded":"2023-10-09T04:20:50.342091+0000","last_fullsized":"2023-10-09T04:20:50.342091+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":207,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.353313+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:20:50.353313+0000","last_peered":"2023-10-09T04:20:50.353313+0000","last_clean":"2023-10-09T04:20:50.353313+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:20:50.353313+0000","last_undegraded":"2023-10-09T04:20:50.353313+0000","last_fullsized":"2023-10-09T04:20:50.353313+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":197,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.350619+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:20:50.350619+0000","last_peered":"2023-10-09T04:20:50.350619+0000","last_clean":"2023-10-09T04:20:50.350619+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:20:50.350619+0000","last_undegraded":"2023-10-09T04:20:50.350619+0000","last_fullsized":"2023-10-09T04:20:50.350619+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":504,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.331925+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:20:50.331925+0000","last_peered":"2023-10-09T04:20:50.331925+0000","last_clean":"2023-10-09T04:20:50.331925+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:20:50.331925+0000","last_undegraded":"2023-10-09T04:20:50.331925+0000","last_fullsized":"2023-10-09T04:20:50.331925+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.352026+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:20:50.352026+0000","last_peered":"2023-10-09T04:20:50.352026+0000","last_clean":"2023-10-09T04:20:50.352026+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:20:50.352026+0000","last_undegraded":"2023-10-09T04:20:50.352026+0000","last_fullsized":"2023-10-09T04:20:50.352026+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":116,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:20:50.352852+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:20:50.352852+0000","last_peered":"2023-10-09T04:20:50.352852+0000","last_clean":"2023-10-09T04:20:50.352852+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:20:50.352852+0000","last_undegraded":"2023-10-09T04:20:50.352852+0000","last_fullsized":"2023-10-09T04:20:50.352852+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2759262232,"num_objects":42104,"num_object_clones":0,"num_object_copies":126312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5188,"num_objects_unfound":0,"num_objects_dirty":42104,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155344,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19758,"num_bytes_recovered":1294794776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8773910528,"data_stored":8773894240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91423,"ondisk_log_size":91423,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201622,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2500220,"kb_used_data":1914548,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":91244932,"statfs":{"total":95995035648,"available":93434810368,"internally_reserved":0,"allocated":1960497152,"data_stored":1959470149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":9.6059999999999999,"5min":3.8919999999999999,"15min":3.5350000000000001},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":148.65899999999999,"5min":148.65899999999999,"15min":148.65899999999999},"last":2.0139999999999998},{"interface":"front","average":{"1min":10.054,"5min":3.8660000000000001,"15min":3.5179999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":148.642,"5min":148.642,"15min":148.642},"last":1.4119999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.2430000000000003,"5min":4.3739999999999997,"15min":4.21},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.21199999999999999},"max":{"1min":29.792000000000002,"5min":103.324,"15min":245.28700000000001},"last":1.2350000000000001},{"interface":"front","average":{"1min":4.9909999999999997,"5min":4.6210000000000004,"15min":4.1660000000000004},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":29.268000000000001,"5min":104.248,"15min":244.626},"last":1.121}]},{"osd":2,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.698,"15min":2.1240000000000001},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.2270000000000003,"5min":14.076000000000001,"15min":76.774000000000001},"last":5.2759999999999998},{"interface":"front","average":{"1min":1.5,"5min":2.528,"15min":2.8540000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":6.5460000000000003,"5min":15.597,"15min":99.372},"last":6.3899999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":2.6419999999999999,"15min":2.633},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":3.7149999999999999,"5min":85.575000000000003,"15min":99.134},"last":1.2949999999999999},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.1080000000000001,"15min":2.8769999999999998},"min":{"1min":0.36899999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":6.1459999999999999,"5min":85.528000000000006,"15min":99.117999999999995},"last":1.272}]},{"osd":4,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.96499999999999997,"15min":0.85799999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.1799999999999997,"5min":8.9359999999999999,"15min":16.195},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.236,"5min":1.714,"15min":1.1539999999999999},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26000000000000001},"max":{"1min":4.4500000000000002,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":1.883,"15min":1.4810000000000001},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":13.122999999999999,"5min":13.122999999999999,"15min":120.066},"last":1.6100000000000001},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.3700000000000001,"15min":1.359},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8360000000000003,"5min":9,"15min":120.09699999999999},"last":1.579}]},{"osd":6,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.667,"15min":1.1140000000000001},"min":{"1min":0.34300000000000003,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":7.3310000000000004,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.149,"5min":1.069,"15min":0.92000000000000004},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.7060000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.61899999999999999}]}]},{"osd":6,"up_from":0,"seq":201863463260,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089568,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2259944,"kb_used_data":1661856,"kb_used_omap":9,"kb_used_meta":598070,"kb_avail":91485208,"statfs":{"total":95995035648,"available":93680852992,"internally_reserved":0,"allocated":1701740544,"data_stored":1700206351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":612424681},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":3.8759999999999999,"15min":4.0629999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":19.617000000000001,"5min":98.257999999999996,"15min":98.257999999999996},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.1949999999999998,"5min":6.3319999999999999,"15min":5.2000000000000002},"min":{"1min":0.27100000000000002,"5min":0.19800000000000001,"15min":0.182},"max":{"1min":9.9960000000000004,"5min":154.654,"15min":154.654},"last":1.448}]},{"osd":2,"last update":"Mon Oct 9 04:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.1699999999999999,"15min":2.7170000000000001},"min":{"1min":0.221,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":12.791,"5min":19.564,"15min":121.54600000000001},"last":4.6449999999999996},{"interface":"front","average":{"1min":3.702,"5min":2.657,"15min":3.1400000000000001},"min":{"1min":0.28100000000000003,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":14.503,"5min":16.114999999999998,"15min":99.899000000000001},"last":1.595}]},{"osd":3,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":2.8660000000000001,"15min":3.7770000000000001},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":19.640000000000001,"5min":68.054000000000002,"15min":138.87299999999999},"last":0.625},{"interface":"front","average":{"1min":2.4609999999999999,"5min":3.8170000000000002,"15min":4.6970000000000001},"min":{"1min":0.307,"5min":0.251,"15min":0.251},"max":{"1min":12.353999999999999,"5min":68.563999999999993,"15min":138.49799999999999},"last":4.524}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":2.2290000000000001,"15min":2.3370000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":7.5250000000000004,"5min":16.27,"15min":99.989999999999995},"last":1.641},{"interface":"front","average":{"1min":1.458,"5min":1.9530000000000001,"15min":2.1829999999999998},"min":{"1min":0.317,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":5.282,"5min":13.428000000000001,"15min":100.249},"last":0.34799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":3.899,"5min":3.2280000000000002,"15min":2.6539999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":25.696999999999999,"5min":38.848999999999997,"15min":99.933999999999997},"last":1.4930000000000001},{"interface":"front","average":{"1min":2.8700000000000001,"5min":3.145,"15min":2.6909999999999998},"min":{"1min":0.34999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":19.977,"5min":39.593000000000004,"15min":100.34099999999999},"last":6.3949999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":2.2120000000000002,"15min":2.5},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":5.3579999999999997,"5min":13.377000000000001,"15min":100.309},"last":6.4269999999999996},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.9570000000000001,"15min":2.1749999999999998},"min":{"1min":0.41399999999999998,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":100.143},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.7050000000000001,"15min":2.6419999999999999},"min":{"1min":0.374,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":5.7430000000000003,"5min":17.460000000000001,"15min":100.214},"last":6.5250000000000004},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.3359999999999999,"15min":2.319},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.317},"max":{"1min":19.593,"5min":19.593,"15min":100.113},"last":0.46200000000000002}]}]},{"osd":0,"up_from":84,"seq":360777253094,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1372072,"kb_used_data":854176,"kb_used_omap":28,"kb_used_meta":517795,"kb_avail":92373080,"statfs":{"total":95995035648,"available":94590033920,"internally_reserved":0,"allocated":874676224,"data_stored":873081641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":530222928},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":8.9990000000000006,"15min":5.2709999999999999},"min":{"1min":0.20200000000000001,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":6.1609999999999996,"5min":209.215,"15min":209.215},"last":2.6320000000000001},{"interface":"front","average":{"1min":3.0499999999999998,"5min":6.8630000000000004,"15min":5.032},"min":{"1min":0.29499999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":10.547000000000001,"5min":120.934,"15min":120.934},"last":13.516999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:20:47 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":2.149,"15min":3.2570000000000001},"min":{"1min":0.216,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.131,"5min":17.466999999999999,"15min":118.54000000000001},"last":11.693},{"interface":"front","average":{"1min":2.7530000000000001,"5min":3.1480000000000001,"15min":3.6899999999999999},"min":{"1min":0.224,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":11.050000000000001,"5min":20.262,"15min":118.727},"last":20.666}]},{"osd":3,"last update":"Mon Oct 9 04:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":3.1400000000000001,"15min":5.2759999999999998},"min":{"1min":0.25700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":9.1669999999999998,"5min":99.709000000000003,"15min":156.488},"last":12.888999999999999},{"interface":"front","average":{"1min":2.3919999999999999,"5min":3.1720000000000002,"15min":5.3620000000000001},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.22500000000000001},"max":{"1min":10.577,"5min":99.560000000000002,"15min":156.50700000000001},"last":11.634}]},{"osd":4,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":3.0619999999999998,"15min":2.8159999999999998},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":8.7509999999999994,"5min":63.375999999999998,"15min":118.633},"last":11.605},{"interface":"front","average":{"1min":2.3500000000000001,"5min":3.4620000000000002,"15min":3.1829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33900000000000002,"15min":0.28199999999999997},"max":{"1min":8.6950000000000003,"5min":49.607999999999997,"15min":118.913},"last":26.734000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":3.661,"15min":4.4139999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":8.7279999999999998,"5min":62.954000000000001,"15min":182.68899999999999},"last":22.399000000000001},{"interface":"front","average":{"1min":2.0880000000000001,"5min":2.6269999999999998,"15min":3.8719999999999999},"min":{"1min":0.36899999999999999,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":5.5469999999999997,"5min":63.289000000000001,"15min":182.66300000000001},"last":2.5190000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":2.5899999999999999,"15min":3.0950000000000002},"min":{"1min":0.40200000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":5.5099999999999998,"5min":63.468000000000004,"15min":137.36099999999999},"last":2.5790000000000002},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.6219999999999999,"15min":3.133},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.317},"max":{"1min":8.8160000000000007,"5min":63.447000000000003,"15min":136.88999999999999},"last":11.66}]},{"osd":7,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":2.681,"15min":3.7869999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":5.5780000000000003,"5min":63.509,"15min":185.024},"last":2.5550000000000002},{"interface":"front","average":{"1min":2.5619999999999998,"5min":2.8090000000000002,"15min":3.4660000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.32700000000000001},"max":{"1min":8.7789999999999999,"5min":63.481999999999999,"15min":184.98599999999999},"last":11.554}]}]},{"osd":2,"up_from":94,"seq":403726926042,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1402492,"kb_used_data":826228,"kb_used_omap":81,"kb_used_meta":576110,"kb_avail":92342660,"statfs":{"total":95995035648,"available":94558883840,"internally_reserved":0,"allocated":846057472,"data_stored":844507954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":589936858},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":5.3929999999999998,"15min":4.024},"min":{"1min":0.22500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":17.122,"5min":81.385999999999996,"15min":81.385999999999996},"last":3.3450000000000002},{"interface":"front","average":{"1min":1.228,"5min":4.617,"15min":3.9540000000000002},"min":{"1min":0.22500000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":4.1070000000000002,"5min":79.453999999999994,"15min":79.453999999999994},"last":7.0629999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9340000000000002,"5min":7.2270000000000003,"15min":4.7549999999999999},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.191},"max":{"1min":72.075999999999993,"5min":158.13399999999999,"15min":158.13399999999999},"last":7.1399999999999997},{"interface":"front","average":{"1min":6.4489999999999998,"5min":7.883,"15min":5.0590000000000002},"min":{"1min":0.27600000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":69.754999999999995,"5min":161.12,"15min":161.12},"last":3.298}]},{"osd":3,"last update":"Mon Oct 9 04:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":3.2909999999999999,"15min":2.7429999999999999},"min":{"1min":0.25900000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.832000000000001,"5min":58.997,"15min":58.997},"last":13.832000000000001},{"interface":"front","average":{"1min":2.5270000000000001,"5min":3.5760000000000001,"15min":3.0329999999999999},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.853999999999999,"5min":66.135000000000005,"15min":66.135000000000005},"last":7.1719999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":2.7549999999999999,"15min":2.2650000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":12.775,"5min":58.798000000000002,"15min":58.798000000000002},"last":3.2269999999999999},{"interface":"front","average":{"1min":1.75,"5min":2.5619999999999998,"15min":2.093},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":6.5010000000000003,"5min":56.616,"15min":56.616},"last":3.3700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":3.9660000000000002,"15min":2.7269999999999999},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":82.030000000000001,"5min":82.030000000000001,"15min":82.030000000000001},"last":6.9950000000000001},{"interface":"front","average":{"1min":7.4290000000000003,"5min":4.2430000000000003,"15min":2.9119999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":82.016000000000005,"5min":82.016000000000005,"15min":82.016000000000005},"last":3.2709999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.6949999999999998,"15min":2.5579999999999998},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.0129999999999999,"5min":56.561,"15min":114.14},"last":3.4340000000000002},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.629,"15min":2.6469999999999998},"min":{"1min":0.40999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2110000000000003,"5min":56.460999999999999,"15min":114.123},"last":6.9450000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.0700000000000003,"15min":3.1989999999999998},"min":{"1min":0.42599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":101.675,"5min":101.675,"15min":117.32599999999999},"last":3.407},{"interface":"front","average":{"1min":8.2639999999999993,"5min":4.1589999999999998,"15min":3.2639999999999998},"min":{"1min":0.46100000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":101.72499999999999,"5min":101.72499999999999,"15min":117.28700000000001},"last":7.0279999999999996}]}]},{"osd":3,"up_from":99,"seq":425201762516,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1878356,"kb_used_data":1102540,"kb_used_omap":7,"kb_used_meta":775736,"kb_avail":91866796,"statfs":{"total":95995035648,"available":94071599104,"internally_reserved":0,"allocated":1129000960,"data_stored":1127484293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":794354280},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,0,0,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":3.3730000000000002,"15min":5.4059999999999997},"min":{"1min":0.214,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":6.4169999999999998,"5min":78.771000000000001,"15min":125.765},"last":2.4900000000000002},{"interface":"front","average":{"1min":2.899,"5min":4.2080000000000002,"15min":5.8280000000000003},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.119999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":6.8499999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":5.8979999999999997,"15min":5.2619999999999996},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":8.3230000000000004,"5min":103.143,"15min":186.21299999999999},"last":4.2729999999999997},{"interface":"front","average":{"1min":1.6060000000000001,"5min":5.7789999999999999,"15min":5.0309999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.193},"max":{"1min":5.9219999999999997,"5min":105.19,"15min":191.78},"last":2.5099999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":4.1660000000000004,"15min":3.899},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":4.8760000000000003,"5min":127.599,"15min":127.599},"last":2.649},{"interface":"front","average":{"1min":2.359,"5min":6.0670000000000002,"15min":5.0119999999999996},"min":{"1min":0.29799999999999999,"5min":0.221,"15min":0.214},"max":{"1min":10.143000000000001,"5min":129.98599999999999,"15min":129.98599999999999},"last":6.6719999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2410000000000001,"5min":2.274,"15min":2.7240000000000002},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.013999999999999,"5min":27.539999999999999,"15min":106.85899999999999},"last":1.4390000000000001},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.2919999999999998,"15min":2.1760000000000002},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26900000000000002},"max":{"1min":3.738,"5min":26.663,"15min":37.442},"last":4.8719999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":2.6280000000000001,"15min":2.2010000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":3.7749999999999999,"5min":17.901,"15min":17.901},"last":4.9109999999999996},{"interface":"front","average":{"1min":1.083,"5min":1.766,"15min":1.887},"min":{"1min":0.375,"5min":0.248,"15min":0.248},"max":{"1min":4.6340000000000003,"5min":18.068000000000001,"15min":18.068000000000001},"last":2.4260000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.7509999999999999,"15min":2.2610000000000001},"min":{"1min":0.378,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4589999999999996,"5min":14.567,"15min":106.036},"last":2.4580000000000002},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.48,"15min":2.6509999999999998},"min":{"1min":0.42699999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.071,"5min":14.529,"15min":106.741},"last":1.377}]},{"osd":7,"last update":"Mon Oct 9 04:20:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":2.6720000000000002,"15min":2.786},"min":{"1min":0.39000000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":28.184000000000001,"5min":28.184000000000001,"15min":106.82599999999999},"last":1.413},{"interface":"front","average":{"1min":3.1480000000000001,"5min":2.827,"15min":2.7490000000000001},"min":{"1min":0.44800000000000001,"5min":0.40000000000000002,"15min":0.35999999999999999},"max":{"1min":26.648,"5min":26.648,"15min":106.084},"last":4.9459999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986532,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1175852,"kb_used_data":866276,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92569300,"statfs":{"total":95995035648,"available":94790963200,"internally_reserved":0,"allocated":887066624,"data_stored":886055036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":6.7859999999999996,"5min":9.9049999999999994,"15min":5.9029999999999996},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":72.759,"5min":154.62100000000001,"15min":154.62100000000001},"last":5.5880000000000001},{"interface":"front","average":{"1min":6.7279999999999998,"5min":10.082000000000001,"15min":5.9210000000000003},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":72.704999999999998,"5min":154.37299999999999,"15min":154.37299999999999},"last":3.4790000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":7.8129999999999997,"5min":9.4969999999999999,"15min":5.1180000000000003},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":69.706000000000003,"5min":135.25399999999999,"15min":135.25399999999999},"last":8.1539999999999999},{"interface":"front","average":{"1min":7.3220000000000001,"5min":9.1270000000000007,"15min":5.0060000000000002},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":69.561000000000007,"5min":134.44300000000001,"15min":134.44300000000001},"last":5.6479999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.8180000000000001,"15min":2.1560000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":8.202,"5min":8.7400000000000002,"15min":123.598},"last":3.4529999999999998},{"interface":"front","average":{"1min":2.5299999999999998,"5min":1.8440000000000001,"15min":2.1160000000000001},"min":{"1min":0.33400000000000002,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":11.064,"5min":11.064,"15min":123.619},"last":5.6210000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":2.052,"15min":2.29},"min":{"1min":0.315,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.0069999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":2.863},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.165,"15min":2.4849999999999999},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.7579999999999991,"5min":8.9169999999999998,"15min":127.90000000000001},"last":3.5059999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:21:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.161,"15min":1.645},"min":{"1min":0.27200000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.4249999999999998,"5min":13.599,"15min":111.809},"last":3.4249999999999998},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.5029999999999999,"15min":1.879},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.25},"max":{"1min":8.0589999999999993,"5min":9.1530000000000005,"15min":112.002},"last":8.0589999999999993}]},{"osd":6,"last update":"Mon Oct 9 04:20:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.395,"15min":1.3680000000000001},"min":{"1min":0.35699999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":6.9470000000000001,"5min":76.307000000000002,"15min":76.307000000000002},"last":8.1829999999999998},{"interface":"front","average":{"1min":1.877,"5min":2.2130000000000001,"15min":1.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9320000000000004,"5min":78.421999999999997,"15min":78.421999999999997},"last":6.1829999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.3149999999999999,"15min":1.079},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7789999999999999,"5min":8.7789999999999999,"15min":34.039000000000001},"last":3.387},{"interface":"front","average":{"1min":2.46,"5min":1.508,"15min":1.194},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.5430000000000001,"5min":10.384,"15min":33.433999999999997},"last":2.8959999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724895,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2034216,"kb_used_data":1436256,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":91710936,"statfs":{"total":95995035648,"available":93911998464,"internally_reserved":0,"allocated":1470726144,"data_stored":1469715062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":3.25,"15min":5.1600000000000001},"min":{"1min":0.31,"5min":0.193,"15min":0.193},"max":{"1min":9.3529999999999998,"5min":111.312,"15min":151.23599999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.552,"15min":5.5739999999999998},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":5.8890000000000002,"5min":111.096,"15min":151.566},"last":0.309}]},{"osd":1,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":3.375,"5min":5.242,"15min":4.0270000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":29.632999999999999,"5min":111.194,"15min":111.194},"last":0.25700000000000001},{"interface":"front","average":{"1min":4.7249999999999996,"5min":5.8289999999999997,"15min":4.4180000000000001},"min":{"1min":0.33900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":29.672000000000001,"5min":112.226,"15min":112.226},"last":0.38400000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.4079999999999999,"15min":3.2759999999999998},"min":{"1min":0.28899999999999998,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":5.8579999999999997,"5min":11.324,"15min":106.508},"last":21.420999999999999},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.8979999999999999,"15min":3.0179999999999998},"min":{"1min":0.28199999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":5.9340000000000002,"5min":8.5280000000000005,"15min":106.462},"last":21.146000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3010000000000002,"5min":2.0449999999999999,"15min":2.5939999999999999},"min":{"1min":0.33600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.555999999999999,"5min":11.555999999999999,"15min":91.834999999999994},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.3260000000000001,"5min":2.2389999999999999,"15min":2.6890000000000001},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3219999999999992,"5min":12.122999999999999,"15min":91.802000000000007},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.27,"15min":1.0029999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.4370000000000003,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.897,"5min":1.2829999999999999,"15min":1.0189999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.5890000000000004,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.35699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.27,"15min":1.3779999999999999},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":9.4090000000000007,"5min":12.068,"15min":123.239},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.615,"5min":1.45,"15min":1.4670000000000001},"min":{"1min":0.313,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":9.7720000000000002,"15min":123.014},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":1.3899999999999999,"15min":1.109},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3800000000000008,"5min":12.096,"15min":12.096},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.5740000000000001,"15min":1.194},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":5.7309999999999999,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.45900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":867041280,"data_stored":867041280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1676742656,"data_stored":1676738584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":828375040,"data_stored":828375040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1120141312,"data_stored":1120141312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":874254336,"data_stored":874250264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1459228672,"data_stored":1459224600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1947668480,"data_stored":1947664408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:21:08.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:21:08.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:08.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:07 smithi181 ceph-mon[196979]: from='client.45929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:08.376 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:09.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:09.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:09.118 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:09.140 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:09.178 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:08 smithi181 ceph-mon[196979]: pgmap v984: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 784 op/s; 4188/117351 objects misplaced (3.569%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:21:09.182 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:09.249 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:09.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:09.272 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:09.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:08 smithi163 ceph-mon[213841]: pgmap v984: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 784 op/s; 4188/117351 objects misplaced (3.569%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:21:09.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:08 smithi163 ceph-mon[221067]: pgmap v984: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 784 op/s; 4188/117351 objects misplaced (3.569%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:21:09.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:09.407 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:09.471 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:09.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:09.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:09.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:09.635 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:09.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:09.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:09.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:09.828 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:09.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:09.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:09.952 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:21:09.952 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:21:09.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:10.007 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:10.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:10.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:10.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:10.199 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:10.569 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:21:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:21:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:21:10] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-10-09T04:21:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: osdmap e363: 8 total, 8 up, 7 in 2023-10-09T04:21:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: failed to encode map e363 with expected crc 2023-10-09T04:21:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: failed to encode map e363 with expected crc 2023-10-09T04:21:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: failed to encode map e363 with expected crc 2023-10-09T04:21:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: pgmap v986: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 4188/114951 objects misplaced (3.643%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:21:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1134041007' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:10.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: osdmap e363: 8 total, 8 up, 7 in 2023-10-09T04:21:10.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: failed to encode map e363 with expected crc 2023-10-09T04:21:10.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: failed to encode map e363 with expected crc 2023-10-09T04:21:10.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: failed to encode map e363 with expected crc 2023-10-09T04:21:10.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: pgmap v986: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 4188/114951 objects misplaced (3.643%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:21:10.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1134041007' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:10.835 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: osdmap e363: 8 total, 8 up, 7 in 2023-10-09T04:21:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: failed to encode map e363 with expected crc 2023-10-09T04:21:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: failed to encode map e363 with expected crc 2023-10-09T04:21:11.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: failed to encode map e363 with expected crc 2023-10-09T04:21:11.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: pgmap v986: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 4188/114951 objects misplaced (3.643%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:21:11.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1134041007' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:11.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:11.408 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:11.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:11.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:11.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:11.562 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:11.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:11.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:11.674 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: osdmap e364: 8 total, 8 up, 7 in 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.675 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[221067]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: osdmap e364: 8 total, 8 up, 7 in 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.678 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:11 smithi163 ceph-mon[213841]: failed to encode map e364 with expected crc 2023-10-09T04:21:11.770 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:11.860 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: failed to encode map e364 with expected crc 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: failed to encode map e364 with expected crc 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: osdmap e364: 8 total, 8 up, 7 in 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: failed to encode map e364 with expected crc 2023-10-09T04:21:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:11 smithi181 ceph-mon[196979]: failed to encode map e364 with expected crc 2023-10-09T04:21:12.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:12 smithi163 ceph-mon[221067]: pgmap v988: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 905 op/s; 2726/105447 objects misplaced (2.585%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:21:12.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:12 smithi163 ceph-mon[213841]: pgmap v988: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 905 op/s; 2726/105447 objects misplaced (2.585%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:21:13.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:12 smithi181 ceph-mon[196979]: pgmap v988: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 905 op/s; 2726/105447 objects misplaced (2.585%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:21:13.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:13.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:13.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:13.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:13.287 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:13.462 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:13.470 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:13.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:13.577 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:13.626 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:13.874 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:13.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:13.985 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:14.031 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:14.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:14.106 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:14.144 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:14.205 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:14.237 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:14.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:14.335 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.348 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:21:14.384 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:14.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:14.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:14.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:14.667 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:21:14.667 INFO:teuthology.orchestra.run.smithi163.stdout:{"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1239,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":364,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824216,"num_in_osds":7,"osd_in_since":1696825107,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":119,"num_pools":5,"num_objects":34448,"data_bytes":2256617657,"bytes_used":11566477312,"bytes_avail":660398772224,"bytes_total":671965249536,"misplaced_objects":2726,"misplaced_total":103344,"misplaced_ratio":0.026377922278990554,"recovering_objects_per_sec":183,"recovering_bytes_per_sec":12022458,"recovering_keys_per_sec":0,"num_objects_recovered":1468,"num_bytes_recovered":96206848,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":959},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi163.crqkdj","status":"up:active","gid":24569}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.163:8443/","prometheus":"http://172.21.15.163:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T04:19:35.820638+0000","services":{}},"progress_events":{"418f847e-17cc-4484-9cd8-4a5c866bb9b6":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"cbdf697a-aea6-4b77-8dba-e1904d62d8bd":{"message":"Global Recovery Event (95s)\n [===========================.] (remaining: 1s)","progress":0.98319327831268311,"add_to_ceph_s":true}}} 2023-10-09T04:21:14.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:14.752 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:14.794 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:14.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:14.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:14.986 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:14 smithi163 ceph-mon[213841]: pgmap v989: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 959 op/s; 2726/103344 objects misplaced (2.638%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:21:14.987 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2650758282' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:14.987 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1503871132' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:21:14.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:14 smithi163 ceph-mon[221067]: pgmap v989: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 959 op/s; 2726/103344 objects misplaced (2.638%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:21:14.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2650758282' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:14.987 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1503871132' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:21:14.995 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:15.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:14 smithi181 ceph-mon[196979]: pgmap v989: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 959 op/s; 2726/103344 objects misplaced (2.638%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:21:15.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2650758282' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:15.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1503871132' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:21:15.156 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:21:16.563 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:16.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:16.676 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:16.717 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:21:16.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:21:17.092 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:17.214 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:17.215 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:16 smithi181 ceph-mon[196979]: pgmap v990: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 2726/102576 objects misplaced (2.658%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:21:17.246 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:21:17.246 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:21:17.246 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:21:17.246 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:21:17.246 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:21:17.286 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:16 smithi163 ceph-mon[221067]: pgmap v990: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 2726/102576 objects misplaced (2.658%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:21:17.286 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:16 smithi163 ceph-mon[213841]: pgmap v990: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 2726/102576 objects misplaced (2.658%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:21:17.301 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:17.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:17.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:17.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:17.475 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:18.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:18.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:18.343 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:18.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:21:18.402 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:21:18.445 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:18.613 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:18.616 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:18.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:18.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:18.814 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:18.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:18 smithi163 ceph-mon[221067]: pgmap v991: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 1734/93729 objects misplaced (1.850%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:21:18.834 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:18 smithi163 ceph-mon[213841]: pgmap v991: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 1734/93729 objects misplaced (1.850%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:21:18.891 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:18.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:18.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:18.899 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:18 smithi181 ceph-mon[196979]: pgmap v991: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 1734/93729 objects misplaced (1.850%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:21:19.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:19.048 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:19.436 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:19.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:19.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:19.633 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:19.743 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:19.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:19.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:19.895 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:19.991 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:20.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:20.008 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:20.011 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:20.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:20.155 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:20.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:20.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:20.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:20.379 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:20 smithi181 ceph-mon[196979]: pgmap v992: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 776 op/s; 1734/91665 objects misplaced (1.892%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:21:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:20 smithi163 ceph-mon[221067]: pgmap v992: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 776 op/s; 1734/91665 objects misplaced (1.892%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:21:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:20.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:20 smithi163 ceph-mon[213841]: pgmap v992: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 776 op/s; 1734/91665 objects misplaced (1.892%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T04:21:20.984 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:20.990 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:21:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:21:20] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-10-09T04:21:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:20.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:21.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:21.165 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:21.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:21.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:21.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:21.486 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:21.525 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:21.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:21.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:21.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:21.635 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:21:21.717 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:21.763 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:21.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2617727613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:21.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:21 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2617727613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:21.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:21 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:21.836 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:21.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2617727613' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:21.841 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:21 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:21.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:21.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:22.000 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:23.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:22 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:23.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:22 smithi181 ceph-mon[196979]: failed to encode map e365 with expected crc 2023-10-09T04:21:23.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:22 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:21:23.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:22 smithi181 ceph-mon[196979]: osdmap e365: 8 total, 8 up, 7 in 2023-10-09T04:21:23.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:22 smithi181 ceph-mon[196979]: failed to encode map e365 with expected crc 2023-10-09T04:21:23.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:22 smithi181 ceph-mon[196979]: pgmap v994: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 650/82020 objects misplaced (0.792%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:23.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:23.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[213841]: failed to encode map e365 with expected crc 2023-10-09T04:21:23.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:21:23.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[213841]: osdmap e365: 8 total, 8 up, 7 in 2023-10-09T04:21:23.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[213841]: failed to encode map e365 with expected crc 2023-10-09T04:21:23.026 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[213841]: pgmap v994: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 650/82020 objects misplaced (0.792%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:23.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:23.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[221067]: failed to encode map e365 with expected crc 2023-10-09T04:21:23.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:21:23.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[221067]: osdmap e365: 8 total, 8 up, 7 in 2023-10-09T04:21:23.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[221067]: failed to encode map e365 with expected crc 2023-10-09T04:21:23.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:22 smithi163 ceph-mon[221067]: pgmap v994: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 650/82020 objects misplaced (0.792%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:23.453 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:21:23.455 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:21:23.753 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:23.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:23.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:23.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:24.014 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:24.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:24.041 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:24.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:24.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:24.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:24.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:24.697 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":994,"stamp":"2023-10-09T04:21:21.873093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1790787769,"num_objects":27340,"num_object_clones":0,"num_object_copies":82020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":650,"num_objects_unfound":0,"num_objects_dirty":27340,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":170225,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22080,"num_bytes_recovered":1443145441,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96857,"ondisk_log_size":96857,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10029720,"kb_used_data":6066784,"kb_used_omap":126,"kb_used_meta":3962561,"kb_avail":646186344,"statfs":{"total":671965249536,"available":661694816256,"internally_reserved":0,"allocated":6212386816,"data_stored":6203086056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4057663099},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,7,4,1,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-511770624,"num_objects":-7809,"num_object_clones":0,"num_object_copies":-23427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2076,"num_objects_unfound":0,"num_objects_dirty":-7809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7809,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":787,"num_bytes_recovered":51576832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002699"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":488,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564190+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:21:09.564190+0000","last_peered":"2023-10-09T04:21:09.564190+0000","last_clean":"2023-10-09T04:21:09.564190+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:21:09.564190+0000","last_undegraded":"2023-10-09T04:21:09.564190+0000","last_fullsized":"2023-10-09T04:21:09.564190+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":108,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582944+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:21:09.582944+0000","last_peered":"2023-10-09T04:21:09.582944+0000","last_clean":"2023-10-09T04:21:09.582944+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:21:09.582944+0000","last_undegraded":"2023-10-09T04:21:09.582944+0000","last_fullsized":"2023-10-09T04:21:09.582944+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":339,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.573370+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:21:09.573370+0000","last_peered":"2023-10-09T04:21:09.573370+0000","last_clean":"2023-10-09T04:21:09.573370+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:21:09.573370+0000","last_undegraded":"2023-10-09T04:21:09.573370+0000","last_fullsized":"2023-10-09T04:21:09.573370+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":481,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564339+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:21:09.564339+0000","last_peered":"2023-10-09T04:21:09.564339+0000","last_clean":"2023-10-09T04:21:09.564339+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:21:09.564339+0000","last_undegraded":"2023-10-09T04:21:09.564339+0000","last_fullsized":"2023-10-09T04:21:09.564339+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":354,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.573371+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:21:09.573371+0000","last_peered":"2023-10-09T04:21:09.573371+0000","last_clean":"2023-10-09T04:21:09.573371+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:21:09.573371+0000","last_undegraded":"2023-10-09T04:21:09.573371+0000","last_fullsized":"2023-10-09T04:21:09.573371+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":343,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.568701+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:21:10.568701+0000","last_peered":"2023-10-09T04:21:10.568701+0000","last_clean":"2023-10-09T04:21:10.568701+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:21:10.568701+0000","last_undegraded":"2023-10-09T04:21:10.568701+0000","last_fullsized":"2023-10-09T04:21:10.568701+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":488,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582734+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:21:09.582734+0000","last_peered":"2023-10-09T04:21:09.582734+0000","last_clean":"2023-10-09T04:21:09.582734+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:21:09.582734+0000","last_undegraded":"2023-10-09T04:21:09.582734+0000","last_fullsized":"2023-10-09T04:21:09.582734+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":341,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T04:21:21.638059+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:21:21.638059+0000","last_peered":"2023-10-09T04:21:21.638059+0000","last_clean":"2023-10-09T04:21:21.638059+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:21:21.638059+0000","last_undegraded":"2023-10-09T04:21:21.638059+0000","last_fullsized":"2023-10-09T04:21:21.638059+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":307,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564797+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:21:09.564797+0000","last_peered":"2023-10-09T04:21:09.564797+0000","last_clean":"2023-10-09T04:21:09.564797+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:21:09.564797+0000","last_undegraded":"2023-10-09T04:21:09.564797+0000","last_fullsized":"2023-10-09T04:21:09.564797+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":763,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569589+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:21:09.569589+0000","last_peered":"2023-10-09T04:21:09.569589+0000","last_clean":"2023-10-09T04:21:09.569589+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:21:09.569589+0000","last_undegraded":"2023-10-09T04:21:09.569589+0000","last_fullsized":"2023-10-09T04:21:09.569589+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":341,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.569085+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:21:10.569085+0000","last_peered":"2023-10-09T04:21:10.569085+0000","last_clean":"2023-10-09T04:21:10.569085+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:21:10.569085+0000","last_undegraded":"2023-10-09T04:21:10.569085+0000","last_fullsized":"2023-10-09T04:21:10.569085+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":74,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.608041+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:21:09.608041+0000","last_peered":"2023-10-09T04:21:09.608041+0000","last_clean":"2023-10-09T04:21:09.608041+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:21:09.608041+0000","last_undegraded":"2023-10-09T04:21:09.608041+0000","last_fullsized":"2023-10-09T04:21:09.608041+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":490,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575309+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:21:09.575309+0000","last_peered":"2023-10-09T04:21:09.575309+0000","last_clean":"2023-10-09T04:21:09.575309+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:21:09.575309+0000","last_undegraded":"2023-10-09T04:21:09.575309+0000","last_fullsized":"2023-10-09T04:21:09.575309+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":323,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564556+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:21:09.564556+0000","last_peered":"2023-10-09T04:21:09.564556+0000","last_clean":"2023-10-09T04:21:09.564556+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:21:09.564556+0000","last_undegraded":"2023-10-09T04:21:09.564556+0000","last_fullsized":"2023-10-09T04:21:09.564556+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":58,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564145+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:21:09.564145+0000","last_peered":"2023-10-09T04:21:09.564145+0000","last_clean":"2023-10-09T04:21:09.564145+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:21:09.564145+0000","last_undegraded":"2023-10-09T04:21:09.564145+0000","last_fullsized":"2023-10-09T04:21:09.564145+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":498,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564638+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:21:09.564638+0000","last_peered":"2023-10-09T04:21:09.564638+0000","last_clean":"2023-10-09T04:21:09.564638+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:21:09.564638+0000","last_undegraded":"2023-10-09T04:21:09.564638+0000","last_fullsized":"2023-10-09T04:21:09.564638+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":406,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.570271+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:21:09.570271+0000","last_peered":"2023-10-09T04:21:09.570271+0000","last_clean":"2023-10-09T04:21:09.570271+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:21:09.570271+0000","last_undegraded":"2023-10-09T04:21:09.570271+0000","last_fullsized":"2023-10-09T04:21:09.570271+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.574800+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:21:09.574800+0000","last_peered":"2023-10-09T04:21:09.574800+0000","last_clean":"2023-10-09T04:21:09.574800+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:21:09.574800+0000","last_undegraded":"2023-10-09T04:21:09.574800+0000","last_fullsized":"2023-10-09T04:21:09.574800+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":506,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.574746+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:21:09.574746+0000","last_peered":"2023-10-09T04:21:09.574746+0000","last_clean":"2023-10-09T04:21:09.574746+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:21:09.574746+0000","last_undegraded":"2023-10-09T04:21:09.574746+0000","last_fullsized":"2023-10-09T04:21:09.574746+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":138,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581548+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:21:09.581548+0000","last_peered":"2023-10-09T04:21:09.581548+0000","last_clean":"2023-10-09T04:21:09.581548+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:21:09.581548+0000","last_undegraded":"2023-10-09T04:21:09.581548+0000","last_fullsized":"2023-10-09T04:21:09.581548+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582032+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:21:09.582032+0000","last_peered":"2023-10-09T04:21:09.582032+0000","last_clean":"2023-10-09T04:21:09.582032+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:21:09.582032+0000","last_undegraded":"2023-10-09T04:21:09.582032+0000","last_fullsized":"2023-10-09T04:21:09.582032+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":515,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567616+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:21:10.567616+0000","last_peered":"2023-10-09T04:21:10.567616+0000","last_clean":"2023-10-09T04:21:10.567616+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:21:10.567616+0000","last_undegraded":"2023-10-09T04:21:10.567616+0000","last_fullsized":"2023-10-09T04:21:10.567616+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":357,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569332+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:21:09.569332+0000","last_peered":"2023-10-09T04:21:09.569332+0000","last_clean":"2023-10-09T04:21:09.569332+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:21:09.569332+0000","last_undegraded":"2023-10-09T04:21:09.569332+0000","last_fullsized":"2023-10-09T04:21:09.569332+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":42,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.574169+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:21:09.574169+0000","last_peered":"2023-10-09T04:21:09.574169+0000","last_clean":"2023-10-09T04:21:09.574169+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:21:09.574169+0000","last_undegraded":"2023-10-09T04:21:09.574169+0000","last_fullsized":"2023-10-09T04:21:09.574169+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":508,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564450+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:21:09.564450+0000","last_peered":"2023-10-09T04:21:09.564450+0000","last_clean":"2023-10-09T04:21:09.564450+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:21:09.564450+0000","last_undegraded":"2023-10-09T04:21:09.564450+0000","last_fullsized":"2023-10-09T04:21:09.564450+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":342,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567630+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:21:10.567630+0000","last_peered":"2023-10-09T04:21:10.567630+0000","last_clean":"2023-10-09T04:21:10.567630+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:21:10.567630+0000","last_undegraded":"2023-10-09T04:21:10.567630+0000","last_fullsized":"2023-10-09T04:21:10.567630+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":342,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.565849+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:21:09.565849+0000","last_peered":"2023-10-09T04:21:09.565849+0000","last_clean":"2023-10-09T04:21:09.565849+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:21:09.565849+0000","last_undegraded":"2023-10-09T04:21:09.565849+0000","last_fullsized":"2023-10-09T04:21:09.565849+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":506,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.573842+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:21:09.573842+0000","last_peered":"2023-10-09T04:21:09.573842+0000","last_clean":"2023-10-09T04:21:09.573842+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:21:09.573842+0000","last_undegraded":"2023-10-09T04:21:09.573842+0000","last_fullsized":"2023-10-09T04:21:09.573842+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581717+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:21:09.581717+0000","last_peered":"2023-10-09T04:21:09.581717+0000","last_clean":"2023-10-09T04:21:09.581717+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:21:09.581717+0000","last_undegraded":"2023-10-09T04:21:09.581717+0000","last_fullsized":"2023-10-09T04:21:09.581717+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":342,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T04:21:21.638097+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:21:21.638097+0000","last_peered":"2023-10-09T04:21:21.638097+0000","last_clean":"2023-10-09T04:21:21.638097+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:21:21.638097+0000","last_undegraded":"2023-10-09T04:21:21.638097+0000","last_fullsized":"2023-10-09T04:21:21.638097+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":508,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.567001+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:21:09.567001+0000","last_peered":"2023-10-09T04:21:09.567001+0000","last_clean":"2023-10-09T04:21:09.567001+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:21:09.567001+0000","last_undegraded":"2023-10-09T04:21:09.567001+0000","last_fullsized":"2023-10-09T04:21:09.567001+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":355,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.569463+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:21:10.569463+0000","last_peered":"2023-10-09T04:21:10.569463+0000","last_clean":"2023-10-09T04:21:10.569463+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:21:10.569463+0000","last_undegraded":"2023-10-09T04:21:10.569463+0000","last_fullsized":"2023-10-09T04:21:10.569463+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.588185+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:21:09.588185+0000","last_peered":"2023-10-09T04:21:09.588185+0000","last_clean":"2023-10-09T04:21:09.588185+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:21:09.588185+0000","last_undegraded":"2023-10-09T04:21:09.588185+0000","last_fullsized":"2023-10-09T04:21:09.588185+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"365'10520","reported_seq":10589,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T04:21:21.853130+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:21:21.853130+0000","last_peered":"2023-10-09T04:21:21.853130+0000","last_clean":"2023-10-09T04:21:21.853130+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:21:21.853130+0000","last_undegraded":"2023-10-09T04:21:21.853130+0000","last_fullsized":"2023-10-09T04:21:21.853130+0000","mapping_epoch":306,"log_start":"325'4200","ondisk_log_start":"325'4200","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6320,"log_dups_size":0,"ondisk_log_size":6320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108789760,"num_objects":1660,"num_object_clones":0,"num_object_copies":4980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10520,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":766,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569983+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:21:09.569983+0000","last_peered":"2023-10-09T04:21:09.569983+0000","last_clean":"2023-10-09T04:21:09.569983+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:21:09.569983+0000","last_undegraded":"2023-10-09T04:21:09.569983+0000","last_fullsized":"2023-10-09T04:21:09.569983+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":352,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569626+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:21:09.569626+0000","last_peered":"2023-10-09T04:21:09.569626+0000","last_clean":"2023-10-09T04:21:09.569626+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:21:09.569626+0000","last_undegraded":"2023-10-09T04:21:09.569626+0000","last_fullsized":"2023-10-09T04:21:09.569626+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":315,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564372+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:21:09.564372+0000","last_peered":"2023-10-09T04:21:09.564372+0000","last_clean":"2023-10-09T04:21:09.564372+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:21:09.564372+0000","last_undegraded":"2023-10-09T04:21:09.564372+0000","last_fullsized":"2023-10-09T04:21:09.564372+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"364'10554","reported_seq":10631,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:18.463213+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:21:18.463213+0000","last_peered":"2023-10-09T04:21:18.463213+0000","last_clean":"2023-10-09T04:21:18.463213+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:21:18.463213+0000","last_undegraded":"2023-10-09T04:21:18.463213+0000","last_fullsized":"2023-10-09T04:21:18.463213+0000","mapping_epoch":310,"log_start":"331'5273","ondisk_log_start":"331'5273","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5281,"log_dups_size":0,"ondisk_log_size":5281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120717312,"num_objects":1842,"num_object_clones":0,"num_object_copies":5526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10554,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":531,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.609653+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:21:09.609653+0000","last_peered":"2023-10-09T04:21:09.609653+0000","last_clean":"2023-10-09T04:21:09.609653+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:21:09.609653+0000","last_undegraded":"2023-10-09T04:21:09.609653+0000","last_fullsized":"2023-10-09T04:21:09.609653+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":356,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569658+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:21:09.569658+0000","last_peered":"2023-10-09T04:21:09.569658+0000","last_clean":"2023-10-09T04:21:09.569658+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:21:09.569658+0000","last_undegraded":"2023-10-09T04:21:09.569658+0000","last_fullsized":"2023-10-09T04:21:09.569658+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":138,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581863+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:21:09.581863+0000","last_peered":"2023-10-09T04:21:09.581863+0000","last_clean":"2023-10-09T04:21:09.581863+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:21:09.581863+0000","last_undegraded":"2023-10-09T04:21:09.581863+0000","last_fullsized":"2023-10-09T04:21:09.581863+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"364'10829","reported_seq":16751,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.862778+0000","last_change":"2023-10-09T04:21:10.590718+0000","last_active":"2023-10-09T04:21:20.862778+0000","last_peered":"2023-10-09T04:21:20.862778+0000","last_clean":"2023-10-09T04:21:20.862778+0000","last_became_active":"2023-10-09T04:21:10.569349+0000","last_became_peered":"2023-10-09T04:21:10.569349+0000","last_unstale":"2023-10-09T04:21:20.862778+0000","last_undegraded":"2023-10-09T04:21:20.862778+0000","last_fullsized":"2023-10-09T04:21:20.862778+0000","mapping_epoch":363,"log_start":"330'4918","ondisk_log_start":"330'4918","created":306,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:41.279693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10829,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2916,"num_bytes_recovered":191102976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":498,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564552+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:21:09.564552+0000","last_peered":"2023-10-09T04:21:09.564552+0000","last_clean":"2023-10-09T04:21:09.564552+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:21:09.564552+0000","last_undegraded":"2023-10-09T04:21:09.564552+0000","last_fullsized":"2023-10-09T04:21:09.564552+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":321,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.570133+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:21:09.570133+0000","last_peered":"2023-10-09T04:21:09.570133+0000","last_clean":"2023-10-09T04:21:09.570133+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:21:09.570133+0000","last_undegraded":"2023-10-09T04:21:09.570133+0000","last_fullsized":"2023-10-09T04:21:09.570133+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":296,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.577133+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:21:09.577133+0000","last_peered":"2023-10-09T04:21:09.577133+0000","last_clean":"2023-10-09T04:21:09.577133+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:21:09.577133+0000","last_undegraded":"2023-10-09T04:21:09.577133+0000","last_fullsized":"2023-10-09T04:21:09.577133+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"364'10673","reported_seq":10739,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.852506+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:21:20.852506+0000","last_peered":"2023-10-09T04:21:20.852506+0000","last_clean":"2023-10-09T04:21:20.852506+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:21:20.852506+0000","last_undegraded":"2023-10-09T04:21:20.852506+0000","last_fullsized":"2023-10-09T04:21:20.852506+0000","mapping_epoch":306,"log_start":"330'4787","ondisk_log_start":"330'4787","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108199936,"num_objects":1651,"num_object_clones":0,"num_object_copies":4953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10673,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":508,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582530+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:21:09.582530+0000","last_peered":"2023-10-09T04:21:09.582530+0000","last_clean":"2023-10-09T04:21:09.582530+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:21:09.582530+0000","last_undegraded":"2023-10-09T04:21:09.582530+0000","last_fullsized":"2023-10-09T04:21:09.582530+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":353,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564603+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:21:09.564603+0000","last_peered":"2023-10-09T04:21:09.564603+0000","last_clean":"2023-10-09T04:21:09.564603+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:21:09.564603+0000","last_undegraded":"2023-10-09T04:21:09.564603+0000","last_fullsized":"2023-10-09T04:21:09.564603+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567751+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:21:10.567751+0000","last_peered":"2023-10-09T04:21:10.567751+0000","last_clean":"2023-10-09T04:21:10.567751+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:21:10.567751+0000","last_undegraded":"2023-10-09T04:21:10.567751+0000","last_fullsized":"2023-10-09T04:21:10.567751+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"364'10237","reported_seq":13676,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.898264+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:21:20.898264+0000","last_peered":"2023-10-09T04:21:20.898264+0000","last_clean":"2023-10-09T04:21:20.898264+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:21:20.898264+0000","last_undegraded":"2023-10-09T04:21:20.898264+0000","last_fullsized":"2023-10-09T04:21:20.898264+0000","mapping_epoch":320,"log_start":"331'4900","ondisk_log_start":"331'4900","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107544576,"num_objects":1641,"num_object_clones":0,"num_object_copies":4923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10237,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":787,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569563+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:21:09.569563+0000","last_peered":"2023-10-09T04:21:09.569563+0000","last_clean":"2023-10-09T04:21:09.569563+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:21:09.569563+0000","last_undegraded":"2023-10-09T04:21:09.569563+0000","last_fullsized":"2023-10-09T04:21:09.569563+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":325,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567842+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:21:10.567842+0000","last_peered":"2023-10-09T04:21:10.567842+0000","last_clean":"2023-10-09T04:21:10.567842+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:21:10.567842+0000","last_undegraded":"2023-10-09T04:21:10.567842+0000","last_fullsized":"2023-10-09T04:21:10.567842+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581624+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:21:09.581624+0000","last_peered":"2023-10-09T04:21:09.581624+0000","last_clean":"2023-10-09T04:21:09.581624+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:21:09.581624+0000","last_undegraded":"2023-10-09T04:21:09.581624+0000","last_fullsized":"2023-10-09T04:21:09.581624+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"364'10417","reported_seq":13996,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:18.476574+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:21:18.476574+0000","last_peered":"2023-10-09T04:21:18.476574+0000","last_clean":"2023-10-09T04:21:18.476574+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:21:18.476574+0000","last_undegraded":"2023-10-09T04:21:18.476574+0000","last_fullsized":"2023-10-09T04:21:18.476574+0000","mapping_epoch":329,"log_start":"331'5060","ondisk_log_start":"331'5060","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121831424,"num_objects":1859,"num_object_clones":0,"num_object_copies":5577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10417,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":533,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564080+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:21:09.564080+0000","last_peered":"2023-10-09T04:21:09.564080+0000","last_clean":"2023-10-09T04:21:09.564080+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:21:09.564080+0000","last_undegraded":"2023-10-09T04:21:09.564080+0000","last_fullsized":"2023-10-09T04:21:09.564080+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582807+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:21:09.582807+0000","last_peered":"2023-10-09T04:21:09.582807+0000","last_clean":"2023-10-09T04:21:09.582807+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:21:09.582807+0000","last_undegraded":"2023-10-09T04:21:09.582807+0000","last_fullsized":"2023-10-09T04:21:09.582807+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582068+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:21:09.582068+0000","last_peered":"2023-10-09T04:21:09.582068+0000","last_clean":"2023-10-09T04:21:09.582068+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:21:09.582068+0000","last_undegraded":"2023-10-09T04:21:09.582068+0000","last_fullsized":"2023-10-09T04:21:09.582068+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"364'10537","reported_seq":10612,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.879583+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:21:20.879583+0000","last_peered":"2023-10-09T04:21:20.879583+0000","last_clean":"2023-10-09T04:21:20.879583+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:21:20.879583+0000","last_undegraded":"2023-10-09T04:21:20.879583+0000","last_fullsized":"2023-10-09T04:21:20.879583+0000","mapping_epoch":310,"log_start":"331'5200","ondisk_log_start":"331'5200","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447424,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10537,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":489,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.573759+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:21:09.573759+0000","last_peered":"2023-10-09T04:21:09.573759+0000","last_clean":"2023-10-09T04:21:09.573759+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:21:09.573759+0000","last_undegraded":"2023-10-09T04:21:09.573759+0000","last_fullsized":"2023-10-09T04:21:09.573759+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":108,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581588+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:21:09.581588+0000","last_peered":"2023-10-09T04:21:09.581588+0000","last_clean":"2023-10-09T04:21:09.581588+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:21:09.581588+0000","last_undegraded":"2023-10-09T04:21:09.581588+0000","last_fullsized":"2023-10-09T04:21:09.581588+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":158,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582701+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:21:09.582701+0000","last_peered":"2023-10-09T04:21:09.582701+0000","last_clean":"2023-10-09T04:21:09.582701+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:21:09.582701+0000","last_undegraded":"2023-10-09T04:21:09.582701+0000","last_fullsized":"2023-10-09T04:21:09.582701+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"364'10562","reported_seq":19676,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:18.501535+0000","last_change":"2023-10-09T04:20:48.014336+0000","last_active":"2023-10-09T04:21:18.501535+0000","last_peered":"2023-10-09T04:21:18.501535+0000","last_clean":"2023-10-09T04:21:18.501535+0000","last_became_active":"2023-10-09T04:20:47.989792+0000","last_became_peered":"2023-10-09T04:20:47.989792+0000","last_unstale":"2023-10-09T04:21:18.501535+0000","last_undegraded":"2023-10-09T04:21:18.501535+0000","last_fullsized":"2023-10-09T04:21:18.501535+0000","mapping_epoch":359,"log_start":"331'5281","ondisk_log_start":"331'5281","created":306,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5281,"log_dups_size":0,"ondisk_log_size":5281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:26.170681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10562,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4512,"num_bytes_recovered":295698432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":508,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.566721+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:21:09.566721+0000","last_peered":"2023-10-09T04:21:09.566721+0000","last_clean":"2023-10-09T04:21:09.566721+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:21:09.566721+0000","last_undegraded":"2023-10-09T04:21:09.566721+0000","last_fullsized":"2023-10-09T04:21:09.566721+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":26,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.607376+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:21:09.607376+0000","last_peered":"2023-10-09T04:21:09.607376+0000","last_clean":"2023-10-09T04:21:09.607376+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:21:09.607376+0000","last_undegraded":"2023-10-09T04:21:09.607376+0000","last_fullsized":"2023-10-09T04:21:09.607376+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":31,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575592+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:21:09.575592+0000","last_peered":"2023-10-09T04:21:09.575592+0000","last_clean":"2023-10-09T04:21:09.575592+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:21:09.575592+0000","last_undegraded":"2023-10-09T04:21:09.575592+0000","last_fullsized":"2023-10-09T04:21:09.575592+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"364'10691","reported_seq":10767,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.862828+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:21:20.862828+0000","last_peered":"2023-10-09T04:21:20.862828+0000","last_clean":"2023-10-09T04:21:20.862828+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:21:20.862828+0000","last_undegraded":"2023-10-09T04:21:20.862828+0000","last_fullsized":"2023-10-09T04:21:20.862828+0000","mapping_epoch":310,"log_start":"330'4734","ondisk_log_start":"330'4734","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112656384,"num_objects":1719,"num_object_clones":0,"num_object_copies":5157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10691,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":528,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564972+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:21:09.564972+0000","last_peered":"2023-10-09T04:21:09.564972+0000","last_clean":"2023-10-09T04:21:09.564972+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:21:09.564972+0000","last_undegraded":"2023-10-09T04:21:09.564972+0000","last_fullsized":"2023-10-09T04:21:09.564972+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":344,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.602877+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:21:10.602877+0000","last_peered":"2023-10-09T04:21:10.602877+0000","last_clean":"2023-10-09T04:21:10.602877+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:21:10.602877+0000","last_undegraded":"2023-10-09T04:21:10.602877+0000","last_fullsized":"2023-10-09T04:21:10.602877+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":360,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.565736+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:21:09.565736+0000","last_peered":"2023-10-09T04:21:09.565736+0000","last_clean":"2023-10-09T04:21:09.565736+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:21:09.565736+0000","last_undegraded":"2023-10-09T04:21:09.565736+0000","last_fullsized":"2023-10-09T04:21:09.565736+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575950+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:21:09.575950+0000","last_peered":"2023-10-09T04:21:09.575950+0000","last_clean":"2023-10-09T04:21:09.575950+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:21:09.575950+0000","last_undegraded":"2023-10-09T04:21:09.575950+0000","last_fullsized":"2023-10-09T04:21:09.575950+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"365'10684","reported_seq":10753,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T04:21:21.847577+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:21:21.847577+0000","last_peered":"2023-10-09T04:21:21.847577+0000","last_clean":"2023-10-09T04:21:21.847577+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:21:21.847577+0000","last_undegraded":"2023-10-09T04:21:21.847577+0000","last_fullsized":"2023-10-09T04:21:21.847577+0000","mapping_epoch":306,"log_start":"327'4400","ondisk_log_start":"327'4400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106037248,"num_objects":1618,"num_object_clones":0,"num_object_copies":4854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10684,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":481,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564756+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:21:09.564756+0000","last_peered":"2023-10-09T04:21:09.564756+0000","last_clean":"2023-10-09T04:21:09.564756+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:21:09.564756+0000","last_undegraded":"2023-10-09T04:21:09.564756+0000","last_fullsized":"2023-10-09T04:21:09.564756+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":337,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.566800+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:21:09.566800+0000","last_peered":"2023-10-09T04:21:09.566800+0000","last_clean":"2023-10-09T04:21:09.566800+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:21:09.566800+0000","last_undegraded":"2023-10-09T04:21:09.566800+0000","last_fullsized":"2023-10-09T04:21:09.566800+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581952+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:21:09.581952+0000","last_peered":"2023-10-09T04:21:09.581952+0000","last_clean":"2023-10-09T04:21:09.581952+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:21:09.581952+0000","last_undegraded":"2023-10-09T04:21:09.581952+0000","last_fullsized":"2023-10-09T04:21:09.581952+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":372,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.602831+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:21:10.602831+0000","last_peered":"2023-10-09T04:21:10.602831+0000","last_clean":"2023-10-09T04:21:10.602831+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:21:10.602831+0000","last_undegraded":"2023-10-09T04:21:10.602831+0000","last_fullsized":"2023-10-09T04:21:10.602831+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"364'10789","reported_seq":10855,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.833428+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:21:20.833428+0000","last_peered":"2023-10-09T04:21:20.833428+0000","last_clean":"2023-10-09T04:21:20.833428+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:21:20.833428+0000","last_undegraded":"2023-10-09T04:21:20.833428+0000","last_fullsized":"2023-10-09T04:21:20.833428+0000","mapping_epoch":306,"log_start":"330'4868","ondisk_log_start":"330'4868","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5921,"log_dups_size":0,"ondisk_log_size":5921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110821376,"num_objects":1691,"num_object_clones":0,"num_object_copies":5073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10789,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":498,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567717+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:21:10.567717+0000","last_peered":"2023-10-09T04:21:10.567717+0000","last_clean":"2023-10-09T04:21:10.567717+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:21:10.567717+0000","last_undegraded":"2023-10-09T04:21:10.567717+0000","last_fullsized":"2023-10-09T04:21:10.567717+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":200,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575129+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:21:09.575129+0000","last_peered":"2023-10-09T04:21:09.575129+0000","last_clean":"2023-10-09T04:21:09.575129+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:21:09.575129+0000","last_undegraded":"2023-10-09T04:21:09.575129+0000","last_fullsized":"2023-10-09T04:21:09.575129+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":347,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.576400+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:21:09.576400+0000","last_peered":"2023-10-09T04:21:09.576400+0000","last_clean":"2023-10-09T04:21:09.576400+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:21:09.576400+0000","last_undegraded":"2023-10-09T04:21:09.576400+0000","last_fullsized":"2023-10-09T04:21:09.576400+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":42,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.574367+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:21:09.574367+0000","last_peered":"2023-10-09T04:21:09.574367+0000","last_clean":"2023-10-09T04:21:09.574367+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:21:09.574367+0000","last_undegraded":"2023-10-09T04:21:09.574367+0000","last_fullsized":"2023-10-09T04:21:09.574367+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"365'10741","reported_seq":19276,"reported_epoch":365,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:21:21.857380+0000","last_change":"2023-10-09T04:21:08.747488+0000","last_active":"2023-10-09T04:21:21.857380+0000","last_peered":"2023-10-09T04:21:21.857380+0000","last_clean":"2023-10-09T04:19:41.729842+0000","last_became_active":"2023-10-09T04:20:02.106154+0000","last_became_peered":"2023-10-09T04:20:02.106154+0000","last_unstale":"2023-10-09T04:21:21.857380+0000","last_undegraded":"2023-10-09T04:21:21.857380+0000","last_fullsized":"2023-10-09T04:21:21.857380+0000","mapping_epoch":348,"log_start":"327'4400","ondisk_log_start":"327'4400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6341,"log_dups_size":0,"ondisk_log_size":6341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:45.805086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":650,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10741,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3732,"num_bytes_recovered":244580352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1647}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":347,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564285+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:21:09.564285+0000","last_peered":"2023-10-09T04:21:09.564285+0000","last_clean":"2023-10-09T04:21:09.564285+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:21:09.564285+0000","last_undegraded":"2023-10-09T04:21:09.564285+0000","last_fullsized":"2023-10-09T04:21:09.564285+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":758,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.570679+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:21:09.570679+0000","last_peered":"2023-10-09T04:21:09.570679+0000","last_clean":"2023-10-09T04:21:09.570679+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:21:09.570679+0000","last_undegraded":"2023-10-09T04:21:09.570679+0000","last_fullsized":"2023-10-09T04:21:09.570679+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":290,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.566873+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:21:09.566873+0000","last_peered":"2023-10-09T04:21:09.566873+0000","last_clean":"2023-10-09T04:21:09.566873+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:21:09.566873+0000","last_undegraded":"2023-10-09T04:21:09.566873+0000","last_fullsized":"2023-10-09T04:21:09.566873+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582135+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:21:09.582135+0000","last_peered":"2023-10-09T04:21:09.582135+0000","last_clean":"2023-10-09T04:21:09.582135+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:21:09.582135+0000","last_undegraded":"2023-10-09T04:21:09.582135+0000","last_fullsized":"2023-10-09T04:21:09.582135+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"365'10695","reported_seq":14343,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T04:21:21.847447+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:21:21.847447+0000","last_peered":"2023-10-09T04:21:21.847447+0000","last_clean":"2023-10-09T04:21:21.847447+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:21:21.847447+0000","last_undegraded":"2023-10-09T04:21:21.847447+0000","last_fullsized":"2023-10-09T04:21:21.847447+0000","mapping_epoch":323,"log_start":"327'4400","ondisk_log_start":"327'4400","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6295,"log_dups_size":0,"ondisk_log_size":6295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107413504,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10695,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":507,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.573927+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:21:09.573927+0000","last_peered":"2023-10-09T04:21:09.573927+0000","last_clean":"2023-10-09T04:21:09.573927+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:21:09.573927+0000","last_undegraded":"2023-10-09T04:21:09.573927+0000","last_fullsized":"2023-10-09T04:21:09.573927+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":344,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569105+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:21:09.569105+0000","last_peered":"2023-10-09T04:21:09.569105+0000","last_clean":"2023-10-09T04:21:09.569105+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:21:09.569105+0000","last_undegraded":"2023-10-09T04:21:09.569105+0000","last_fullsized":"2023-10-09T04:21:09.569105+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":361,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567961+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:21:10.567961+0000","last_peered":"2023-10-09T04:21:10.567961+0000","last_clean":"2023-10-09T04:21:10.567961+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:21:10.567961+0000","last_undegraded":"2023-10-09T04:21:10.567961+0000","last_fullsized":"2023-10-09T04:21:10.567961+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":357,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.568540+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:21:10.568540+0000","last_peered":"2023-10-09T04:21:10.568540+0000","last_clean":"2023-10-09T04:21:10.568540+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:21:10.568540+0000","last_undegraded":"2023-10-09T04:21:10.568540+0000","last_fullsized":"2023-10-09T04:21:10.568540+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"364'10715","reported_seq":10781,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.927979+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:21:20.927979+0000","last_peered":"2023-10-09T04:21:20.927979+0000","last_clean":"2023-10-09T04:21:20.927979+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:21:20.927979+0000","last_undegraded":"2023-10-09T04:21:20.927979+0000","last_fullsized":"2023-10-09T04:21:20.927979+0000","mapping_epoch":306,"log_start":"331'5400","ondisk_log_start":"331'5400","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5315,"log_dups_size":0,"ondisk_log_size":5315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115277824,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10715,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":517,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582398+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:21:09.582398+0000","last_peered":"2023-10-09T04:21:09.582398+0000","last_clean":"2023-10-09T04:21:09.582398+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:21:09.582398+0000","last_undegraded":"2023-10-09T04:21:09.582398+0000","last_fullsized":"2023-10-09T04:21:09.582398+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":347,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.570199+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:21:09.570199+0000","last_peered":"2023-10-09T04:21:09.570199+0000","last_clean":"2023-10-09T04:21:09.570199+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:21:09.570199+0000","last_undegraded":"2023-10-09T04:21:09.570199+0000","last_fullsized":"2023-10-09T04:21:09.570199+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":338,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.602132+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:21:10.602132+0000","last_peered":"2023-10-09T04:21:10.602132+0000","last_clean":"2023-10-09T04:21:10.602132+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:21:10.602132+0000","last_undegraded":"2023-10-09T04:21:10.602132+0000","last_fullsized":"2023-10-09T04:21:10.602132+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":63,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564251+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:21:09.564251+0000","last_peered":"2023-10-09T04:21:09.564251+0000","last_clean":"2023-10-09T04:21:09.564251+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:21:09.564251+0000","last_undegraded":"2023-10-09T04:21:09.564251+0000","last_fullsized":"2023-10-09T04:21:09.564251+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"364'10654","reported_seq":22063,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.865397+0000","last_change":"2023-10-09T04:20:12.010618+0000","last_active":"2023-10-09T04:21:20.865397+0000","last_peered":"2023-10-09T04:21:20.865397+0000","last_clean":"2023-10-09T04:21:20.865397+0000","last_became_active":"2023-10-09T04:20:12.002110+0000","last_became_peered":"2023-10-09T04:20:12.002110+0000","last_unstale":"2023-10-09T04:21:20.865397+0000","last_undegraded":"2023-10-09T04:21:20.865397+0000","last_fullsized":"2023-10-09T04:21:20.865397+0000","mapping_epoch":354,"log_start":"331'5300","ondisk_log_start":"331'5300","created":306,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5354,"log_dups_size":0,"ondisk_log_size":5354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:12.102225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108593176,"num_objects":1658,"num_object_clones":0,"num_object_copies":4974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1658,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10654,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":508,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581904+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:21:09.581904+0000","last_peered":"2023-10-09T04:21:09.581904+0000","last_clean":"2023-10-09T04:21:09.581904+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:21:09.581904+0000","last_undegraded":"2023-10-09T04:21:09.581904+0000","last_fullsized":"2023-10-09T04:21:09.581904+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":296,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569827+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:21:09.569827+0000","last_peered":"2023-10-09T04:21:09.569827+0000","last_clean":"2023-10-09T04:21:09.569827+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:21:09.569827+0000","last_undegraded":"2023-10-09T04:21:09.569827+0000","last_fullsized":"2023-10-09T04:21:09.569827+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":331,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.574859+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:21:09.574859+0000","last_peered":"2023-10-09T04:21:09.574859+0000","last_clean":"2023-10-09T04:21:09.574859+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:21:09.574859+0000","last_undegraded":"2023-10-09T04:21:09.574859+0000","last_fullsized":"2023-10-09T04:21:09.574859+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":314,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.570840+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:21:09.570840+0000","last_peered":"2023-10-09T04:21:09.570840+0000","last_clean":"2023-10-09T04:21:09.570840+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:21:09.570840+0000","last_undegraded":"2023-10-09T04:21:09.570840+0000","last_fullsized":"2023-10-09T04:21:09.570840+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"364'10834","reported_seq":10910,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:20.411665+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:21:20.411665+0000","last_peered":"2023-10-09T04:21:20.411665+0000","last_clean":"2023-10-09T04:21:20.411665+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:21:20.411665+0000","last_undegraded":"2023-10-09T04:21:20.411665+0000","last_fullsized":"2023-10-09T04:21:20.411665+0000","mapping_epoch":310,"log_start":"331'5200","ondisk_log_start":"331'5200","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5634,"log_dups_size":0,"ondisk_log_size":5634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112459776,"num_objects":1716,"num_object_clones":0,"num_object_copies":5148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10834,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":502,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.588418+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:21:09.588418+0000","last_peered":"2023-10-09T04:21:09.588418+0000","last_clean":"2023-10-09T04:21:09.588418+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:21:09.588418+0000","last_undegraded":"2023-10-09T04:21:09.588418+0000","last_fullsized":"2023-10-09T04:21:09.588418+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":196,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575517+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:21:09.575517+0000","last_peered":"2023-10-09T04:21:09.575517+0000","last_clean":"2023-10-09T04:21:09.575517+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:21:09.575517+0000","last_undegraded":"2023-10-09T04:21:09.575517+0000","last_fullsized":"2023-10-09T04:21:09.575517+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":370,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.568627+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:21:10.568627+0000","last_peered":"2023-10-09T04:21:10.568627+0000","last_clean":"2023-10-09T04:21:10.568627+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:21:10.568627+0000","last_undegraded":"2023-10-09T04:21:10.568627+0000","last_fullsized":"2023-10-09T04:21:10.568627+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581790+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:21:09.581790+0000","last_peered":"2023-10-09T04:21:09.581790+0000","last_clean":"2023-10-09T04:21:09.581790+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:21:09.581790+0000","last_undegraded":"2023-10-09T04:21:09.581790+0000","last_fullsized":"2023-10-09T04:21:09.581790+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":524,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.568035+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:21:10.568035+0000","last_peered":"2023-10-09T04:21:10.568035+0000","last_clean":"2023-10-09T04:21:10.568035+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:21:10.568035+0000","last_undegraded":"2023-10-09T04:21:10.568035+0000","last_fullsized":"2023-10-09T04:21:10.568035+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":339,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.567934+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:21:10.567934+0000","last_peered":"2023-10-09T04:21:10.567934+0000","last_clean":"2023-10-09T04:21:10.567934+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:21:10.567934+0000","last_undegraded":"2023-10-09T04:21:10.567934+0000","last_fullsized":"2023-10-09T04:21:10.567934+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":199,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575493+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:21:09.575493+0000","last_peered":"2023-10-09T04:21:09.575493+0000","last_clean":"2023-10-09T04:21:09.575493+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:21:09.575493+0000","last_undegraded":"2023-10-09T04:21:09.575493+0000","last_fullsized":"2023-10-09T04:21:09.575493+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":751,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.569627+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:21:09.569627+0000","last_peered":"2023-10-09T04:21:09.569627+0000","last_clean":"2023-10-09T04:21:09.569627+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:21:09.569627+0000","last_undegraded":"2023-10-09T04:21:09.569627+0000","last_fullsized":"2023-10-09T04:21:09.569627+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":353,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.564681+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:21:09.564681+0000","last_peered":"2023-10-09T04:21:09.564681+0000","last_clean":"2023-10-09T04:21:09.564681+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:21:09.564681+0000","last_undegraded":"2023-10-09T04:21:09.564681+0000","last_fullsized":"2023-10-09T04:21:09.564681+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":128,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582111+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:21:09.582111+0000","last_peered":"2023-10-09T04:21:09.582111+0000","last_clean":"2023-10-09T04:21:09.582111+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:21:09.582111+0000","last_undegraded":"2023-10-09T04:21:09.582111+0000","last_fullsized":"2023-10-09T04:21:09.582111+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":350,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.565667+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:21:09.565667+0000","last_peered":"2023-10-09T04:21:09.565667+0000","last_clean":"2023-10-09T04:21:09.565667+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:21:09.565667+0000","last_undegraded":"2023-10-09T04:21:09.565667+0000","last_fullsized":"2023-10-09T04:21:09.565667+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.582870+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:21:09.582870+0000","last_peered":"2023-10-09T04:21:09.582870+0000","last_clean":"2023-10-09T04:21:09.582870+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:21:09.582870+0000","last_undegraded":"2023-10-09T04:21:09.582870+0000","last_fullsized":"2023-10-09T04:21:09.582870+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":786,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.570773+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:21:09.570773+0000","last_peered":"2023-10-09T04:21:09.570773+0000","last_clean":"2023-10-09T04:21:09.570773+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:21:09.570773+0000","last_undegraded":"2023-10-09T04:21:09.570773+0000","last_fullsized":"2023-10-09T04:21:09.570773+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":209,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.576298+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:21:09.576298+0000","last_peered":"2023-10-09T04:21:09.576298+0000","last_clean":"2023-10-09T04:21:09.576298+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:21:09.576298+0000","last_undegraded":"2023-10-09T04:21:09.576298+0000","last_fullsized":"2023-10-09T04:21:09.576298+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":199,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.575213+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:21:09.575213+0000","last_peered":"2023-10-09T04:21:09.575213+0000","last_clean":"2023-10-09T04:21:09.575213+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:21:09.575213+0000","last_undegraded":"2023-10-09T04:21:09.575213+0000","last_fullsized":"2023-10-09T04:21:09.575213+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":506,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.573558+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:21:09.573558+0000","last_peered":"2023-10-09T04:21:09.573558+0000","last_clean":"2023-10-09T04:21:09.573558+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:21:09.573558+0000","last_undegraded":"2023-10-09T04:21:09.573558+0000","last_fullsized":"2023-10-09T04:21:09.573558+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":359,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T04:21:10.569169+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:21:10.569169+0000","last_peered":"2023-10-09T04:21:10.569169+0000","last_clean":"2023-10-09T04:21:10.569169+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:21:10.569169+0000","last_undegraded":"2023-10-09T04:21:10.569169+0000","last_fullsized":"2023-10-09T04:21:10.569169+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":118,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T04:21:09.581444+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:21:09.581444+0000","last_peered":"2023-10-09T04:21:09.581444+0000","last_clean":"2023-10-09T04:21:09.581444+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:21:09.581444+0000","last_undegraded":"2023-10-09T04:21:09.581444+0000","last_fullsized":"2023-10-09T04:21:09.581444+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1790115864,"num_objects":27316,"num_object_clones":0,"num_object_copies":81948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":650,"num_objects_unfound":0,"num_objects_dirty":27316,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170132,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22013,"num_bytes_recovered":1442578456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6091587584,"data_stored":6091571296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91811,"ondisk_log_size":91811,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201626,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1839188,"kb_used_data":1249420,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":91905964,"statfs":{"total":95995035648,"available":94111707136,"internally_reserved":0,"allocated":1279406080,"data_stored":1278371059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":9.6059999999999999,"5min":3.8919999999999999,"15min":3.5350000000000001},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":148.65899999999999,"5min":148.65899999999999,"15min":148.65899999999999},"last":0.82399999999999995},{"interface":"front","average":{"1min":10.054,"5min":3.8660000000000001,"15min":3.5179999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":148.642,"5min":148.642,"15min":148.642},"last":3.0859999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:21:08 2023","interfaces":[{"interface":"back","average":{"1min":21.657,"5min":6.6159999999999997,"15min":5.6079999999999997},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23200000000000001},"max":{"1min":224.215,"5min":224.215,"15min":245.28700000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":19.774000000000001,"5min":6.4779999999999998,"15min":5.4169999999999998},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":223.928,"5min":223.928,"15min":244.626},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.698,"15min":2.1240000000000001},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.2270000000000003,"5min":14.076000000000001,"15min":76.774000000000001},"last":2.6320000000000001},{"interface":"front","average":{"1min":1.5,"5min":2.528,"15min":2.8540000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":6.5460000000000003,"5min":15.597,"15min":99.372},"last":2.6240000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.712,"15min":2.5590000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.0250000000000004,"5min":8.8550000000000004,"15min":99.134},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.133,"5min":2.1539999999999999,"15min":2.8260000000000001},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":7.0369999999999999,"5min":8.0489999999999995,"15min":99.117999999999995},"last":0.378}]},{"osd":4,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.96499999999999997,"15min":0.85799999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.1799999999999997,"5min":8.9359999999999999,"15min":16.195},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.236,"5min":1.714,"15min":1.1539999999999999},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26000000000000001},"max":{"1min":4.4500000000000002,"5min":12.566000000000001,"15min":12.566000000000001},"last":2.7970000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":1.883,"15min":1.4810000000000001},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":13.122999999999999,"5min":13.122999999999999,"15min":120.066},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.3700000000000001,"15min":1.359},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8360000000000003,"5min":9,"15min":120.09699999999999},"last":0.312}]},{"osd":6,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.667,"15min":1.1140000000000001},"min":{"1min":0.34300000000000003,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":7.3310000000000004,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.149,"5min":1.069,"15min":0.92000000000000004},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.7060000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.39200000000000002}]}]},{"osd":6,"up_from":0,"seq":201863463263,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089571,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1820096,"kb_used_data":1199864,"kb_used_omap":9,"kb_used_meta":620214,"kb_avail":91925056,"statfs":{"total":95995035648,"available":94131257344,"internally_reserved":0,"allocated":1228660736,"data_stored":1227118525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":635100137},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,1,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1589999999999998,"5min":3.968,"15min":4.1699999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17199999999999999},"max":{"1min":12.067,"5min":98.257999999999996,"15min":98.257999999999996},"last":1.006},{"interface":"front","average":{"1min":3.1160000000000001,"5min":6.4640000000000004,"15min":5.2960000000000003},"min":{"1min":0.28599999999999998,"5min":0.215,"15min":0.182},"max":{"1min":9.2260000000000009,"5min":154.654,"15min":154.654},"last":1.02}]},{"osd":2,"last update":"Mon Oct 9 04:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.1699999999999999,"15min":2.7170000000000001},"min":{"1min":0.221,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":12.791,"5min":19.564,"15min":121.54600000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.702,"5min":2.657,"15min":3.1400000000000001},"min":{"1min":0.28100000000000003,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":14.503,"5min":16.114999999999998,"15min":99.899000000000001},"last":0.32100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1389999999999998,"5min":2.3959999999999999,"15min":3.8639999999999999},"min":{"1min":0.27000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.5210000000000008,"5min":19.640000000000001,"15min":138.87299999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.7799999999999998,"5min":3.3530000000000002,"15min":4.843},"min":{"1min":0.33800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":12.241,"5min":21.859000000000002,"15min":138.49799999999999},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0640000000000001,"5min":2.3900000000000001,"15min":2.4430000000000001},"min":{"1min":0.31900000000000001,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":9.3149999999999995,"5min":16.27,"15min":99.989999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.238,"5min":1.865,"15min":2.2440000000000002},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":8.2149999999999999,"5min":9.2929999999999993,"15min":100.249},"last":0.55900000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":3.3570000000000002,"15min":2.7530000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29099999999999998},"max":{"1min":10.222,"5min":38.848999999999997,"15min":99.933999999999997},"last":0.36399999999999999},{"interface":"front","average":{"1min":3.6779999999999999,"5min":3.218,"15min":2.8450000000000002},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":21.893999999999998,"5min":39.593000000000004,"15min":100.34099999999999},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":2.1080000000000001,"15min":2.577},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":8.3529999999999998,"5min":9.3640000000000008,"15min":100.309},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.7529999999999999,"5min":1.8029999999999999,"15min":2.1859999999999999},"min":{"1min":0.38600000000000001,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":9.2949999999999999,"5min":19.440000000000001,"15min":100.143},"last":0.82299999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":11.962,"5min":4.5270000000000001,"15min":3.339},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":183.66399999999999,"5min":183.66399999999999,"15min":183.66399999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":10.627000000000001,"5min":3.8370000000000002,"15min":2.9199999999999999},"min":{"1min":0.40600000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":183.71600000000001,"5min":183.71600000000001,"15min":183.71600000000001},"last":0.62}]}]},{"osd":0,"up_from":84,"seq":360777253098,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1061248,"kb_used_data":543416,"kb_used_omap":28,"kb_used_meta":517795,"kb_avail":92683904,"statfs":{"total":95995035648,"available":94908317696,"internally_reserved":0,"allocated":556457984,"data_stored":554855383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":530222928},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:21:12 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":7.8330000000000002,"15min":5.3920000000000003},"min":{"1min":0.21199999999999999,"5min":0.19500000000000001,"15min":0.17599999999999999},"max":{"1min":6.7549999999999999,"5min":209.215,"15min":209.215},"last":5.3499999999999996},{"interface":"front","average":{"1min":9.1720000000000006,"5min":6.9870000000000001,"15min":5.5659999999999998},"min":{"1min":0.36299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":88.019000000000005,"5min":120.934,"15min":120.934},"last":8.9009999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:20:47 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":2.149,"15min":3.2570000000000001},"min":{"1min":0.216,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.131,"5min":17.466999999999999,"15min":118.54000000000001},"last":7.6660000000000004},{"interface":"front","average":{"1min":2.7530000000000001,"5min":3.1480000000000001,"15min":3.6899999999999999},"min":{"1min":0.224,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":11.050000000000001,"5min":20.262,"15min":118.727},"last":0.33300000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:21:12 2023","interfaces":[{"interface":"back","average":{"1min":9.3230000000000004,"5min":3.391,"15min":5.4740000000000002},"min":{"1min":0.32500000000000001,"5min":0.192,"15min":0.192},"max":{"1min":87.891999999999996,"5min":87.891999999999996,"15min":156.488},"last":5.4779999999999998},{"interface":"front","average":{"1min":9.0839999999999996,"5min":3.5259999999999998,"15min":5.5060000000000002},"min":{"1min":0.27300000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":88.147999999999996,"5min":88.147999999999996,"15min":156.50700000000001},"last":7.1989999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":3.0619999999999998,"15min":2.8159999999999998},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":8.7509999999999994,"5min":63.375999999999998,"15min":118.633},"last":7.2240000000000002},{"interface":"front","average":{"1min":2.3500000000000001,"5min":3.4620000000000002,"15min":3.1829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33900000000000002,"15min":0.28199999999999997},"max":{"1min":8.6950000000000003,"5min":49.607999999999997,"15min":118.913},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":3.661,"15min":4.4139999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":8.7279999999999998,"5min":62.954000000000001,"15min":182.68899999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.0880000000000001,"5min":2.6269999999999998,"15min":3.8719999999999999},"min":{"1min":0.36899999999999999,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":5.5469999999999997,"5min":63.289000000000001,"15min":182.66300000000001},"last":5.3849999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":2.5899999999999999,"15min":3.0950000000000002},"min":{"1min":0.40200000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":5.5099999999999998,"5min":63.468000000000004,"15min":137.36099999999999},"last":5.4550000000000001},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.6219999999999999,"15min":3.133},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.317},"max":{"1min":8.8160000000000007,"5min":63.447000000000003,"15min":136.88999999999999},"last":7.173}]},{"osd":7,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":2.681,"15min":3.7869999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":5.5780000000000003,"5min":63.509,"15min":185.024},"last":5.4299999999999997},{"interface":"front","average":{"1min":2.5619999999999998,"5min":2.8090000000000002,"15min":3.4660000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.32700000000000001},"max":{"1min":8.7789999999999999,"5min":63.481999999999999,"15min":184.98599999999999},"last":7.1319999999999997}]}]},{"osd":2,"up_from":94,"seq":403726926045,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1170516,"kb_used_data":598284,"kb_used_omap":81,"kb_used_meta":572014,"kb_avail":92574636,"statfs":{"total":95995035648,"available":94796427264,"internally_reserved":0,"allocated":612642816,"data_stored":611085280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":585742554},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":5.3929999999999998,"15min":4.024},"min":{"1min":0.22500000000000001,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":17.122,"5min":81.385999999999996,"15min":81.385999999999996},"last":1.1479999999999999},{"interface":"front","average":{"1min":1.228,"5min":4.617,"15min":3.9540000000000002},"min":{"1min":0.22500000000000001,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":4.1070000000000002,"5min":79.453999999999994,"15min":79.453999999999994},"last":0.252}]},{"osd":1,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":5.9340000000000002,"5min":7.2270000000000003,"15min":4.7549999999999999},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.191},"max":{"1min":72.075999999999993,"5min":158.13399999999999,"15min":158.13399999999999},"last":1.859},{"interface":"front","average":{"1min":6.4489999999999998,"5min":7.883,"15min":5.0590000000000002},"min":{"1min":0.27600000000000002,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":69.754999999999995,"5min":161.12,"15min":161.12},"last":2.2970000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":3.2909999999999999,"15min":2.7429999999999999},"min":{"1min":0.25900000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.832000000000001,"5min":58.997,"15min":58.997},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.5270000000000001,"5min":3.5760000000000001,"15min":3.0329999999999999},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.853999999999999,"5min":66.135000000000005,"15min":66.135000000000005},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":2.7549999999999999,"15min":2.2650000000000001},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":12.775,"5min":58.798000000000002,"15min":58.798000000000002},"last":0.433},{"interface":"front","average":{"1min":1.75,"5min":2.5619999999999998,"15min":2.093},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":6.5010000000000003,"5min":56.616,"15min":56.616},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":3.9660000000000002,"15min":2.7269999999999999},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":82.030000000000001,"5min":82.030000000000001,"15min":82.030000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":7.4290000000000003,"5min":4.2430000000000003,"15min":2.9119999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":82.016000000000005,"5min":82.016000000000005,"15min":82.016000000000005},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.6949999999999998,"15min":2.5579999999999998},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":4.0129999999999999,"5min":56.561,"15min":114.14},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.629,"15min":2.6469999999999998},"min":{"1min":0.40999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2110000000000003,"5min":56.460999999999999,"15min":114.123},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:20:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7409999999999997,"5min":4.0700000000000003,"15min":3.1989999999999998},"min":{"1min":0.42599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":101.675,"5min":101.675,"15min":117.32599999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":8.2639999999999993,"5min":4.1589999999999998,"15min":3.2639999999999998},"min":{"1min":0.46100000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":101.72499999999999,"5min":101.72499999999999,"15min":117.28700000000001},"last":0.44700000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762519,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1637292,"kb_used_data":865636,"kb_used_omap":7,"kb_used_meta":771640,"kb_avail":92107860,"statfs":{"total":95995035648,"available":94318448640,"internally_reserved":0,"allocated":886411264,"data_stored":884886579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":790159976},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":3.411,"15min":4.8719999999999999},"min":{"1min":0.26700000000000002,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":10.539,"5min":78.771000000000001,"15min":125.765},"last":1.345},{"interface":"front","average":{"1min":4.6210000000000004,"5min":4.4409999999999998,"15min":5.3920000000000003},"min":{"1min":0.254,"5min":0.23799999999999999,"15min":0.193},"max":{"1min":22.446999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":1.6850000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":5.5780000000000003,"15min":5.032},"min":{"1min":0.29999999999999999,"5min":0.222,"15min":0.20499999999999999},"max":{"1min":12.792999999999999,"5min":103.143,"15min":186.21299999999999},"last":1.361},{"interface":"front","average":{"1min":3.7850000000000001,"5min":5.4119999999999999,"15min":5.2229999999999999},"min":{"1min":0.33000000000000002,"5min":0.221,"15min":0.193},"max":{"1min":12.69,"5min":105.19,"15min":191.78},"last":8.9320000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":3.4199999999999999,"15min":3.5299999999999998},"min":{"1min":0.28799999999999998,"5min":0.187,"15min":0.187},"max":{"1min":5.4500000000000002,"5min":127.599,"15min":127.599},"last":4.7859999999999996},{"interface":"front","average":{"1min":3.085,"5min":5.4589999999999996,"15min":4.7229999999999999},"min":{"1min":0.28699999999999998,"5min":0.221,"15min":0.214},"max":{"1min":8.0239999999999991,"5min":129.98599999999999,"15min":129.98599999999999},"last":7.7999999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.9239999999999999,"15min":2.3460000000000001},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.798,"5min":10.013999999999999,"15min":37.652000000000001},"last":4.3550000000000004},{"interface":"front","average":{"1min":2.6840000000000002,"5min":2.097,"15min":2.2919999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.26900000000000002},"max":{"1min":8.6080000000000005,"5min":9.7170000000000005,"15min":37.442},"last":1.6120000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.5169999999999999,"15min":2.3399999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5679999999999996,"5min":12.144,"15min":17.901},"last":1.5669999999999999},{"interface":"front","average":{"1min":2.3340000000000001,"5min":1.613,"15min":1.996},"min":{"1min":0.38400000000000001,"5min":0.248,"15min":0.248},"max":{"1min":8.4670000000000005,"5min":8.4670000000000005,"15min":18.068000000000001},"last":4.7510000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.48,"15min":1.8799999999999999},"min":{"1min":0.41699999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.5300000000000002,"5min":5.758,"15min":14.976000000000001},"last":1.371},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.1259999999999999,"15min":2.27},"min":{"1min":0.39200000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8479999999999999,"5min":10.071,"15min":15.622},"last":1.792}]},{"osd":7,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.5590000000000002,"15min":2.4209999999999998},"min":{"1min":0.441,"5min":0.36199999999999999,"15min":0.34399999999999997},"max":{"1min":6.8239999999999998,"5min":28.184000000000001,"15min":28.184000000000001},"last":7.7560000000000002},{"interface":"front","average":{"1min":2.7400000000000002,"5min":2.758,"15min":2.4420000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.6370000000000005,"5min":26.648,"15min":26.648},"last":0.70799999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986535,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951492,"kb_used_data":646012,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92793660,"statfs":{"total":95995035648,"available":95020707840,"internally_reserved":0,"allocated":661516288,"data_stored":660496682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":6.7859999999999996,"5min":9.9049999999999994,"15min":5.9029999999999996},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":72.759,"5min":154.62100000000001,"15min":154.62100000000001},"last":1.395},{"interface":"front","average":{"1min":6.7279999999999998,"5min":10.082000000000001,"15min":5.9210000000000003},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":72.704999999999998,"5min":154.37299999999999,"15min":154.37299999999999},"last":3.1960000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:21:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":8.2810000000000006,"15min":5.2080000000000002},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":8.1539999999999999,"5min":135.25399999999999,"15min":135.25399999999999},"last":1.6870000000000001},{"interface":"front","average":{"1min":2.282,"5min":8.1039999999999992,"15min":5.1029999999999998},"min":{"1min":0.25700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":8.5749999999999993,"5min":134.44300000000001,"15min":134.44300000000001},"last":0.315}]},{"osd":2,"last update":"Mon Oct 9 04:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.8180000000000001,"15min":2.1560000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":8.202,"5min":8.7400000000000002,"15min":123.598},"last":1.6719999999999999},{"interface":"front","average":{"1min":2.5299999999999998,"5min":1.8440000000000001,"15min":2.1160000000000001},"min":{"1min":0.33400000000000002,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":11.064,"5min":11.064,"15min":123.619},"last":1.659}]},{"osd":3,"last update":"Mon Oct 9 04:21:05 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":2.0299999999999998,"15min":2.3279999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.2329999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.339,"15min":2.5649999999999999},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":11.18,"5min":11.18,"15min":127.90000000000001},"last":1.357}]},{"osd":5,"last update":"Mon Oct 9 04:21:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.161,"15min":1.645},"min":{"1min":0.27200000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.4249999999999998,"5min":13.599,"15min":111.809},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.5029999999999999,"15min":1.879},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.25},"max":{"1min":8.0589999999999993,"5min":9.1530000000000005,"15min":112.002},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:21:05 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.331,"15min":1.454},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":8.1829999999999998,"5min":8.1829999999999998,"15min":76.307000000000002},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.867,"5min":1.1899999999999999,"15min":1.3240000000000001},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6340000000000003,"5min":8.6340000000000003,"15min":78.421999999999997},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.3149999999999999,"15min":1.079},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7789999999999999,"5min":8.7789999999999999,"15min":34.039000000000001},"last":0.375},{"interface":"front","average":{"1min":2.46,"5min":1.508,"15min":1.194},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.5430000000000001,"5min":10.384,"15min":33.433999999999997},"last":0.38500000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724899,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1549888,"kb_used_data":964152,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":92195264,"statfs":{"total":95995035648,"available":94407950336,"internally_reserved":0,"allocated":987291648,"data_stored":986272548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":3.25,"15min":5.1600000000000001},"min":{"1min":0.31,"5min":0.193,"15min":0.193},"max":{"1min":9.3529999999999998,"5min":111.312,"15min":151.23599999999999},"last":0.98299999999999998},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.552,"15min":5.5739999999999998},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":5.8890000000000002,"5min":111.096,"15min":151.566},"last":0.45800000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:21:01 2023","interfaces":[{"interface":"back","average":{"1min":4.9359999999999999,"5min":4.5049999999999999,"15min":4.2999999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":48.781999999999996,"5min":111.194,"15min":111.194},"last":0.66700000000000004},{"interface":"front","average":{"1min":4.077,"5min":4.6539999999999999,"15min":4.6440000000000001},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":48.868000000000002,"5min":111.247,"15min":112.226},"last":1.409}]},{"osd":2,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.4079999999999999,"15min":3.2759999999999998},"min":{"1min":0.28899999999999998,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":5.8579999999999997,"5min":11.324,"15min":106.508},"last":0.497},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.8979999999999999,"15min":3.0179999999999998},"min":{"1min":0.28199999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":5.9340000000000002,"5min":8.5280000000000005,"15min":106.462},"last":1.2829999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:21:01 2023","interfaces":[{"interface":"back","average":{"1min":4.3710000000000004,"5min":2.5739999999999998,"15min":2.5920000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":41,"5min":41,"15min":91.834999999999994},"last":0.55000000000000004},{"interface":"front","average":{"1min":4.5919999999999996,"5min":2.7999999999999998,"15min":2.702},"min":{"1min":0.33500000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":40.988,"5min":40.988,"15min":91.802000000000007},"last":0.623}]},{"osd":4,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.27,"15min":1.0029999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.4370000000000003,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.897,"5min":1.2829999999999999,"15min":1.0189999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.5890000000000004,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.372}]},{"osd":6,"last update":"Mon Oct 9 04:21:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.214,"15min":1.4099999999999999},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":6.2519999999999998,"5min":12.068,"15min":123.239},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.204,"15min":1.4830000000000001},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":3.5369999999999999,"5min":9.7720000000000002,"15min":123.014},"last":0.47799999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":1.3899999999999999,"15min":1.109},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3800000000000008,"5min":12.096,"15min":12.096},"last":1.536},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.5740000000000001,"15min":1.194},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":5.7309999999999999,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":1.4419999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":541065216,"data_stored":541065216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1192562688,"data_stored":1192558616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":600047616,"data_stored":600047616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":874577920,"data_stored":874577920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":648548352,"data_stored":648544280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":971771904,"data_stored":971767832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1262555136,"data_stored":1262551064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:21:24.804 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:24.916 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:24 smithi163 ceph-mon[221067]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:24.916 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:24 smithi163 ceph-mon[221067]: pgmap v995: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 650/79551 objects misplaced (0.817%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:24.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:24 smithi163 ceph-mon[213841]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:24.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:24 smithi163 ceph-mon[213841]: pgmap v995: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 650/79551 objects misplaced (0.817%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:24.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:25.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:25.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:25.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:25.189 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:28476 9.6 e315/306 9:6fdc7cdd:::benchmark_data_smithi163_464650_object28475:head v 315'1827, mlcod=314'1823)", 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:27.803246+0000", 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 177.37235965900001, 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.10015397600000001, 2023-10-09T04:21:25.190 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803246+0000", 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803246+0000", 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803246+0000", 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803282+0000", 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6717000000000001e-05 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803291+0000", 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2509999999999994e-06 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803298+0000", 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0659999999999998e-06 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803358+0000", 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9929000000000002e-05 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.803382+0000", 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4093000000000001e-05 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.903371+0000", 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.099989638000000006 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:27.903400+0000", 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8946999999999999e-05 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(9.b 321/316 [PushOp(9:de3a6d3f:::benchmark_data_smithi163_464650_object21077:head, version: 314'1333, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:de3a6d3f:::benchmark_data_smithi163_464650_object21077:head@314'1333, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:42.991561+0000", 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 162.18404535799999, 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.082487610000000003, 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991561+0000", 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991561+0000", 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991560+0000", 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991585+0000", 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5443e-05 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991598+0000", 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2531000000000001e-05 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991608+0000", 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0601999999999999e-05 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991658+0000", 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9310000000000001e-05 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:42.991666+0000", 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0600000000000008e-06 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:43.074048+0000", 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.082382518000000002 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:49091 9.b e324/323 9:d8f37e79:::benchmark_data_smithi163_464650_object49090:head v 324'3030, mlcod=0'0)", 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:18:47.951327+0000", 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 157.224279075, 2023-10-09T04:21:25.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039282368999999998, 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.951327+0000", 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.951327+0000", 2023-10-09T04:21:25.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.951321+0000", 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999938 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.951352+0000", 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0827999999999997e-05 2023-10-09T04:21:25.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.985068+0000", 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.033716304000000002 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.985079+0000", 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.134e-05 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.985221+0000", 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000141879 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.985246+0000", 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4731000000000001e-05 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.990575+0000", 2023-10-09T04:21:25.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053294029999999999 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:18:47.990609+0000", 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3892e-05 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:69784 9.0 e327/316 9:0e558d12:::benchmark_data_smithi163_464650_object69783:head v 327'4287, mlcod=314'1401)", 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:06.935093+0000", 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 138.24051295199999, 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.046661198000000001, 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935093+0000", 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935093+0000", 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935093+0000", 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:25.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935117+0000", 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4535000000000001e-05 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935129+0000", 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1727e-05 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935137+0000", 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6009999999999995e-06 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935193+0000", 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5954e-05 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.935219+0000", 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5871999999999999e-05 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.981718+0000", 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.046499071000000003 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.981754+0000", 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5871999999999998e-05 2023-10-09T04:21:25.210 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:69789 9.2 e327/306 9:4d7abda8:::benchmark_data_smithi163_464650_object69788:head v 327'4362, mlcod=326'4360)", 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:06.943351+0000", 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 138.23225478200001, 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.041994797, 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.211 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.943351+0000", 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.943351+0000", 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.943350+0000", 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.943376+0000", 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5749e-05 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.981569+0000", 2023-10-09T04:21:25.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.038193344999999997 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.981579+0000", 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6979999999999993e-06 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.982063+0000", 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00048359199999999997 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.982083+0000", 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0404e-05 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.985297+0000", 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003214064 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:06.985346+0000", 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9035000000000001e-05 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:88484 9.0 e334/329 9:0a168f03:::benchmark_data_smithi163_464650_object88483:head v 334'5433, mlcod=334'5430)", 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:29.842201+0000", 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 115.33340513100001, 2023-10-09T04:21:25.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034267233000000001, 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.842201+0000", 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.842201+0000", 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.842200+0000", 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.842226+0000", 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6457000000000001e-05 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.842244+0000", 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7368999999999999e-05 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.842252+0000", 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0970000000000006e-06 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.848217+0000", 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0059657069999999998 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.848240+0000", 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.281e-05 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.876421+0000", 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028180752999999999 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:29.876468+0000", 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7083000000000003e-05 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:117546 9.b e355/323 9:da36b125:::benchmark_data_smithi163_464650_object18820:head v 355'7351, mlcod=355'7348)", 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:13.110744+0000", 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 72.064862039000005, 2023-10-09T04:21:25.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.050704828, 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110744+0000", 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110744+0000", 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110743+0000", 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110750+0000", 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9720000000000003e-06 2023-10-09T04:21:25.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110760+0000", 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.028e-05 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110768+0000", 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5050000000000001e-06 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110805+0000", 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7459999999999997e-05 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.110824+0000", 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8882999999999999e-05 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.161412+0000", 2023-10-09T04:21:25.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.050587904000000003 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.161449+0000", 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6649000000000001e-05 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.226 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:117543 9.7 e355/349 9:e87899e6:::benchmark_data_smithi163_464650_object18817:head v 355'7523, mlcod=355'7521)", 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:13.114210+0000", 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 72.061396130000006, 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.051471990000000002, 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114210+0000", 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114210+0000", 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114209+0000", 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114217+0000", 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4800000000000001e-06 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.154744+0000", 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.040526230000000003 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.154759+0000", 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5274999999999999e-05 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.154981+0000", 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000221807 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.155004+0000", 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3467000000000001e-05 2023-10-09T04:21:25.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.165646+0000", 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010642097999999999 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.165682+0000", 2023-10-09T04:21:25.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5639000000000002e-05 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:117547 9.b e355/323 9:debb6fa3:::benchmark_data_smithi163_464650_object18821:head v 355'7352, mlcod=355'7349)", 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:13.114918+0000", 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 72.060688075000002, 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.046856633000000002, 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.233 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114918+0000", 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114918+0000", 2023-10-09T04:21:25.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114917+0000", 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.114925+0000", 2023-10-09T04:21:25.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4399999999999999e-06 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.154745+0000", 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039819861999999998 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.154759+0000", 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3875e-05 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.158405+0000", 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003646411 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.158425+0000", 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9660000000000002e-05 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.161735+0000", 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033100740000000001 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.161775+0000", 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9913999999999998e-05 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:121683 9.6 e355/306 9:67c6d5a1:::benchmark_data_smithi163_464650_object22957:head v 355'7627, mlcod=355'7625)", 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:18.538354+0000", 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 66.637252371000002, 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.044389407999999998, 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.538354+0000", 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.538354+0000", 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.538352+0000", 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.538365+0000", 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2301e-05 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.538386+0000", 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1537000000000001e-05 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.538396+0000", 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5810000000000003e-06 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541942+0000", 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035465000000000002 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541959+0000", 2023-10-09T04:21:25.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6971999999999999e-05 2023-10-09T04:21:25.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.582709+0000", 2023-10-09T04:21:25.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.040749484000000002 2023-10-09T04:21:25.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.582743+0000", 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4400000000000003e-05 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:121685 9.6 e355/306 9:696fa1e0:::benchmark_data_smithi163_464650_object22959:head v 355'7628, mlcod=355'7625)", 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:18.541754+0000", 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 66.633851703999994, 2023-10-09T04:21:25.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.041045267000000003, 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541754+0000", 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541754+0000", 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541754+0000", 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541761+0000", 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1500000000000002e-06 2023-10-09T04:21:25.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541771+0000", 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0733e-05 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.541779+0000", 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8369999999999997e-06 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.572796+0000", 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031017115000000001 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.572825+0000", 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9045999999999999e-05 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.582750+0000", 2023-10-09T04:21:25.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0099241889999999999 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:18.582800+0000", 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9947999999999997e-05 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:124242 9.0 e356/329 9:09d6ecf1:::benchmark_data_smithi163_464650_object25516:head v 356'7741, mlcod=356'7738)", 2023-10-09T04:21:25.251 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:22.395394+0000", 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 62.780211889999997, 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.042884670999999999, 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.395394+0000", 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.395394+0000", 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.395393+0000", 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.395404+0000", 2023-10-09T04:21:25.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0779e-05 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.395426+0000", 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2861e-05 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.395436+0000", 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8400000000000007e-06 2023-10-09T04:21:25.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.398786+0000", 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033497029999999999 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.398812+0000", 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6135e-05 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.438244+0000", 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.039431569999999999 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.438279+0000", 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5135000000000002e-05 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:124252 9.0 e356/329 9:07d6a725:::benchmark_data_smithi163_464650_object25526:head v 356'7742, mlcod=356'7739)", 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:22.399526+0000", 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 62.776080487999998, 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.038780493999999999, 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.399526+0000", 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.399526+0000", 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.399525+0000", 2023-10-09T04:21:25.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.399533+0000", 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6139999999999992e-06 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.399546+0000", 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2974e-05 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.399555+0000", 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8789999999999992e-06 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.401569+0000", 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020138679999999998 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.401589+0000", 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0611999999999998e-05 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.438285+0000", 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.036695759000000001 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:22.438306+0000", 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0758e-05 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:152025 9.0 e361/329 9:06b23e8b:::benchmark_data_smithi163_464650_object53299:head v 361'9423, mlcod=361'9421)", 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:57.432423+0000", 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 27.743183131999999, 2023-10-09T04:21:25.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.080940011000000006, 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432423+0000", 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432423+0000", 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432422+0000", 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432430+0000", 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2530000000000004e-06 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432441+0000", 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1267000000000001e-05 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432449+0000", 2023-10-09T04:21:25.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3270000000000003e-06 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432518+0000", 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8862999999999994e-05 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.432537+0000", 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9335999999999999e-05 2023-10-09T04:21:25.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.513305+0000", 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.080767748 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.513363+0000", 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8213e-05 2023-10-09T04:21:25.267 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:152029 9.b e361/323 9:d9527590:::benchmark_data_smithi163_464650_object53303:head v 361'9467, mlcod=361'9465)", 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:57.435091+0000", 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 27.740515046999999, 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.081355134999999995, 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.435091+0000", 2023-10-09T04:21:25.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.435091+0000", 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.435090+0000", 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.435098+0000", 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0839999999999992e-06 2023-10-09T04:21:25.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507533+0000", 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.07243455 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507542+0000", 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.935e-06 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507733+0000", 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000191027 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507751+0000", 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8082999999999999e-05 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.516412+0000", 2023-10-09T04:21:25.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086616309999999995 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.516446+0000", 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3664999999999997e-05 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:152026 9.6 e361/306 9:6c998bc7:::benchmark_data_smithi163_464650_object53300:head v 361'9520, mlcod=361'9518)", 2023-10-09T04:21:25.273 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:57.437408+0000", 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 27.738197845999998, 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.080580526, 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.437408+0000", 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.437408+0000", 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.437407+0000", 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.437416+0000", 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1509999999999993e-06 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507510+0000", 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.070093572000000007 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507535+0000", 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5386000000000001e-05 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507591+0000", 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5733999999999999e-05 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.507614+0000", 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3354e-05 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.517963+0000", 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010348593999999999 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:57.517989+0000", 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5811000000000001e-05 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:171787 9.2 e365/359 9:4ed3335c:::benchmark_data_smithi163_464650_object73061:head v 365'10814, mlcod=365'10810)", 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:22.542251+0000", 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.6333554139999999, 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.054118925999999998, 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542251+0000", 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542251+0000", 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542250+0000", 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542259+0000", 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7720000000000006e-06 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542272+0000", 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3079e-05 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542281+0000", 2023-10-09T04:21:25.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3289999999999994e-06 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542361+0000", 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9740999999999994e-05 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.542381+0000", 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0798e-05 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596333+0000", 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.053951746000000002 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596370+0000", 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6463e-05 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:171791 9.b e365/323 9:d830c7b9:::benchmark_data_smithi163_464650_object73065:head v 365'10744, mlcod=365'10742)", 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:22.543808+0000", 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.6317978220000002, 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.058390958999999999, 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543808+0000", 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543808+0000", 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543807+0000", 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543815+0000", 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4050000000000003e-06 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543827+0000", 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1839e-05 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543835+0000", 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8320000000000006e-06 2023-10-09T04:21:25.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543899+0000", 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4287000000000003e-05 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.543919+0000", 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.031e-05 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.602157+0000", 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.058237573000000001 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.602199+0000", 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2457999999999999e-05 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.288 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:171792 9.6 e365/306 9:65f43d67:::benchmark_data_smithi163_464650_object73066:head v 365'10746, mlcod=365'10744)", 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:22.544596+0000", 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.6310095489999998, 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.060634189999999998, 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.544596+0000", 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.544596+0000", 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.544596+0000", 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.544603+0000", 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1169999999999998e-06 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596305+0000", 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.051701849000000001 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596314+0000", 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0929999999999992e-06 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596605+0000", 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029154000000000002 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596624+0000", 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9143999999999998e-05 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.605193+0000", 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085680930000000006 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.605231+0000", 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8071000000000002e-05 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:171805 9.b e365/323 9:d65e732e:::benchmark_data_smithi163_464650_object73079:head v 365'10745, mlcod=365'10742)", 2023-10-09T04:21:25.293 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:22.563470+0000", 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 2.6121363099999999, 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.038891544, 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.563470+0000", 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.563470+0000", 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.563469+0000", 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.563478+0000", 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5680000000000005e-06 2023-10-09T04:21:25.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596294+0000", 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032815957 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.596307+0000", 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2877999999999999e-05 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.599414+0000", 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031072790000000001 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.599438+0000", 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3830999999999999e-05 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.602311+0000", 2023-10-09T04:21:25.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028729530000000001 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:22.602361+0000", 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0112999999999999e-05 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:21:25.298 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:21:25.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:24 smithi181 ceph-mon[196979]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:25.299 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:24 smithi181 ceph-mon[196979]: pgmap v995: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 650/79551 objects misplaced (0.817%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:25.299 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:25.300 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:25.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:25.370 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:25.415 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:25.449 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:25.483 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:25.522 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:25.556 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:21:25.558 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:21:26.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3288226920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:26.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:21:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3288226920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3288226920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:26.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:26.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:26.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:26.389 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:26.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:21:26.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:26.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:26.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:26.667 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:26.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:26.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:26.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[213841]: failed to encode map e366 with expected crc 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[213841]: osdmap e366: 8 total, 8 up, 7 in 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[213841]: failed to encode map e366 with expected crc 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[213841]: failed to encode map e366 with expected crc 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[213841]: pgmap v997: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 650/78711 objects misplaced (0.826%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[221067]: failed to encode map e366 with expected crc 2023-10-09T04:21:26.782 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[221067]: osdmap e366: 8 total, 8 up, 7 in 2023-10-09T04:21:26.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[221067]: failed to encode map e366 with expected crc 2023-10-09T04:21:26.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[221067]: failed to encode map e366 with expected crc 2023-10-09T04:21:26.783 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:26 smithi163 ceph-mon[221067]: pgmap v997: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 650/78711 objects misplaced (0.826%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:21:26.881 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:27.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:26 smithi181 ceph-mon[196979]: failed to encode map e366 with expected crc 2023-10-09T04:21:27.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:26 smithi181 ceph-mon[196979]: osdmap e366: 8 total, 8 up, 7 in 2023-10-09T04:21:27.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:26 smithi181 ceph-mon[196979]: failed to encode map e366 with expected crc 2023-10-09T04:21:27.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:26 smithi181 ceph-mon[196979]: failed to encode map e366 with expected crc 2023-10-09T04:21:27.025 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:26 smithi181 ceph-mon[196979]: pgmap v997: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 650/78711 objects misplaced (0.826%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:21:27.025 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:27.139 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:27.302 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:27.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:21:28.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:27 smithi181 ceph-mon[196979]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:27 smithi181 ceph-mon[196979]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.076 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:27 smithi181 ceph-mon[196979]: osdmap e367: 8 total, 8 up, 7 in 2023-10-09T04:21:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:27 smithi181 ceph-mon[196979]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.077 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:27 smithi181 ceph-mon[196979]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:28.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[213841]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[213841]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[213841]: osdmap e367: 8 total, 8 up, 7 in 2023-10-09T04:21:28.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[213841]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[213841]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[221067]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[221067]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[221067]: osdmap e367: 8 total, 8 up, 7 in 2023-10-09T04:21:28.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[221067]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:27 smithi163 ceph-mon[221067]: failed to encode map e367 with expected crc 2023-10-09T04:21:28.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:28.228 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:28.323 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:28.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:28.428 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:28.469 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:28.898 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:21:29.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:28 smithi163 ceph-mon[213841]: pgmap v999: 119 pgs: 119 active+clean; 1.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 782 op/s; 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:29.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:28 smithi163 ceph-mon[221067]: pgmap v999: 119 pgs: 119 active+clean; 1.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 782 op/s; 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:29.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:29.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:28 smithi181 ceph-mon[196979]: pgmap v999: 119 pgs: 119 active+clean; 1.4 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 782 op/s; 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:21:29.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:29.446 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:29.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:29.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:29.706 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:29.741 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:29.787 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:29.870 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:29.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:29.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:30.072 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:30.245 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:21:30.245 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:21:30.254 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:30.334 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:30.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:30.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:30.505 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:30.505 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:30.505 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:30 smithi163 ceph-mon[221067]: pgmap v1000: 119 pgs: 119 active+clean; 1.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:21:30.505 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:30.505 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:30.506 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:30 smithi163 ceph-mon[213841]: pgmap v1000: 119 pgs: 119 active+clean; 1.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:21:30.507 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:30.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:30.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:30.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:30 smithi181 ceph-mon[196979]: pgmap v1000: 119 pgs: 119 active+clean; 1.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:21:30.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:21:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:21:30] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-10-09T04:21:31.452 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:31.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:31.555 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:31.581 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:21:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2961631084' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:31.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2961631084' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:31.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:31.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2961631084' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:31.601 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:31.605 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:31.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:31.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:31.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:31.794 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:31.842 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:31.946 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:31.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:31.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:32.055 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:32.082 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:32.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:32.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:32.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:32.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:32.307 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[213841]: failed to encode map e368 with expected crc 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[213841]: osdmap e368: 8 total, 8 up, 7 in 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[213841]: failed to encode map e368 with expected crc 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[213841]: pgmap v1002: 119 pgs: 119 active+clean; 1.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:32.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[221067]: failed to encode map e368 with expected crc 2023-10-09T04:21:32.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:32.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[221067]: osdmap e368: 8 total, 8 up, 7 in 2023-10-09T04:21:32.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[221067]: failed to encode map e368 with expected crc 2023-10-09T04:21:32.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:32 smithi163 ceph-mon[221067]: pgmap v1002: 119 pgs: 119 active+clean; 1.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:21:32.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:32 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:32.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:32 smithi181 ceph-mon[196979]: failed to encode map e368 with expected crc 2023-10-09T04:21:32.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:32.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:32 smithi181 ceph-mon[196979]: osdmap e368: 8 total, 8 up, 7 in 2023-10-09T04:21:32.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:32 smithi181 ceph-mon[196979]: failed to encode map e368 with expected crc 2023-10-09T04:21:32.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:32 smithi181 ceph-mon[196979]: pgmap v1002: 119 pgs: 119 active+clean; 1.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:21:32.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:32.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:33.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:33.043 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:33.084 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:33.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:33.190 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:33.246 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:33.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:33.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:33.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:33.485 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:33.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:33.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:33.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:33.666 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:33.726 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:33.826 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:33.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:33.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:33.992 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:34.032 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:21:34.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:34.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:34.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:34.351 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:34.381 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:34.452 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:34.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:34.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:34.636 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:34.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:34.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:34.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:34.812 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:21:34.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:34.918 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:21:34.940 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:21:35.032 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:35.089 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:35.139 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:35.176 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:35.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[221067]: pgmap v1003: 119 pgs: 119 active+clean; 1.2 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 903 op/s; 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:21:35.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:35.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:35.199 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:35.199 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[213841]: pgmap v1003: 119 pgs: 119 active+clean; 1.2 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 903 op/s; 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:21:35.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:35.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:35.200 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:35.220 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:34 smithi181 ceph-mon[196979]: pgmap v1003: 119 pgs: 119 active+clean; 1.2 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 903 op/s; 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:21:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:35.269 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:35.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:35.353 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:35.640 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:35.640 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.640 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.640 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.640 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.640 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.641 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.641 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:21:35.641 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:21:36.060 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:36.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:36.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:36.221 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:35 smithi181 ceph-mon[196979]: from='client.45965 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:35 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/282286814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:36.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:36.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:36.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:35 smithi163 ceph-mon[221067]: from='client.45965 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/282286814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:35 smithi163 ceph-mon[213841]: from='client.45965 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:21:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:35 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/282286814' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:36.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:36.427 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:36.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:36.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:36.785 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1003,"stamp":"2023-10-09T04:21:33.877780+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1244676281,"num_objects":19007,"num_object_clones":0,"num_object_copies":57021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19007,"num_whiteouts":0,"num_read":64,"num_read_kb":119,"num_write":178558,"num_write_kb":6318614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22380,"num_bytes_recovered":1462806241,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96790,"ondisk_log_size":96790,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8367292,"kb_used_data":4394628,"kb_used_omap":126,"kb_used_meta":3972417,"kb_avail":647848772,"statfs":{"total":671965249536,"available":663397142528,"internally_reserved":0,"allocated":4500099072,"data_stored":4490684195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":4067755643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-473825280,"num_objects":-7230,"num_object_clones":0,"num_object_copies":-21690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-650,"num_objects_unfound":0,"num_objects_dirty":-7230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7230,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":300,"num_bytes_recovered":19660800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003647"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":491,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780397+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:21:25.780397+0000","last_peered":"2023-10-09T04:21:25.780397+0000","last_clean":"2023-10-09T04:21:25.780397+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:21:25.780397+0000","last_undegraded":"2023-10-09T04:21:25.780397+0000","last_fullsized":"2023-10-09T04:21:25.780397+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":111,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893509+0000","last_change":"2023-10-09T04:15:25.431910+0000","last_active":"2023-10-09T04:21:25.893509+0000","last_peered":"2023-10-09T04:21:25.893509+0000","last_clean":"2023-10-09T04:21:25.893509+0000","last_became_active":"2023-10-09T04:15:25.431438+0000","last_became_peered":"2023-10-09T04:15:25.431438+0000","last_unstale":"2023-10-09T04:21:25.893509+0000","last_undegraded":"2023-10-09T04:21:25.893509+0000","last_fullsized":"2023-10-09T04:21:25.893509+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:53.552234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":343,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.823829+0000","last_change":"2023-10-09T04:18:15.731961+0000","last_active":"2023-10-09T04:21:26.823829+0000","last_peered":"2023-10-09T04:21:26.823829+0000","last_clean":"2023-10-09T04:21:26.823829+0000","last_became_active":"2023-10-09T04:18:15.731637+0000","last_became_peered":"2023-10-09T04:18:15.731637+0000","last_unstale":"2023-10-09T04:21:26.823829+0000","last_undegraded":"2023-10-09T04:21:26.823829+0000","last_fullsized":"2023-10-09T04:21:26.823829+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:35.699985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":484,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780376+0000","last_change":"2023-10-09T04:19:10.416036+0000","last_active":"2023-10-09T04:21:25.780376+0000","last_peered":"2023-10-09T04:21:25.780376+0000","last_clean":"2023-10-09T04:21:25.780376+0000","last_became_active":"2023-10-09T04:18:04.679399+0000","last_became_peered":"2023-10-09T04:18:04.679399+0000","last_unstale":"2023-10-09T04:21:25.780376+0000","last_undegraded":"2023-10-09T04:21:25.780376+0000","last_fullsized":"2023-10-09T04:21:25.780376+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":358,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822220+0000","last_change":"2023-10-09T04:18:15.727589+0000","last_active":"2023-10-09T04:21:26.822220+0000","last_peered":"2023-10-09T04:21:26.822220+0000","last_clean":"2023-10-09T04:21:26.822220+0000","last_became_active":"2023-10-09T04:18:15.727207+0000","last_became_peered":"2023-10-09T04:18:15.727207+0000","last_unstale":"2023-10-09T04:21:26.822220+0000","last_undegraded":"2023-10-09T04:21:26.822220+0000","last_fullsized":"2023-10-09T04:21:26.822220+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:30.211925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":346,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787783+0000","last_change":"2023-10-09T04:20:01.981129+0000","last_active":"2023-10-09T04:21:26.787783+0000","last_peered":"2023-10-09T04:21:26.787783+0000","last_clean":"2023-10-09T04:21:26.787783+0000","last_became_active":"2023-10-09T04:20:01.980750+0000","last_became_peered":"2023-10-09T04:20:01.980750+0000","last_unstale":"2023-10-09T04:21:26.787783+0000","last_undegraded":"2023-10-09T04:21:26.787783+0000","last_fullsized":"2023-10-09T04:21:26.787783+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:55.165833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":491,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.892943+0000","last_change":"2023-10-09T04:18:23.768082+0000","last_active":"2023-10-09T04:21:25.892943+0000","last_peered":"2023-10-09T04:21:25.892943+0000","last_clean":"2023-10-09T04:21:25.892943+0000","last_became_active":"2023-10-09T04:18:04.683223+0000","last_became_peered":"2023-10-09T04:18:04.683223+0000","last_unstale":"2023-10-09T04:21:25.892943+0000","last_undegraded":"2023-10-09T04:21:25.892943+0000","last_fullsized":"2023-10-09T04:21:25.892943+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:23.926585+0000","scrub_duration":0.0039510450000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":344,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787828+0000","last_change":"2023-10-09T04:18:15.747197+0000","last_active":"2023-10-09T04:21:26.787828+0000","last_peered":"2023-10-09T04:21:26.787828+0000","last_clean":"2023-10-09T04:21:26.787828+0000","last_became_active":"2023-10-09T04:18:15.746904+0000","last_became_peered":"2023-10-09T04:18:15.746904+0000","last_unstale":"2023-10-09T04:21:26.787828+0000","last_undegraded":"2023-10-09T04:21:26.787828+0000","last_fullsized":"2023-10-09T04:21:26.787828+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:48.658216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":310,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780306+0000","last_change":"2023-10-09T04:18:28.824980+0000","last_active":"2023-10-09T04:21:25.780306+0000","last_peered":"2023-10-09T04:21:25.780306+0000","last_clean":"2023-10-09T04:21:25.780306+0000","last_became_active":"2023-10-09T04:18:28.824333+0000","last_became_peered":"2023-10-09T04:18:28.824333+0000","last_unstale":"2023-10-09T04:21:25.780306+0000","last_undegraded":"2023-10-09T04:21:25.780306+0000","last_fullsized":"2023-10-09T04:21:25.780306+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:03.331055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":766,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781702+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:21:25.781702+0000","last_peered":"2023-10-09T04:21:25.781702+0000","last_clean":"2023-10-09T04:21:25.781702+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:21:25.781702+0000","last_undegraded":"2023-10-09T04:21:25.781702+0000","last_fullsized":"2023-10-09T04:21:25.781702+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":345,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.880632+0000","last_change":"2023-10-09T04:18:15.737344+0000","last_active":"2023-10-09T04:21:26.880632+0000","last_peered":"2023-10-09T04:21:26.880632+0000","last_clean":"2023-10-09T04:21:26.880632+0000","last_became_active":"2023-10-09T04:18:15.737056+0000","last_became_peered":"2023-10-09T04:18:15.737056+0000","last_unstale":"2023-10-09T04:21:26.880632+0000","last_undegraded":"2023-10-09T04:21:26.880632+0000","last_fullsized":"2023-10-09T04:21:26.880632+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:26.544709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.833586+0000","last_change":"2023-10-09T04:18:28.823564+0000","last_active":"2023-10-09T04:21:25.833586+0000","last_peered":"2023-10-09T04:21:25.833586+0000","last_clean":"2023-10-09T04:21:25.833586+0000","last_became_active":"2023-10-09T04:18:28.823216+0000","last_became_peered":"2023-10-09T04:18:28.823216+0000","last_unstale":"2023-10-09T04:21:25.833586+0000","last_undegraded":"2023-10-09T04:21:25.833586+0000","last_fullsized":"2023-10-09T04:21:25.833586+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:56.770128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":493,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.898188+0000","last_change":"2023-10-09T04:18:20.363722+0000","last_active":"2023-10-09T04:21:25.898188+0000","last_peered":"2023-10-09T04:21:25.898188+0000","last_clean":"2023-10-09T04:21:25.898188+0000","last_became_active":"2023-10-09T04:18:04.682873+0000","last_became_peered":"2023-10-09T04:18:04.682873+0000","last_unstale":"2023-10-09T04:21:25.898188+0000","last_undegraded":"2023-10-09T04:21:25.898188+0000","last_fullsized":"2023-10-09T04:21:25.898188+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:34.010052+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":326,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780676+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:21:25.780676+0000","last_peered":"2023-10-09T04:21:25.780676+0000","last_clean":"2023-10-09T04:21:25.780676+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:21:25.780676+0000","last_undegraded":"2023-10-09T04:21:25.780676+0000","last_fullsized":"2023-10-09T04:21:25.780676+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780967+0000","last_change":"2023-10-09T04:18:28.839303+0000","last_active":"2023-10-09T04:21:25.780967+0000","last_peered":"2023-10-09T04:21:25.780967+0000","last_clean":"2023-10-09T04:21:25.780967+0000","last_became_active":"2023-10-09T04:18:28.838999+0000","last_became_peered":"2023-10-09T04:18:28.838999+0000","last_unstale":"2023-10-09T04:21:25.780967+0000","last_undegraded":"2023-10-09T04:21:25.780967+0000","last_fullsized":"2023-10-09T04:21:25.780967+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:56.059761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":501,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780222+0000","last_change":"2023-10-09T04:18:31.851743+0000","last_active":"2023-10-09T04:21:25.780222+0000","last_peered":"2023-10-09T04:21:25.780222+0000","last_clean":"2023-10-09T04:21:25.780222+0000","last_became_active":"2023-10-09T04:18:31.851296+0000","last_became_peered":"2023-10-09T04:18:31.851296+0000","last_unstale":"2023-10-09T04:21:25.780222+0000","last_undegraded":"2023-10-09T04:21:25.780222+0000","last_fullsized":"2023-10-09T04:21:25.780222+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:14.378661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:14.378661+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":409,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781934+0000","last_change":"2023-10-09T04:19:50.943352+0000","last_active":"2023-10-09T04:21:25.781934+0000","last_peered":"2023-10-09T04:21:25.781934+0000","last_clean":"2023-10-09T04:21:25.781934+0000","last_became_active":"2023-10-09T04:19:50.897811+0000","last_became_peered":"2023-10-09T04:19:50.897811+0000","last_unstale":"2023-10-09T04:21:25.781934+0000","last_undegraded":"2023-10-09T04:21:25.781934+0000","last_fullsized":"2023-10-09T04:21:25.781934+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:00.237368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822319+0000","last_change":"2023-10-09T04:19:27.113487+0000","last_active":"2023-10-09T04:21:26.822319+0000","last_peered":"2023-10-09T04:21:26.822319+0000","last_clean":"2023-10-09T04:21:26.822319+0000","last_became_active":"2023-10-09T04:19:27.113183+0000","last_became_peered":"2023-10-09T04:19:27.113183+0000","last_unstale":"2023-10-09T04:21:26.822319+0000","last_undegraded":"2023-10-09T04:21:26.822319+0000","last_fullsized":"2023-10-09T04:21:26.822319+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:49.432797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":510,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.824766+0000","last_change":"2023-10-09T04:18:25.748545+0000","last_active":"2023-10-09T04:21:26.824766+0000","last_peered":"2023-10-09T04:21:26.824766+0000","last_clean":"2023-10-09T04:21:26.824766+0000","last_became_active":"2023-10-09T04:18:18.498583+0000","last_became_peered":"2023-10-09T04:18:18.498583+0000","last_unstale":"2023-10-09T04:21:26.824766+0000","last_undegraded":"2023-10-09T04:21:26.824766+0000","last_fullsized":"2023-10-09T04:21:26.824766+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:59.278965+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":141,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.892937+0000","last_change":"2023-10-09T04:18:28.851336+0000","last_active":"2023-10-09T04:21:25.892937+0000","last_peered":"2023-10-09T04:21:25.892937+0000","last_clean":"2023-10-09T04:21:25.892937+0000","last_became_active":"2023-10-09T04:18:28.839044+0000","last_became_peered":"2023-10-09T04:18:28.839044+0000","last_unstale":"2023-10-09T04:21:25.892937+0000","last_undegraded":"2023-10-09T04:21:25.892937+0000","last_fullsized":"2023-10-09T04:21:25.892937+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:36.541953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893328+0000","last_change":"2023-10-09T04:18:28.822655+0000","last_active":"2023-10-09T04:21:25.893328+0000","last_peered":"2023-10-09T04:21:25.893328+0000","last_clean":"2023-10-09T04:21:25.893328+0000","last_became_active":"2023-10-09T04:18:28.822309+0000","last_became_peered":"2023-10-09T04:18:28.822309+0000","last_unstale":"2023-10-09T04:21:25.893328+0000","last_undegraded":"2023-10-09T04:21:25.893328+0000","last_fullsized":"2023-10-09T04:21:25.893328+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:43.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":518,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787353+0000","last_change":"2023-10-09T04:18:25.694537+0000","last_active":"2023-10-09T04:21:26.787353+0000","last_peered":"2023-10-09T04:21:26.787353+0000","last_clean":"2023-10-09T04:21:26.787353+0000","last_became_active":"2023-10-09T04:18:18.494417+0000","last_became_peered":"2023-10-09T04:18:18.494417+0000","last_unstale":"2023-10-09T04:21:26.787353+0000","last_undegraded":"2023-10-09T04:21:26.787353+0000","last_fullsized":"2023-10-09T04:21:26.787353+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:23.308605+0000","scrub_duration":0.0078937020000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":360,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.779820+0000","last_change":"2023-10-09T04:15:25.530158+0000","last_active":"2023-10-09T04:21:25.779820+0000","last_peered":"2023-10-09T04:21:25.779820+0000","last_clean":"2023-10-09T04:21:25.779820+0000","last_became_active":"2023-10-09T04:15:25.409638+0000","last_became_peered":"2023-10-09T04:15:25.409638+0000","last_unstale":"2023-10-09T04:21:25.779820+0000","last_undegraded":"2023-10-09T04:21:25.779820+0000","last_fullsized":"2023-10-09T04:21:25.779820+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:22.161110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":46,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822956+0000","last_change":"2023-10-09T04:19:27.112713+0000","last_active":"2023-10-09T04:21:26.822956+0000","last_peered":"2023-10-09T04:21:26.822956+0000","last_clean":"2023-10-09T04:21:26.822956+0000","last_became_active":"2023-10-09T04:19:27.112429+0000","last_became_peered":"2023-10-09T04:19:27.112429+0000","last_unstale":"2023-10-09T04:21:26.822956+0000","last_undegraded":"2023-10-09T04:21:26.822956+0000","last_fullsized":"2023-10-09T04:21:26.822956+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:54.359436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":511,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780309+0000","last_change":"2023-10-09T04:19:16.338589+0000","last_active":"2023-10-09T04:21:25.780309+0000","last_peered":"2023-10-09T04:21:25.780309+0000","last_clean":"2023-10-09T04:21:25.780309+0000","last_became_active":"2023-10-09T04:18:31.847993+0000","last_became_peered":"2023-10-09T04:18:31.847993+0000","last_unstale":"2023-10-09T04:21:25.780309+0000","last_undegraded":"2023-10-09T04:21:25.780309+0000","last_fullsized":"2023-10-09T04:21:25.780309+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:15:00.425536+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":345,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787472+0000","last_change":"2023-10-09T04:18:15.742366+0000","last_active":"2023-10-09T04:21:26.787472+0000","last_peered":"2023-10-09T04:21:26.787472+0000","last_clean":"2023-10-09T04:21:26.787472+0000","last_became_active":"2023-10-09T04:18:15.742049+0000","last_became_peered":"2023-10-09T04:18:15.742049+0000","last_unstale":"2023-10-09T04:21:26.787472+0000","last_undegraded":"2023-10-09T04:21:26.787472+0000","last_fullsized":"2023-10-09T04:21:26.787472+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:33.762730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":345,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781156+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:21:25.781156+0000","last_peered":"2023-10-09T04:21:25.781156+0000","last_clean":"2023-10-09T04:21:25.781156+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:21:25.781156+0000","last_undegraded":"2023-10-09T04:21:25.781156+0000","last_fullsized":"2023-10-09T04:21:25.781156+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":510,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822638+0000","last_change":"2023-10-09T04:20:05.041214+0000","last_active":"2023-10-09T04:21:26.822638+0000","last_peered":"2023-10-09T04:21:26.822638+0000","last_clean":"2023-10-09T04:21:26.822638+0000","last_became_active":"2023-10-09T04:20:05.040867+0000","last_became_peered":"2023-10-09T04:20:05.040867+0000","last_unstale":"2023-10-09T04:21:26.822638+0000","last_undegraded":"2023-10-09T04:21:26.822638+0000","last_fullsized":"2023-10-09T04:21:26.822638+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:02.674367+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.892934+0000","last_change":"2023-10-09T04:18:28.841579+0000","last_active":"2023-10-09T04:21:25.892934+0000","last_peered":"2023-10-09T04:21:25.892934+0000","last_clean":"2023-10-09T04:21:25.892934+0000","last_became_active":"2023-10-09T04:18:28.841313+0000","last_became_peered":"2023-10-09T04:18:28.841313+0000","last_unstale":"2023-10-09T04:21:25.892934+0000","last_undegraded":"2023-10-09T04:21:25.892934+0000","last_fullsized":"2023-10-09T04:21:25.892934+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:36.403950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":345,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787854+0000","last_change":"2023-10-09T04:18:15.741839+0000","last_active":"2023-10-09T04:21:26.787854+0000","last_peered":"2023-10-09T04:21:26.787854+0000","last_clean":"2023-10-09T04:21:26.787854+0000","last_became_active":"2023-10-09T04:18:15.741513+0000","last_became_peered":"2023-10-09T04:18:15.741513+0000","last_unstale":"2023-10-09T04:21:26.787854+0000","last_undegraded":"2023-10-09T04:21:26.787854+0000","last_fullsized":"2023-10-09T04:21:26.787854+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:36.988190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":511,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781086+0000","last_change":"2023-10-09T04:19:19.353864+0000","last_active":"2023-10-09T04:21:25.781086+0000","last_peered":"2023-10-09T04:21:25.781086+0000","last_clean":"2023-10-09T04:21:25.781086+0000","last_became_active":"2023-10-09T04:18:31.847170+0000","last_became_peered":"2023-10-09T04:18:31.847170+0000","last_unstale":"2023-10-09T04:21:25.781086+0000","last_undegraded":"2023-10-09T04:21:25.781086+0000","last_fullsized":"2023-10-09T04:21:25.781086+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:04.785831+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":358,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787393+0000","last_change":"2023-10-09T04:19:42.520684+0000","last_active":"2023-10-09T04:21:26.787393+0000","last_peered":"2023-10-09T04:21:26.787393+0000","last_clean":"2023-10-09T04:21:26.787393+0000","last_became_active":"2023-10-09T04:19:42.515894+0000","last_became_peered":"2023-10-09T04:19:42.515894+0000","last_unstale":"2023-10-09T04:21:26.787393+0000","last_undegraded":"2023-10-09T04:21:26.787393+0000","last_fullsized":"2023-10-09T04:21:26.787393+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:06.090795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":29,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.831903+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:21:25.831903+0000","last_peered":"2023-10-09T04:21:25.831903+0000","last_clean":"2023-10-09T04:21:25.831903+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:21:25.831903+0000","last_undegraded":"2023-10-09T04:21:25.831903+0000","last_fullsized":"2023-10-09T04:21:25.831903+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"368'11007","reported_seq":11079,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:31.834132+0000","last_change":"2023-10-09T04:18:07.678091+0000","last_active":"2023-10-09T04:21:31.834132+0000","last_peered":"2023-10-09T04:21:31.834132+0000","last_clean":"2023-10-09T04:21:31.834132+0000","last_became_active":"2023-10-09T04:18:07.677824+0000","last_became_peered":"2023-10-09T04:18:07.677824+0000","last_unstale":"2023-10-09T04:21:31.834132+0000","last_undegraded":"2023-10-09T04:21:31.834132+0000","last_fullsized":"2023-10-09T04:21:31.834132+0000","mapping_epoch":306,"log_start":"330'4700","ondisk_log_start":"330'4700","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:33.385987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76873728,"num_objects":1173,"num_object_clones":0,"num_object_copies":3519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11007,"num_write_kb":389760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":769,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781780+0000","last_change":"2023-10-09T04:19:14.471197+0000","last_active":"2023-10-09T04:21:25.781780+0000","last_peered":"2023-10-09T04:21:25.781780+0000","last_clean":"2023-10-09T04:21:25.781780+0000","last_became_active":"2023-10-09T04:18:04.686581+0000","last_became_peered":"2023-10-09T04:18:04.686581+0000","last_unstale":"2023-10-09T04:21:25.781780+0000","last_undegraded":"2023-10-09T04:21:25.781780+0000","last_fullsized":"2023-10-09T04:21:25.781780+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:17.228494+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":355,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780606+0000","last_change":"2023-10-09T04:15:25.557813+0000","last_active":"2023-10-09T04:21:25.780606+0000","last_peered":"2023-10-09T04:21:25.780606+0000","last_clean":"2023-10-09T04:21:25.780606+0000","last_became_active":"2023-10-09T04:15:25.413969+0000","last_became_peered":"2023-10-09T04:15:25.413969+0000","last_unstale":"2023-10-09T04:21:25.780606+0000","last_undegraded":"2023-10-09T04:21:25.780606+0000","last_fullsized":"2023-10-09T04:21:25.780606+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:47.696322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":318,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780550+0000","last_change":"2023-10-09T04:18:28.841028+0000","last_active":"2023-10-09T04:21:25.780550+0000","last_peered":"2023-10-09T04:21:25.780550+0000","last_clean":"2023-10-09T04:21:25.780550+0000","last_became_active":"2023-10-09T04:18:28.840747+0000","last_became_peered":"2023-10-09T04:18:28.840747+0000","last_unstale":"2023-10-09T04:21:25.780550+0000","last_undegraded":"2023-10-09T04:21:25.780550+0000","last_fullsized":"2023-10-09T04:21:25.780550+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:31:01.322695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"368'11286","reported_seq":11365,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:33.493651+0000","last_change":"2023-10-09T04:18:15.731429+0000","last_active":"2023-10-09T04:21:33.493651+0000","last_peered":"2023-10-09T04:21:33.493651+0000","last_clean":"2023-10-09T04:21:33.493651+0000","last_became_active":"2023-10-09T04:18:15.731091+0000","last_became_peered":"2023-10-09T04:18:15.731091+0000","last_unstale":"2023-10-09T04:21:33.493651+0000","last_undegraded":"2023-10-09T04:21:33.493651+0000","last_fullsized":"2023-10-09T04:21:33.493651+0000","mapping_epoch":310,"log_start":"336'5973","ondisk_log_start":"336'5973","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5313,"log_dups_size":0,"ondisk_log_size":5313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.128502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72744960,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11286,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":534,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.832291+0000","last_change":"2023-10-09T04:18:31.846705+0000","last_active":"2023-10-09T04:21:25.832291+0000","last_peered":"2023-10-09T04:21:25.832291+0000","last_clean":"2023-10-09T04:21:25.832291+0000","last_became_active":"2023-10-09T04:18:31.846345+0000","last_became_peered":"2023-10-09T04:18:31.846345+0000","last_unstale":"2023-10-09T04:21:25.832291+0000","last_undegraded":"2023-10-09T04:21:25.832291+0000","last_fullsized":"2023-10-09T04:21:25.832291+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:47.381465+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":359,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781568+0000","last_change":"2023-10-09T04:16:20.084271+0000","last_active":"2023-10-09T04:21:25.781568+0000","last_peered":"2023-10-09T04:21:25.781568+0000","last_clean":"2023-10-09T04:21:25.781568+0000","last_became_active":"2023-10-09T04:15:25.414693+0000","last_became_peered":"2023-10-09T04:15:25.414693+0000","last_unstale":"2023-10-09T04:21:25.781568+0000","last_undegraded":"2023-10-09T04:21:25.781568+0000","last_fullsized":"2023-10-09T04:21:25.781568+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:42.431063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893015+0000","last_change":"2023-10-09T04:20:02.106062+0000","last_active":"2023-10-09T04:21:25.893015+0000","last_peered":"2023-10-09T04:21:25.893015+0000","last_clean":"2023-10-09T04:21:25.893015+0000","last_became_active":"2023-10-09T04:20:02.105781+0000","last_became_peered":"2023-10-09T04:20:02.105781+0000","last_unstale":"2023-10-09T04:21:25.893015+0000","last_undegraded":"2023-10-09T04:21:25.893015+0000","last_fullsized":"2023-10-09T04:21:25.893015+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:13.301758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"367'11311","reported_seq":17236,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.851267+0000","last_change":"2023-10-09T04:21:10.590718+0000","last_active":"2023-10-09T04:21:30.851267+0000","last_peered":"2023-10-09T04:21:30.851267+0000","last_clean":"2023-10-09T04:21:30.851267+0000","last_became_active":"2023-10-09T04:21:10.569349+0000","last_became_peered":"2023-10-09T04:21:10.569349+0000","last_unstale":"2023-10-09T04:21:30.851267+0000","last_undegraded":"2023-10-09T04:21:30.851267+0000","last_fullsized":"2023-10-09T04:21:30.851267+0000","mapping_epoch":363,"log_start":"331'5418","ondisk_log_start":"331'5418","created":306,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:41.279693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11311,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2916,"num_bytes_recovered":191102976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":501,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780173+0000","last_change":"2023-10-09T04:18:31.849056+0000","last_active":"2023-10-09T04:21:25.780173+0000","last_peered":"2023-10-09T04:21:25.780173+0000","last_clean":"2023-10-09T04:21:25.780173+0000","last_became_active":"2023-10-09T04:18:31.848099+0000","last_became_peered":"2023-10-09T04:18:31.848099+0000","last_unstale":"2023-10-09T04:21:25.780173+0000","last_undegraded":"2023-10-09T04:21:25.780173+0000","last_fullsized":"2023-10-09T04:21:25.780173+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T03:50:59.589675+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":324,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781882+0000","last_change":"2023-10-09T04:18:28.844980+0000","last_active":"2023-10-09T04:21:25.781882+0000","last_peered":"2023-10-09T04:21:25.781882+0000","last_clean":"2023-10-09T04:21:25.781882+0000","last_became_active":"2023-10-09T04:18:28.837067+0000","last_became_peered":"2023-10-09T04:18:28.837067+0000","last_unstale":"2023-10-09T04:21:25.781882+0000","last_undegraded":"2023-10-09T04:21:25.781882+0000","last_fullsized":"2023-10-09T04:21:25.781882+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:20.410270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":300,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822738+0000","last_change":"2023-10-09T04:18:28.826960+0000","last_active":"2023-10-09T04:21:26.822738+0000","last_peered":"2023-10-09T04:21:26.822738+0000","last_clean":"2023-10-09T04:21:26.822738+0000","last_became_active":"2023-10-09T04:18:28.826661+0000","last_became_peered":"2023-10-09T04:18:28.826661+0000","last_unstale":"2023-10-09T04:21:26.822738+0000","last_undegraded":"2023-10-09T04:21:26.822738+0000","last_fullsized":"2023-10-09T04:21:26.822738+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:49.871283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"367'11148","reported_seq":11217,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.860708+0000","last_change":"2023-10-09T04:18:07.758456+0000","last_active":"2023-10-09T04:21:30.860708+0000","last_peered":"2023-10-09T04:21:30.860708+0000","last_clean":"2023-10-09T04:21:30.860708+0000","last_became_active":"2023-10-09T04:18:07.758139+0000","last_became_peered":"2023-10-09T04:18:07.758139+0000","last_unstale":"2023-10-09T04:21:30.860708+0000","last_undegraded":"2023-10-09T04:21:30.860708+0000","last_fullsized":"2023-10-09T04:21:30.860708+0000","mapping_epoch":306,"log_start":"331'5187","ondisk_log_start":"331'5187","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5961,"log_dups_size":0,"ondisk_log_size":5961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:59.849297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77070336,"num_objects":1176,"num_object_clones":0,"num_object_copies":3528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11148,"num_write_kb":394368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":511,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893182+0000","last_change":"2023-10-09T04:18:26.800928+0000","last_active":"2023-10-09T04:21:25.893182+0000","last_peered":"2023-10-09T04:21:25.893182+0000","last_clean":"2023-10-09T04:21:25.893182+0000","last_became_active":"2023-10-09T04:18:18.488144+0000","last_became_peered":"2023-10-09T04:18:18.488144+0000","last_unstale":"2023-10-09T04:21:25.893182+0000","last_undegraded":"2023-10-09T04:21:25.893182+0000","last_fullsized":"2023-10-09T04:21:25.893182+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:49.461621+0000","scrub_duration":0.001561245,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":356,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.779971+0000","last_change":"2023-10-09T04:18:28.826656+0000","last_active":"2023-10-09T04:21:25.779971+0000","last_peered":"2023-10-09T04:21:25.779971+0000","last_clean":"2023-10-09T04:21:25.779971+0000","last_became_active":"2023-10-09T04:18:28.826313+0000","last_became_peered":"2023-10-09T04:18:28.826313+0000","last_unstale":"2023-10-09T04:21:25.779971+0000","last_undegraded":"2023-10-09T04:21:25.779971+0000","last_fullsized":"2023-10-09T04:21:25.779971+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:22.502566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":345,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787597+0000","last_change":"2023-10-09T04:15:25.475415+0000","last_active":"2023-10-09T04:21:26.787597+0000","last_peered":"2023-10-09T04:21:26.787597+0000","last_clean":"2023-10-09T04:21:26.787597+0000","last_became_active":"2023-10-09T04:15:25.475051+0000","last_became_peered":"2023-10-09T04:15:25.475051+0000","last_unstale":"2023-10-09T04:21:26.787597+0000","last_undegraded":"2023-10-09T04:21:26.787597+0000","last_fullsized":"2023-10-09T04:21:26.787597+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:03.546125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"367'10705","reported_seq":14147,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.939602+0000","last_change":"2023-10-09T04:18:41.922254+0000","last_active":"2023-10-09T04:21:30.939602+0000","last_peered":"2023-10-09T04:21:30.939602+0000","last_clean":"2023-10-09T04:21:30.939602+0000","last_became_active":"2023-10-09T04:18:41.905144+0000","last_became_peered":"2023-10-09T04:18:41.905144+0000","last_unstale":"2023-10-09T04:21:30.939602+0000","last_undegraded":"2023-10-09T04:21:30.939602+0000","last_fullsized":"2023-10-09T04:21:30.939602+0000","mapping_epoch":320,"log_start":"334'5400","ondisk_log_start":"334'5400","created":306,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:47.918579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76873728,"num_objects":1173,"num_object_clones":0,"num_object_copies":3519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10705,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":790,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.779731+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:21:25.779731+0000","last_peered":"2023-10-09T04:21:25.779731+0000","last_clean":"2023-10-09T04:21:25.779731+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:21:25.779731+0000","last_undegraded":"2023-10-09T04:21:25.779731+0000","last_fullsized":"2023-10-09T04:21:25.779731+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":328,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787687+0000","last_change":"2023-10-09T04:18:15.750336+0000","last_active":"2023-10-09T04:21:26.787687+0000","last_peered":"2023-10-09T04:21:26.787687+0000","last_clean":"2023-10-09T04:21:26.787687+0000","last_became_active":"2023-10-09T04:18:15.740930+0000","last_became_peered":"2023-10-09T04:18:15.740930+0000","last_unstale":"2023-10-09T04:21:26.787687+0000","last_undegraded":"2023-10-09T04:21:26.787687+0000","last_fullsized":"2023-10-09T04:21:26.787687+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:04.694041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":141,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893016+0000","last_change":"2023-10-09T04:18:28.823294+0000","last_active":"2023-10-09T04:21:25.893016+0000","last_peered":"2023-10-09T04:21:25.893016+0000","last_clean":"2023-10-09T04:21:25.893016+0000","last_became_active":"2023-10-09T04:18:28.822796+0000","last_became_peered":"2023-10-09T04:18:28.822796+0000","last_unstale":"2023-10-09T04:21:25.893016+0000","last_undegraded":"2023-10-09T04:21:25.893016+0000","last_fullsized":"2023-10-09T04:21:25.893016+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:44:18.052772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"368'11134","reported_seq":14715,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:33.510105+0000","last_change":"2023-10-09T04:19:12.982279+0000","last_active":"2023-10-09T04:21:33.510105+0000","last_peered":"2023-10-09T04:21:33.510105+0000","last_clean":"2023-10-09T04:21:33.510105+0000","last_became_active":"2023-10-09T04:19:12.975073+0000","last_became_peered":"2023-10-09T04:19:12.975073+0000","last_unstale":"2023-10-09T04:21:33.510105+0000","last_undegraded":"2023-10-09T04:21:33.510105+0000","last_fullsized":"2023-10-09T04:21:33.510105+0000","mapping_epoch":329,"log_start":"336'5860","ondisk_log_start":"336'5860","created":306,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:16.362234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74842112,"num_objects":1142,"num_object_clones":0,"num_object_copies":3426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11134,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1746,"num_bytes_recovered":114425856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":536,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780197+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:21:25.780197+0000","last_peered":"2023-10-09T04:21:25.780197+0000","last_clean":"2023-10-09T04:21:25.780197+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:21:25.780197+0000","last_undegraded":"2023-10-09T04:21:25.780197+0000","last_fullsized":"2023-10-09T04:21:25.780197+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893369+0000","last_change":"2023-10-09T04:18:28.823772+0000","last_active":"2023-10-09T04:21:25.893369+0000","last_peered":"2023-10-09T04:21:25.893369+0000","last_clean":"2023-10-09T04:21:25.893369+0000","last_became_active":"2023-10-09T04:18:28.823153+0000","last_became_peered":"2023-10-09T04:18:28.823153+0000","last_unstale":"2023-10-09T04:21:25.893369+0000","last_undegraded":"2023-10-09T04:21:25.893369+0000","last_fullsized":"2023-10-09T04:21:25.893369+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:08.045244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":111,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893349+0000","last_change":"2023-10-09T04:15:25.424135+0000","last_active":"2023-10-09T04:21:25.893349+0000","last_peered":"2023-10-09T04:21:25.893349+0000","last_clean":"2023-10-09T04:21:25.893349+0000","last_became_active":"2023-10-09T04:15:25.423743+0000","last_became_peered":"2023-10-09T04:15:25.423743+0000","last_unstale":"2023-10-09T04:21:25.893349+0000","last_undegraded":"2023-10-09T04:21:25.893349+0000","last_fullsized":"2023-10-09T04:21:25.893349+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:06:59.548960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"367'10958","reported_seq":11037,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.915006+0000","last_change":"2023-10-09T04:18:15.746663+0000","last_active":"2023-10-09T04:21:30.915006+0000","last_peered":"2023-10-09T04:21:30.915006+0000","last_clean":"2023-10-09T04:21:30.915006+0000","last_became_active":"2023-10-09T04:18:15.746284+0000","last_became_peered":"2023-10-09T04:18:15.746284+0000","last_unstale":"2023-10-09T04:21:30.915006+0000","last_undegraded":"2023-10-09T04:21:30.915006+0000","last_fullsized":"2023-10-09T04:21:30.915006+0000","mapping_epoch":310,"log_start":"334'5600","ondisk_log_start":"334'5600","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:03.035710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77856768,"num_objects":1188,"num_object_clones":0,"num_object_copies":3564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10958,"num_write_kb":388672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":493,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822554+0000","last_change":"2023-10-09T04:18:20.720428+0000","last_active":"2023-10-09T04:21:26.822554+0000","last_peered":"2023-10-09T04:21:26.822554+0000","last_clean":"2023-10-09T04:21:26.822554+0000","last_became_active":"2023-10-09T04:18:04.677142+0000","last_became_peered":"2023-10-09T04:18:04.677142+0000","last_unstale":"2023-10-09T04:21:26.822554+0000","last_undegraded":"2023-10-09T04:21:26.822554+0000","last_fullsized":"2023-10-09T04:21:26.822554+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.720321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.720321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:01.967657+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.892836+0000","last_change":"2023-10-09T04:15:25.409399+0000","last_active":"2023-10-09T04:21:25.892836+0000","last_peered":"2023-10-09T04:21:25.892836+0000","last_clean":"2023-10-09T04:21:25.892836+0000","last_became_active":"2023-10-09T04:15:25.409101+0000","last_became_peered":"2023-10-09T04:15:25.409101+0000","last_unstale":"2023-10-09T04:21:25.892836+0000","last_undegraded":"2023-10-09T04:21:25.892836+0000","last_fullsized":"2023-10-09T04:21:25.892836+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:43.626092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":161,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893261+0000","last_change":"2023-10-09T04:19:27.113033+0000","last_active":"2023-10-09T04:21:25.893261+0000","last_peered":"2023-10-09T04:21:25.893261+0000","last_clean":"2023-10-09T04:21:25.893261+0000","last_became_active":"2023-10-09T04:19:27.112636+0000","last_became_peered":"2023-10-09T04:19:27.112636+0000","last_unstale":"2023-10-09T04:21:25.893261+0000","last_undegraded":"2023-10-09T04:21:25.893261+0000","last_fullsized":"2023-10-09T04:21:25.893261+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:23:35.731180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"368'11313","reported_seq":20431,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:33.469956+0000","last_change":"2023-10-09T04:20:48.014336+0000","last_active":"2023-10-09T04:21:33.469956+0000","last_peered":"2023-10-09T04:21:33.469956+0000","last_clean":"2023-10-09T04:21:33.469956+0000","last_became_active":"2023-10-09T04:20:47.989792+0000","last_became_peered":"2023-10-09T04:20:47.989792+0000","last_unstale":"2023-10-09T04:21:33.469956+0000","last_undegraded":"2023-10-09T04:21:33.469956+0000","last_fullsized":"2023-10-09T04:21:33.469956+0000","mapping_epoch":359,"log_start":"336'5981","ondisk_log_start":"336'5981","created":306,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:26.170681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72679424,"num_objects":1109,"num_object_clones":0,"num_object_copies":3327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11313,"num_write_kb":397504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4512,"num_bytes_recovered":295698432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":511,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780795+0000","last_change":"2023-10-09T04:19:25.277863+0000","last_active":"2023-10-09T04:21:25.780795+0000","last_peered":"2023-10-09T04:21:25.780795+0000","last_clean":"2023-10-09T04:21:25.780795+0000","last_became_active":"2023-10-09T04:18:31.849835+0000","last_became_peered":"2023-10-09T04:18:31.849835+0000","last_unstale":"2023-10-09T04:21:25.780795+0000","last_undegraded":"2023-10-09T04:21:25.780795+0000","last_fullsized":"2023-10-09T04:21:25.780795+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:11.092533+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":29,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.833499+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:21:25.833499+0000","last_peered":"2023-10-09T04:21:25.833499+0000","last_clean":"2023-10-09T04:21:25.833499+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:21:25.833499+0000","last_undegraded":"2023-10-09T04:21:25.833499+0000","last_fullsized":"2023-10-09T04:21:25.833499+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":34,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.898528+0000","last_change":"2023-10-09T04:19:50.895337+0000","last_active":"2023-10-09T04:21:25.898528+0000","last_peered":"2023-10-09T04:21:25.898528+0000","last_clean":"2023-10-09T04:21:25.898528+0000","last_became_active":"2023-10-09T04:19:50.895005+0000","last_became_peered":"2023-10-09T04:19:50.895005+0000","last_unstale":"2023-10-09T04:21:25.898528+0000","last_undegraded":"2023-10-09T04:21:25.898528+0000","last_fullsized":"2023-10-09T04:21:25.898528+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:45.147088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"367'11152","reported_seq":11231,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.822041+0000","last_change":"2023-10-09T04:18:15.740852+0000","last_active":"2023-10-09T04:21:30.822041+0000","last_peered":"2023-10-09T04:21:30.822041+0000","last_clean":"2023-10-09T04:21:30.822041+0000","last_became_active":"2023-10-09T04:18:15.739682+0000","last_became_peered":"2023-10-09T04:18:15.739682+0000","last_unstale":"2023-10-09T04:21:30.822041+0000","last_undegraded":"2023-10-09T04:21:30.822041+0000","last_fullsized":"2023-10-09T04:21:30.822041+0000","mapping_epoch":310,"log_start":"331'5234","ondisk_log_start":"331'5234","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:12.051899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82444288,"num_objects":1258,"num_object_clones":0,"num_object_copies":3774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11152,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":531,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780396+0000","last_change":"2023-10-09T04:20:05.041561+0000","last_active":"2023-10-09T04:21:25.780396+0000","last_peered":"2023-10-09T04:21:25.780396+0000","last_clean":"2023-10-09T04:21:25.780396+0000","last_became_active":"2023-10-09T04:20:05.041200+0000","last_became_peered":"2023-10-09T04:20:05.041200+0000","last_unstale":"2023-10-09T04:21:25.780396+0000","last_undegraded":"2023-10-09T04:21:25.780396+0000","last_fullsized":"2023-10-09T04:21:25.780396+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:50.780234+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":348,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.823390+0000","last_change":"2023-10-09T04:02:35.091658+0000","last_active":"2023-10-09T04:21:26.823390+0000","last_peered":"2023-10-09T04:21:26.823390+0000","last_clean":"2023-10-09T04:21:26.823390+0000","last_became_active":"2023-10-09T04:02:35.091215+0000","last_became_peered":"2023-10-09T04:02:35.091215+0000","last_unstale":"2023-10-09T04:21:26.823390+0000","last_undegraded":"2023-10-09T04:21:26.823390+0000","last_fullsized":"2023-10-09T04:21:26.823390+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:06.408303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":363,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781019+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:21:25.781019+0000","last_peered":"2023-10-09T04:21:25.781019+0000","last_clean":"2023-10-09T04:21:25.781019+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:21:25.781019+0000","last_undegraded":"2023-10-09T04:21:25.781019+0000","last_fullsized":"2023-10-09T04:21:25.781019+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.897663+0000","last_change":"2023-10-09T04:19:50.899091+0000","last_active":"2023-10-09T04:21:25.897663+0000","last_peered":"2023-10-09T04:21:25.897663+0000","last_clean":"2023-10-09T04:21:25.897663+0000","last_became_active":"2023-10-09T04:19:50.898743+0000","last_became_peered":"2023-10-09T04:19:50.898743+0000","last_unstale":"2023-10-09T04:21:25.897663+0000","last_undegraded":"2023-10-09T04:21:25.897663+0000","last_fullsized":"2023-10-09T04:21:25.897663+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:49.576833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"368'11133","reported_seq":11205,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:31.851163+0000","last_change":"2023-10-09T04:18:07.678369+0000","last_active":"2023-10-09T04:21:31.851163+0000","last_peered":"2023-10-09T04:21:31.851163+0000","last_clean":"2023-10-09T04:21:31.851163+0000","last_became_active":"2023-10-09T04:18:07.677583+0000","last_became_peered":"2023-10-09T04:18:07.677583+0000","last_unstale":"2023-10-09T04:21:31.851163+0000","last_undegraded":"2023-10-09T04:21:31.851163+0000","last_fullsized":"2023-10-09T04:21:31.851163+0000","mapping_epoch":306,"log_start":"330'4800","ondisk_log_start":"330'4800","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.498364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11133,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":484,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781141+0000","last_change":"2023-10-09T04:19:11.394952+0000","last_active":"2023-10-09T04:21:25.781141+0000","last_peered":"2023-10-09T04:21:25.781141+0000","last_clean":"2023-10-09T04:21:25.781141+0000","last_became_active":"2023-10-09T04:18:04.684208+0000","last_became_peered":"2023-10-09T04:18:04.684208+0000","last_unstale":"2023-10-09T04:21:25.781141+0000","last_undegraded":"2023-10-09T04:21:25.781141+0000","last_fullsized":"2023-10-09T04:21:25.781141+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:45.315268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:45.315268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":340,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780872+0000","last_change":"2023-10-09T04:18:28.826716+0000","last_active":"2023-10-09T04:21:25.780872+0000","last_peered":"2023-10-09T04:21:25.780872+0000","last_clean":"2023-10-09T04:21:25.780872+0000","last_became_active":"2023-10-09T04:18:28.826428+0000","last_became_peered":"2023-10-09T04:18:28.826428+0000","last_unstale":"2023-10-09T04:21:25.780872+0000","last_undegraded":"2023-10-09T04:21:25.780872+0000","last_fullsized":"2023-10-09T04:21:25.780872+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:09.233507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893463+0000","last_change":"2023-10-09T04:18:28.824412+0000","last_active":"2023-10-09T04:21:25.893463+0000","last_peered":"2023-10-09T04:21:25.893463+0000","last_clean":"2023-10-09T04:21:25.893463+0000","last_became_active":"2023-10-09T04:18:28.824095+0000","last_became_peered":"2023-10-09T04:18:28.824095+0000","last_unstale":"2023-10-09T04:21:25.893463+0000","last_undegraded":"2023-10-09T04:21:25.893463+0000","last_fullsized":"2023-10-09T04:21:25.893463+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:19.221802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":376,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.823299+0000","last_change":"2023-10-09T04:18:15.727810+0000","last_active":"2023-10-09T04:21:26.823299+0000","last_peered":"2023-10-09T04:21:26.823299+0000","last_clean":"2023-10-09T04:21:26.823299+0000","last_became_active":"2023-10-09T04:18:15.727516+0000","last_became_peered":"2023-10-09T04:18:15.727516+0000","last_unstale":"2023-10-09T04:21:26.823299+0000","last_undegraded":"2023-10-09T04:21:26.823299+0000","last_fullsized":"2023-10-09T04:21:26.823299+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:34.365818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"367'11252","reported_seq":11321,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.847529+0000","last_change":"2023-10-09T04:18:07.677583+0000","last_active":"2023-10-09T04:21:30.847529+0000","last_peered":"2023-10-09T04:21:30.847529+0000","last_clean":"2023-10-09T04:21:30.847529+0000","last_became_active":"2023-10-09T04:18:07.677258+0000","last_became_peered":"2023-10-09T04:18:07.677258+0000","last_unstale":"2023-10-09T04:21:30.847529+0000","last_undegraded":"2023-10-09T04:21:30.847529+0000","last_fullsized":"2023-10-09T04:21:30.847529+0000","mapping_epoch":306,"log_start":"331'5368","ondisk_log_start":"331'5368","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5884,"log_dups_size":0,"ondisk_log_size":5884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:44.275966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80478208,"num_objects":1228,"num_object_clones":0,"num_object_copies":3684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11252,"num_write_kb":399360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":501,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787487+0000","last_change":"2023-10-09T04:18:24.663913+0000","last_active":"2023-10-09T04:21:26.787487+0000","last_peered":"2023-10-09T04:21:26.787487+0000","last_clean":"2023-10-09T04:21:26.787487+0000","last_became_active":"2023-10-09T04:18:04.679468+0000","last_became_peered":"2023-10-09T04:18:04.679468+0000","last_unstale":"2023-10-09T04:21:26.787487+0000","last_undegraded":"2023-10-09T04:21:26.787487+0000","last_fullsized":"2023-10-09T04:21:26.787487+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:29.786448+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":203,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.897730+0000","last_change":"2023-10-09T04:18:15.742051+0000","last_active":"2023-10-09T04:21:25.897730+0000","last_peered":"2023-10-09T04:21:25.897730+0000","last_clean":"2023-10-09T04:21:25.897730+0000","last_became_active":"2023-10-09T04:18:15.741750+0000","last_became_peered":"2023-10-09T04:18:15.741750+0000","last_unstale":"2023-10-09T04:21:25.897730+0000","last_undegraded":"2023-10-09T04:21:25.897730+0000","last_fullsized":"2023-10-09T04:21:25.897730+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:34.439460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":351,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822236+0000","last_change":"2023-10-09T04:18:28.821704+0000","last_active":"2023-10-09T04:21:26.822236+0000","last_peered":"2023-10-09T04:21:26.822236+0000","last_clean":"2023-10-09T04:21:26.822236+0000","last_became_active":"2023-10-09T04:18:28.821271+0000","last_became_peered":"2023-10-09T04:18:28.821271+0000","last_unstale":"2023-10-09T04:21:26.822236+0000","last_undegraded":"2023-10-09T04:21:26.822236+0000","last_fullsized":"2023-10-09T04:21:26.822236+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:30.792325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":46,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.824256+0000","last_change":"2023-10-09T04:19:27.112577+0000","last_active":"2023-10-09T04:21:26.824256+0000","last_peered":"2023-10-09T04:21:26.824256+0000","last_clean":"2023-10-09T04:21:26.824256+0000","last_became_active":"2023-10-09T04:19:27.112211+0000","last_became_peered":"2023-10-09T04:19:27.112211+0000","last_unstale":"2023-10-09T04:21:26.824256+0000","last_undegraded":"2023-10-09T04:21:26.824256+0000","last_fullsized":"2023-10-09T04:21:26.824256+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:53.139744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"368'11235","reported_seq":20681,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:31.859000+0000","last_change":"2023-10-09T04:21:26.820466+0000","last_active":"2023-10-09T04:21:31.859000+0000","last_peered":"2023-10-09T04:21:31.859000+0000","last_clean":"2023-10-09T04:21:31.859000+0000","last_became_active":"2023-10-09T04:21:26.793215+0000","last_became_peered":"2023-10-09T04:21:26.793215+0000","last_unstale":"2023-10-09T04:21:31.859000+0000","last_undegraded":"2023-10-09T04:21:31.859000+0000","last_fullsized":"2023-10-09T04:21:31.859000+0000","mapping_epoch":366,"log_start":"330'4900","ondisk_log_start":"330'4900","created":306,"last_epoch_clean":367,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6335,"log_dups_size":0,"ondisk_log_size":6335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:46.683876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75563008,"num_objects":1153,"num_object_clones":0,"num_object_copies":3459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11235,"num_write_kb":396416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4032,"num_bytes_recovered":264241152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":350,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780286+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:21:25.780286+0000","last_peered":"2023-10-09T04:21:25.780286+0000","last_clean":"2023-10-09T04:21:25.780286+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:21:25.780286+0000","last_undegraded":"2023-10-09T04:21:25.780286+0000","last_fullsized":"2023-10-09T04:21:25.780286+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":761,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780189+0000","last_change":"2023-10-09T04:18:31.857328+0000","last_active":"2023-10-09T04:21:25.780189+0000","last_peered":"2023-10-09T04:21:25.780189+0000","last_clean":"2023-10-09T04:21:25.780189+0000","last_became_active":"2023-10-09T04:18:31.857034+0000","last_became_peered":"2023-10-09T04:18:31.857034+0000","last_unstale":"2023-10-09T04:21:25.780189+0000","last_undegraded":"2023-10-09T04:21:25.780189+0000","last_fullsized":"2023-10-09T04:21:25.780189+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:08.017419+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":293,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780950+0000","last_change":"2023-10-09T04:18:28.829098+0000","last_active":"2023-10-09T04:21:25.780950+0000","last_peered":"2023-10-09T04:21:25.780950+0000","last_clean":"2023-10-09T04:21:25.780950+0000","last_became_active":"2023-10-09T04:18:28.828779+0000","last_became_peered":"2023-10-09T04:18:28.828779+0000","last_unstale":"2023-10-09T04:21:25.780950+0000","last_undegraded":"2023-10-09T04:21:25.780950+0000","last_fullsized":"2023-10-09T04:21:25.780950+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:41.603562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893371+0000","last_change":"2023-10-09T04:18:28.840169+0000","last_active":"2023-10-09T04:21:25.893371+0000","last_peered":"2023-10-09T04:21:25.893371+0000","last_clean":"2023-10-09T04:21:25.893371+0000","last_became_active":"2023-10-09T04:18:28.839867+0000","last_became_peered":"2023-10-09T04:18:28.839867+0000","last_unstale":"2023-10-09T04:21:25.893371+0000","last_undegraded":"2023-10-09T04:21:25.893371+0000","last_fullsized":"2023-10-09T04:21:25.893371+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:26.575652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"368'11171","reported_seq":14822,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:31.854433+0000","last_change":"2023-10-09T04:18:47.952816+0000","last_active":"2023-10-09T04:21:31.854433+0000","last_peered":"2023-10-09T04:21:31.854433+0000","last_clean":"2023-10-09T04:21:31.854433+0000","last_became_active":"2023-10-09T04:18:47.948229+0000","last_became_peered":"2023-10-09T04:18:47.948229+0000","last_unstale":"2023-10-09T04:21:31.854433+0000","last_undegraded":"2023-10-09T04:21:31.854433+0000","last_fullsized":"2023-10-09T04:21:31.854433+0000","mapping_epoch":323,"log_start":"330'4900","ondisk_log_start":"330'4900","created":306,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":6271,"log_dups_size":0,"ondisk_log_size":6271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:47.118221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11171,"num_write_kb":394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1780,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":511,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.822722+0000","last_change":"2023-10-09T04:18:31.848389+0000","last_active":"2023-10-09T04:21:26.822722+0000","last_peered":"2023-10-09T04:21:26.822722+0000","last_clean":"2023-10-09T04:21:26.822722+0000","last_became_active":"2023-10-09T04:18:31.848041+0000","last_became_peered":"2023-10-09T04:18:31.848041+0000","last_unstale":"2023-10-09T04:21:26.822722+0000","last_undegraded":"2023-10-09T04:21:26.822722+0000","last_fullsized":"2023-10-09T04:21:26.822722+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.708867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.708867+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:50.895678+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":347,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.779601+0000","last_change":"2023-10-09T04:02:35.095799+0000","last_active":"2023-10-09T04:21:25.779601+0000","last_peered":"2023-10-09T04:21:25.779601+0000","last_clean":"2023-10-09T04:21:25.779601+0000","last_became_active":"2023-10-09T04:02:35.095328+0000","last_became_peered":"2023-10-09T04:02:35.095328+0000","last_unstale":"2023-10-09T04:21:25.779601+0000","last_undegraded":"2023-10-09T04:21:25.779601+0000","last_fullsized":"2023-10-09T04:21:25.779601+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:40.390834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":364,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787643+0000","last_change":"2023-10-09T04:20:01.981614+0000","last_active":"2023-10-09T04:21:26.787643+0000","last_peered":"2023-10-09T04:21:26.787643+0000","last_clean":"2023-10-09T04:21:26.787643+0000","last_became_active":"2023-10-09T04:20:01.981304+0000","last_became_peered":"2023-10-09T04:20:01.981304+0000","last_unstale":"2023-10-09T04:21:26.787643+0000","last_undegraded":"2023-10-09T04:21:26.787643+0000","last_fullsized":"2023-10-09T04:21:26.787643+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:41.268790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":361,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:31.313019+0000","last_change":"2023-10-09T04:10:38.338803+0000","last_active":"2023-10-09T04:21:31.313019+0000","last_peered":"2023-10-09T04:21:31.313019+0000","last_clean":"2023-10-09T04:21:31.313019+0000","last_became_active":"2023-10-09T04:10:38.338509+0000","last_became_peered":"2023-10-09T04:10:38.338509+0000","last_unstale":"2023-10-09T04:21:31.313019+0000","last_undegraded":"2023-10-09T04:21:31.313019+0000","last_fullsized":"2023-10-09T04:21:31.313019+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:32.854123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"367'11220","reported_seq":11289,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.940178+0000","last_change":"2023-10-09T04:18:07.679089+0000","last_active":"2023-10-09T04:21:30.940178+0000","last_peered":"2023-10-09T04:21:30.940178+0000","last_clean":"2023-10-09T04:21:30.940178+0000","last_became_active":"2023-10-09T04:18:07.678750+0000","last_became_peered":"2023-10-09T04:18:07.678750+0000","last_unstale":"2023-10-09T04:21:30.940178+0000","last_undegraded":"2023-10-09T04:21:30.940178+0000","last_fullsized":"2023-10-09T04:21:30.940178+0000","mapping_epoch":306,"log_start":"335'5900","ondisk_log_start":"335'5900","created":306,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5320,"log_dups_size":0,"ondisk_log_size":5320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:50.857496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82182144,"num_objects":1254,"num_object_clones":0,"num_object_copies":3762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11220,"num_write_kb":399168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":520,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893037+0000","last_change":"2023-10-09T04:19:06.997185+0000","last_active":"2023-10-09T04:21:25.893037+0000","last_peered":"2023-10-09T04:21:25.893037+0000","last_clean":"2023-10-09T04:21:25.893037+0000","last_became_active":"2023-10-09T04:19:06.996737+0000","last_became_peered":"2023-10-09T04:19:06.996737+0000","last_unstale":"2023-10-09T04:21:25.893037+0000","last_undegraded":"2023-10-09T04:21:25.893037+0000","last_fullsized":"2023-10-09T04:21:25.893037+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:37.164563+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":350,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781907+0000","last_change":"2023-10-09T04:15:25.409820+0000","last_active":"2023-10-09T04:21:25.781907+0000","last_peered":"2023-10-09T04:21:25.781907+0000","last_clean":"2023-10-09T04:21:25.781907+0000","last_became_active":"2023-10-09T04:15:25.409305+0000","last_became_peered":"2023-10-09T04:15:25.409305+0000","last_unstale":"2023-10-09T04:21:25.781907+0000","last_undegraded":"2023-10-09T04:21:25.781907+0000","last_fullsized":"2023-10-09T04:21:25.781907+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:24.253522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":342,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.823470+0000","last_change":"2023-10-09T04:10:38.331358+0000","last_active":"2023-10-09T04:21:26.823470+0000","last_peered":"2023-10-09T04:21:26.823470+0000","last_clean":"2023-10-09T04:21:26.823470+0000","last_became_active":"2023-10-09T04:10:38.331052+0000","last_became_peered":"2023-10-09T04:10:38.331052+0000","last_unstale":"2023-10-09T04:21:26.823470+0000","last_undegraded":"2023-10-09T04:21:26.823470+0000","last_fullsized":"2023-10-09T04:21:26.823470+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:06.754109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":66,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780210+0000","last_change":"2023-10-09T04:19:06.997166+0000","last_active":"2023-10-09T04:21:25.780210+0000","last_peered":"2023-10-09T04:21:25.780210+0000","last_clean":"2023-10-09T04:21:25.780210+0000","last_became_active":"2023-10-09T04:19:06.996865+0000","last_became_peered":"2023-10-09T04:19:06.996865+0000","last_unstale":"2023-10-09T04:21:25.780210+0000","last_undegraded":"2023-10-09T04:21:25.780210+0000","last_fullsized":"2023-10-09T04:21:25.780210+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:11:03.794387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"367'11103","reported_seq":22515,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.901500+0000","last_change":"2023-10-09T04:20:12.010618+0000","last_active":"2023-10-09T04:21:30.901500+0000","last_peered":"2023-10-09T04:21:30.901500+0000","last_clean":"2023-10-09T04:21:30.901500+0000","last_became_active":"2023-10-09T04:20:12.002110+0000","last_became_peered":"2023-10-09T04:20:12.002110+0000","last_unstale":"2023-10-09T04:21:30.901500+0000","last_undegraded":"2023-10-09T04:21:30.901500+0000","last_fullsized":"2023-10-09T04:21:30.901500+0000","mapping_epoch":354,"log_start":"335'5800","ondisk_log_start":"335'5800","created":306,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5303,"log_dups_size":0,"ondisk_log_size":5303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:12.102225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79167512,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11103,"num_write_kb":393921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5657,"num_bytes_recovered":370671640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":511,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893220+0000","last_change":"2023-10-09T04:18:31.850771+0000","last_active":"2023-10-09T04:21:25.893220+0000","last_peered":"2023-10-09T04:21:25.893220+0000","last_clean":"2023-10-09T04:21:25.893220+0000","last_became_active":"2023-10-09T04:18:31.850379+0000","last_became_peered":"2023-10-09T04:18:31.850379+0000","last_unstale":"2023-10-09T04:21:25.893220+0000","last_undegraded":"2023-10-09T04:21:25.893220+0000","last_fullsized":"2023-10-09T04:21:25.893220+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:17.219197+0000","scrub_duration":0.0032394350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":299,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780823+0000","last_change":"2023-10-09T04:18:15.770941+0000","last_active":"2023-10-09T04:21:25.780823+0000","last_peered":"2023-10-09T04:21:25.780823+0000","last_clean":"2023-10-09T04:21:25.780823+0000","last_became_active":"2023-10-09T04:18:15.746015+0000","last_became_peered":"2023-10-09T04:18:15.746015+0000","last_unstale":"2023-10-09T04:21:25.780823+0000","last_undegraded":"2023-10-09T04:21:25.780823+0000","last_fullsized":"2023-10-09T04:21:25.780823+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:11.685240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":335,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.824871+0000","last_change":"2023-10-09T04:18:28.825690+0000","last_active":"2023-10-09T04:21:26.824871+0000","last_peered":"2023-10-09T04:21:26.824871+0000","last_clean":"2023-10-09T04:21:26.824871+0000","last_became_active":"2023-10-09T04:18:28.825357+0000","last_became_peered":"2023-10-09T04:18:28.825357+0000","last_unstale":"2023-10-09T04:21:26.824871+0000","last_undegraded":"2023-10-09T04:21:26.824871+0000","last_fullsized":"2023-10-09T04:21:26.824871+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:39.340830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":317,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780377+0000","last_change":"2023-10-09T04:20:01.987327+0000","last_active":"2023-10-09T04:21:25.780377+0000","last_peered":"2023-10-09T04:21:25.780377+0000","last_clean":"2023-10-09T04:21:25.780377+0000","last_became_active":"2023-10-09T04:20:01.987052+0000","last_became_peered":"2023-10-09T04:20:01.987052+0000","last_unstale":"2023-10-09T04:21:25.780377+0000","last_undegraded":"2023-10-09T04:21:25.780377+0000","last_fullsized":"2023-10-09T04:21:25.780377+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:54.642879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"367'11337","reported_seq":11416,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:30.451546+0000","last_change":"2023-10-09T04:18:15.743995+0000","last_active":"2023-10-09T04:21:30.451546+0000","last_peered":"2023-10-09T04:21:30.451546+0000","last_clean":"2023-10-09T04:21:30.451546+0000","last_became_active":"2023-10-09T04:18:15.743665+0000","last_became_peered":"2023-10-09T04:18:15.743665+0000","last_unstale":"2023-10-09T04:21:30.451546+0000","last_undegraded":"2023-10-09T04:21:30.451546+0000","last_fullsized":"2023-10-09T04:21:30.451546+0000","mapping_epoch":310,"log_start":"334'5700","ondisk_log_start":"334'5700","created":306,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:18:06.656780+0000","last_clean_scrub_stamp":"2023-10-09T04:18:06.656780+0000","objects_scrubbed":0,"log_size":5637,"log_dups_size":0,"ondisk_log_size":5637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:09.982667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11337,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":505,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.832222+0000","last_change":"2023-10-09T04:18:20.432247+0000","last_active":"2023-10-09T04:21:25.832222+0000","last_peered":"2023-10-09T04:21:25.832222+0000","last_clean":"2023-10-09T04:21:25.832222+0000","last_became_active":"2023-10-09T04:18:18.494123+0000","last_became_peered":"2023-10-09T04:18:18.494123+0000","last_unstale":"2023-10-09T04:21:25.832222+0000","last_undegraded":"2023-10-09T04:21:25.832222+0000","last_fullsized":"2023-10-09T04:21:25.832222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:55.755572+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":199,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.898557+0000","last_change":"2023-10-09T04:18:28.820905+0000","last_active":"2023-10-09T04:21:25.898557+0000","last_peered":"2023-10-09T04:21:25.898557+0000","last_clean":"2023-10-09T04:21:25.898557+0000","last_became_active":"2023-10-09T04:18:28.820608+0000","last_became_peered":"2023-10-09T04:18:28.820608+0000","last_unstale":"2023-10-09T04:21:25.898557+0000","last_undegraded":"2023-10-09T04:21:25.898557+0000","last_fullsized":"2023-10-09T04:21:25.898557+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.326772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":374,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:21:31.313098+0000","last_change":"2023-10-09T04:19:50.929246+0000","last_active":"2023-10-09T04:21:31.313098+0000","last_peered":"2023-10-09T04:21:31.313098+0000","last_clean":"2023-10-09T04:21:31.313098+0000","last_became_active":"2023-10-09T04:19:50.898795+0000","last_became_peered":"2023-10-09T04:19:50.898795+0000","last_unstale":"2023-10-09T04:21:31.313098+0000","last_undegraded":"2023-10-09T04:21:31.313098+0000","last_fullsized":"2023-10-09T04:21:31.313098+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:17.560700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.892940+0000","last_change":"2023-10-09T04:18:28.821864+0000","last_active":"2023-10-09T04:21:25.892940+0000","last_peered":"2023-10-09T04:21:25.892940+0000","last_clean":"2023-10-09T04:21:25.892940+0000","last_became_active":"2023-10-09T04:18:28.821556+0000","last_became_peered":"2023-10-09T04:18:28.821556+0000","last_unstale":"2023-10-09T04:21:25.892940+0000","last_undegraded":"2023-10-09T04:21:25.892940+0000","last_fullsized":"2023-10-09T04:21:25.892940+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:16.165488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":527,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787351+0000","last_change":"2023-10-09T04:19:18.658467+0000","last_active":"2023-10-09T04:21:26.787351+0000","last_peered":"2023-10-09T04:21:26.787351+0000","last_clean":"2023-10-09T04:21:26.787351+0000","last_became_active":"2023-10-09T04:18:31.842888+0000","last_became_peered":"2023-10-09T04:18:31.842888+0000","last_unstale":"2023-10-09T04:21:26.787351+0000","last_undegraded":"2023-10-09T04:21:26.787351+0000","last_fullsized":"2023-10-09T04:21:26.787351+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:52.628708+0000","scrub_duration":0.011289099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":342,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.787792+0000","last_change":"2023-10-09T04:19:50.888360+0000","last_active":"2023-10-09T04:21:26.787792+0000","last_peered":"2023-10-09T04:21:26.787792+0000","last_clean":"2023-10-09T04:21:26.787792+0000","last_became_active":"2023-10-09T04:19:50.888026+0000","last_became_peered":"2023-10-09T04:19:50.888026+0000","last_unstale":"2023-10-09T04:21:26.787792+0000","last_undegraded":"2023-10-09T04:21:26.787792+0000","last_fullsized":"2023-10-09T04:21:26.787792+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:29.811940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":202,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.898343+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:21:25.898343+0000","last_peered":"2023-10-09T04:21:25.898343+0000","last_clean":"2023-10-09T04:21:25.898343+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:21:25.898343+0000","last_undegraded":"2023-10-09T04:21:25.898343+0000","last_fullsized":"2023-10-09T04:21:25.898343+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":754,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.781470+0000","last_change":"2023-10-09T04:19:16.437914+0000","last_active":"2023-10-09T04:21:25.781470+0000","last_peered":"2023-10-09T04:21:25.781470+0000","last_clean":"2023-10-09T04:21:25.781470+0000","last_became_active":"2023-10-09T04:18:31.854107+0000","last_became_peered":"2023-10-09T04:18:31.854107+0000","last_unstale":"2023-10-09T04:21:25.781470+0000","last_undegraded":"2023-10-09T04:21:25.781470+0000","last_fullsized":"2023-10-09T04:21:25.781470+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.437847+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.437847+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:45.606375+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":356,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780056+0000","last_change":"2023-10-09T04:15:25.553216+0000","last_active":"2023-10-09T04:21:25.780056+0000","last_peered":"2023-10-09T04:21:25.780056+0000","last_clean":"2023-10-09T04:21:25.780056+0000","last_became_active":"2023-10-09T04:15:25.424912+0000","last_became_peered":"2023-10-09T04:15:25.424912+0000","last_unstale":"2023-10-09T04:21:25.780056+0000","last_undegraded":"2023-10-09T04:21:25.780056+0000","last_fullsized":"2023-10-09T04:21:25.780056+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:05.491152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":131,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893403+0000","last_change":"2023-10-09T04:18:15.730146+0000","last_active":"2023-10-09T04:21:25.893403+0000","last_peered":"2023-10-09T04:21:25.893403+0000","last_clean":"2023-10-09T04:21:25.893403+0000","last_became_active":"2023-10-09T04:18:15.729780+0000","last_became_peered":"2023-10-09T04:18:15.729780+0000","last_unstale":"2023-10-09T04:21:25.893403+0000","last_undegraded":"2023-10-09T04:21:25.893403+0000","last_fullsized":"2023-10-09T04:21:25.893403+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:11.850515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":353,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780934+0000","last_change":"2023-10-09T04:18:28.831575+0000","last_active":"2023-10-09T04:21:25.780934+0000","last_peered":"2023-10-09T04:21:25.780934+0000","last_clean":"2023-10-09T04:21:25.780934+0000","last_became_active":"2023-10-09T04:18:28.828802+0000","last_became_peered":"2023-10-09T04:18:28.828802+0000","last_unstale":"2023-10-09T04:21:25.780934+0000","last_undegraded":"2023-10-09T04:21:25.780934+0000","last_fullsized":"2023-10-09T04:21:25.780934+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:15.854718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.893439+0000","last_change":"2023-10-09T04:18:15.736494+0000","last_active":"2023-10-09T04:21:25.893439+0000","last_peered":"2023-10-09T04:21:25.893439+0000","last_clean":"2023-10-09T04:21:25.893439+0000","last_became_active":"2023-10-09T04:18:15.736170+0000","last_became_peered":"2023-10-09T04:18:15.736170+0000","last_unstale":"2023-10-09T04:21:25.893439+0000","last_undegraded":"2023-10-09T04:21:25.893439+0000","last_fullsized":"2023-10-09T04:21:25.893439+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:01.009545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":789,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.780114+0000","last_change":"2023-10-09T04:18:31.847724+0000","last_active":"2023-10-09T04:21:25.780114+0000","last_peered":"2023-10-09T04:21:25.780114+0000","last_clean":"2023-10-09T04:21:25.780114+0000","last_became_active":"2023-10-09T04:18:31.847378+0000","last_became_peered":"2023-10-09T04:18:31.847378+0000","last_unstale":"2023-10-09T04:21:25.780114+0000","last_undegraded":"2023-10-09T04:21:25.780114+0000","last_fullsized":"2023-10-09T04:21:25.780114+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.309400+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.309400+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.490566+0000","scrub_duration":0.0079074899999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":212,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.898480+0000","last_change":"2023-10-09T04:18:28.820502+0000","last_active":"2023-10-09T04:21:25.898480+0000","last_peered":"2023-10-09T04:21:25.898480+0000","last_clean":"2023-10-09T04:21:25.898480+0000","last_became_active":"2023-10-09T04:18:28.820072+0000","last_became_peered":"2023-10-09T04:18:28.820072+0000","last_unstale":"2023-10-09T04:21:25.898480+0000","last_undegraded":"2023-10-09T04:21:25.898480+0000","last_fullsized":"2023-10-09T04:21:25.898480+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:15.379781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.897828+0000","last_change":"2023-10-09T04:18:15.741579+0000","last_active":"2023-10-09T04:21:25.897828+0000","last_peered":"2023-10-09T04:21:25.897828+0000","last_clean":"2023-10-09T04:21:25.897828+0000","last_became_active":"2023-10-09T04:18:15.741259+0000","last_became_peered":"2023-10-09T04:18:15.741259+0000","last_unstale":"2023-10-09T04:21:25.897828+0000","last_undegraded":"2023-10-09T04:21:25.897828+0000","last_fullsized":"2023-10-09T04:21:25.897828+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:29.219516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":510,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.824029+0000","last_change":"2023-10-09T04:20:05.043020+0000","last_active":"2023-10-09T04:21:26.824029+0000","last_peered":"2023-10-09T04:21:26.824029+0000","last_clean":"2023-10-09T04:21:26.824029+0000","last_became_active":"2023-10-09T04:20:05.042712+0000","last_became_peered":"2023-10-09T04:20:05.042712+0000","last_unstale":"2023-10-09T04:21:26.824029+0000","last_undegraded":"2023-10-09T04:21:26.824029+0000","last_fullsized":"2023-10-09T04:21:26.824029+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.731815+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.731815+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:48.564601+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":362,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:21:26.788048+0000","last_change":"2023-10-09T04:10:38.354396+0000","last_active":"2023-10-09T04:21:26.788048+0000","last_peered":"2023-10-09T04:21:26.788048+0000","last_clean":"2023-10-09T04:21:26.788048+0000","last_became_active":"2023-10-09T04:10:38.338671+0000","last_became_peered":"2023-10-09T04:10:38.338671+0000","last_unstale":"2023-10-09T04:21:26.788048+0000","last_undegraded":"2023-10-09T04:21:26.788048+0000","last_fullsized":"2023-10-09T04:21:26.788048+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:15.265268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":121,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:21:25.892740+0000","last_change":"2023-10-09T04:18:28.822602+0000","last_active":"2023-10-09T04:21:25.892740+0000","last_peered":"2023-10-09T04:21:25.892740+0000","last_clean":"2023-10-09T04:21:25.892740+0000","last_became_active":"2023-10-09T04:18:28.822261+0000","last_became_peered":"2023-10-09T04:18:28.822261+0000","last_unstale":"2023-10-09T04:21:25.892740+0000","last_undegraded":"2023-10-09T04:21:25.892740+0000","last_fullsized":"2023-10-09T04:21:25.892740+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:51.780349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":11208,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":666401,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":63,"num_read_kb":118,"num_write":32,"num_write_kb":312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":555777,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1244004376,"num_objects":18983,"num_object_clones":0,"num_object_copies":56949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18983,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178465,"num_write_kb":6318273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22313,"num_bytes_recovered":1462239256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4391976960,"data_stored":4391960672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91744,"ondisk_log_size":91744,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201628,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514296,"kb_used_data":928624,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":92230856,"statfs":{"total":95995035648,"available":94444396544,"internally_reserved":0,"allocated":950910976,"data_stored":949863862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":9.6059999999999999,"5min":3.8919999999999999,"15min":3.5350000000000001},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":148.65899999999999,"5min":148.65899999999999,"15min":148.65899999999999},"last":4.1029999999999998},{"interface":"front","average":{"1min":10.054,"5min":3.8660000000000001,"15min":3.5179999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":148.642,"5min":148.642,"15min":148.642},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:21:08 2023","interfaces":[{"interface":"back","average":{"1min":21.657,"5min":6.6159999999999997,"15min":5.6079999999999997},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23200000000000001},"max":{"1min":224.215,"5min":224.215,"15min":245.28700000000001},"last":1.075},{"interface":"front","average":{"1min":19.774000000000001,"5min":6.4779999999999998,"15min":5.4169999999999998},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":223.928,"5min":223.928,"15min":244.626},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.698,"15min":2.1240000000000001},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.27400000000000002},"max":{"1min":4.2270000000000003,"5min":14.076000000000001,"15min":76.774000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.5,"5min":2.528,"15min":2.8540000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":6.5460000000000003,"5min":15.597,"15min":99.372},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.712,"15min":2.5590000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.0250000000000004,"5min":8.8550000000000004,"15min":99.134},"last":0.82199999999999995},{"interface":"front","average":{"1min":2.133,"5min":2.1539999999999999,"15min":2.8260000000000001},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":7.0369999999999999,"5min":8.0489999999999995,"15min":99.117999999999995},"last":0.89300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.96499999999999997,"15min":0.85799999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.23400000000000001},"max":{"1min":4.1799999999999997,"5min":8.9359999999999999,"15min":16.195},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.236,"5min":1.714,"15min":1.1539999999999999},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26000000000000001},"max":{"1min":4.4500000000000002,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.378}]},{"osd":5,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":1.883,"15min":1.4810000000000001},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":13.122999999999999,"5min":13.122999999999999,"15min":120.066},"last":6.8070000000000004},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.3700000000000001,"15min":1.359},"min":{"1min":0.28999999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8360000000000003,"5min":9,"15min":120.09699999999999},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:20:38 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.667,"15min":1.1140000000000001},"min":{"1min":0.34300000000000003,"5min":0.31900000000000001,"15min":0.307},"max":{"1min":7.3310000000000004,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.149,"5min":1.069,"15min":0.92000000000000004},"min":{"1min":0.40799999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.7060000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":1.472}]}]},{"osd":6,"up_from":0,"seq":201863463265,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":89,"seq":382252089574,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357944,"kb_used_data":729520,"kb_used_omap":9,"kb_used_meta":628406,"kb_avail":92387208,"statfs":{"total":95995035648,"available":94604500992,"internally_reserved":0,"allocated":747028480,"data_stored":745462873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":643488745},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1589999999999998,"5min":3.968,"15min":4.1699999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17199999999999999},"max":{"1min":12.067,"5min":98.257999999999996,"15min":98.257999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":3.1160000000000001,"5min":6.4640000000000004,"15min":5.2960000000000003},"min":{"1min":0.28599999999999998,"5min":0.215,"15min":0.182},"max":{"1min":9.2260000000000009,"5min":154.654,"15min":154.654},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":2.1699999999999999,"15min":2.7170000000000001},"min":{"1min":0.221,"5min":0.21199999999999999,"15min":0.191},"max":{"1min":12.791,"5min":19.564,"15min":121.54600000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":3.702,"5min":2.657,"15min":3.1400000000000001},"min":{"1min":0.28100000000000003,"5min":0.20399999999999999,"15min":0.186},"max":{"1min":14.503,"5min":16.114999999999998,"15min":99.899000000000001},"last":1.694}]},{"osd":3,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1389999999999998,"5min":2.3959999999999999,"15min":3.8639999999999999},"min":{"1min":0.27000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.5210000000000008,"5min":19.640000000000001,"15min":138.87299999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":3.7799999999999998,"5min":3.3530000000000002,"15min":4.843},"min":{"1min":0.33800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":12.241,"5min":21.859000000000002,"15min":138.49799999999999},"last":1.665}]},{"osd":4,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0640000000000001,"5min":2.3900000000000001,"15min":2.4430000000000001},"min":{"1min":0.31900000000000001,"5min":0.27400000000000002,"15min":0.254},"max":{"1min":9.3149999999999995,"5min":16.27,"15min":99.989999999999995},"last":1.6259999999999999},{"interface":"front","average":{"1min":2.238,"5min":1.865,"15min":2.2440000000000002},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":8.2149999999999999,"5min":9.2929999999999993,"15min":100.249},"last":0.59099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":3.3570000000000002,"15min":2.7530000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29099999999999998},"max":{"1min":10.222,"5min":38.848999999999997,"15min":99.933999999999997},"last":1.5960000000000001},{"interface":"front","average":{"1min":3.6779999999999999,"5min":3.218,"15min":2.8450000000000002},"min":{"1min":0.33100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":21.893999999999998,"5min":39.593000000000004,"15min":100.34099999999999},"last":3.6989999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":2.1080000000000001,"15min":2.577},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.30299999999999999},"max":{"1min":8.3529999999999998,"5min":9.3640000000000008,"15min":100.309},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.7529999999999999,"5min":1.8029999999999999,"15min":2.1859999999999999},"min":{"1min":0.38600000000000001,"5min":0.34300000000000003,"15min":0.29799999999999999},"max":{"1min":9.2949999999999999,"5min":19.440000000000001,"15min":100.143},"last":0.53300000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:21:10 2023","interfaces":[{"interface":"back","average":{"1min":11.962,"5min":4.5270000000000001,"15min":3.339},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":183.66399999999999,"5min":183.66399999999999,"15min":183.66399999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":10.627000000000001,"5min":3.8370000000000002,"15min":2.9199999999999999},"min":{"1min":0.40600000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":183.71600000000001,"5min":183.71600000000001,"15min":183.71600000000001},"last":0.55700000000000005}]}]},{"osd":0,"up_from":84,"seq":360777253100,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":922924,"kb_used_data":395236,"kb_used_omap":28,"kb_used_meta":527651,"kb_avail":92822228,"statfs":{"total":95995035648,"available":95049961472,"internally_reserved":0,"allocated":404721664,"data_stored":403099709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":540315472},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:21:12 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":7.8330000000000002,"15min":5.3920000000000003},"min":{"1min":0.21199999999999999,"5min":0.19500000000000001,"15min":0.17599999999999999},"max":{"1min":6.7549999999999999,"5min":209.215,"15min":209.215},"last":2.488},{"interface":"front","average":{"1min":9.1720000000000006,"5min":6.9870000000000001,"15min":5.5659999999999998},"min":{"1min":0.36299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":88.019000000000005,"5min":120.934,"15min":120.934},"last":0.997}]},{"osd":2,"last update":"Mon Oct 9 04:20:47 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":2.149,"15min":3.2570000000000001},"min":{"1min":0.216,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.131,"5min":17.466999999999999,"15min":118.54000000000001},"last":2.3690000000000002},{"interface":"front","average":{"1min":2.7530000000000001,"5min":3.1480000000000001,"15min":3.6899999999999999},"min":{"1min":0.224,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":11.050000000000001,"5min":20.262,"15min":118.727},"last":2.6680000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:21:12 2023","interfaces":[{"interface":"back","average":{"1min":9.3230000000000004,"5min":3.391,"15min":5.4740000000000002},"min":{"1min":0.32500000000000001,"5min":0.192,"15min":0.192},"max":{"1min":87.891999999999996,"5min":87.891999999999996,"15min":156.488},"last":2.5310000000000001},{"interface":"front","average":{"1min":9.0839999999999996,"5min":3.5259999999999998,"15min":5.5060000000000002},"min":{"1min":0.27300000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":88.147999999999996,"5min":88.147999999999996,"15min":156.50700000000001},"last":1.139}]},{"osd":4,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":3.0619999999999998,"15min":2.8159999999999998},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":8.7509999999999994,"5min":63.375999999999998,"15min":118.633},"last":1.109},{"interface":"front","average":{"1min":2.3500000000000001,"5min":3.4620000000000002,"15min":3.1829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33900000000000002,"15min":0.28199999999999997},"max":{"1min":8.6950000000000003,"5min":49.607999999999997,"15min":118.913},"last":1.97}]},{"osd":5,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":3.661,"15min":4.4139999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29499999999999998},"max":{"1min":8.7279999999999998,"5min":62.954000000000001,"15min":182.68899999999999},"last":4.7130000000000001},{"interface":"front","average":{"1min":2.0880000000000001,"5min":2.6269999999999998,"15min":3.8719999999999999},"min":{"1min":0.36899999999999999,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":5.5469999999999997,"5min":63.289000000000001,"15min":182.66300000000001},"last":2.5899999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":2.5899999999999999,"15min":3.0950000000000002},"min":{"1min":0.40200000000000002,"5min":0.34300000000000003,"15min":0.30599999999999999},"max":{"1min":5.5099999999999998,"5min":63.468000000000004,"15min":137.36099999999999},"last":2.649},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.6219999999999999,"15min":3.133},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.317},"max":{"1min":8.8160000000000007,"5min":63.447000000000003,"15min":136.88999999999999},"last":1.1679999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:20:52 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":2.681,"15min":3.7869999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":5.5780000000000003,"5min":63.509,"15min":185.024},"last":2.6139999999999999},{"interface":"front","average":{"1min":2.5619999999999998,"5min":2.8090000000000002,"15min":3.4660000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.32700000000000001},"max":{"1min":8.7789999999999999,"5min":63.481999999999999,"15min":184.98599999999999},"last":1.155}]}]},{"osd":2,"up_from":94,"seq":403726926047,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1037180,"kb_used_data":456884,"kb_used_omap":81,"kb_used_meta":580206,"kb_avail":92707972,"statfs":{"total":95995035648,"available":94932963328,"internally_reserved":0,"allocated":467849216,"data_stored":466275946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":594131162},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":4.6479999999999997,"15min":4.0670000000000002},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17399999999999999},"max":{"1min":21.792000000000002,"5min":81.385999999999996,"15min":81.385999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":3.9790000000000001,"5min":4.5990000000000002,"15min":4.0250000000000004},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.20000000000000001},"max":{"1min":21.603000000000002,"5min":79.453999999999994,"15min":79.453999999999994},"last":0.28100000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.98,"5min":7.4539999999999997,"15min":4.6539999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":21.556999999999999,"5min":158.13399999999999,"15min":158.13399999999999},"last":2.7999999999999998},{"interface":"front","average":{"1min":3.165,"5min":7.8869999999999996,"15min":4.8719999999999999},"min":{"1min":0.255,"5min":0.253,"15min":0.20799999999999999},"max":{"1min":24.759,"5min":161.12,"15min":161.12},"last":0.26900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":3.2909999999999999,"15min":2.7429999999999999},"min":{"1min":0.25900000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.832000000000001,"5min":58.997,"15min":58.997},"last":26.521999999999998},{"interface":"front","average":{"1min":2.5270000000000001,"5min":3.5760000000000001,"15min":3.0329999999999999},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":18.853999999999999,"5min":66.135000000000005,"15min":66.135000000000005},"last":23.969999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":2.6829999999999998,"15min":2.234},"min":{"1min":0.32800000000000001,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":24.814,"5min":58.798000000000002,"15min":58.798000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.77700000000000002,"5min":2.3599999999999999,"15min":2.0529999999999999},"min":{"1min":0.33200000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":3.3700000000000001,"5min":56.616,"15min":56.616},"last":0.36099999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":3.9140000000000001,"15min":2.8029999999999999},"min":{"1min":0.376,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":15.371,"5min":82.030000000000001,"15min":82.030000000000001},"last":0.376},{"interface":"front","average":{"1min":2.3839999999999999,"5min":4.1970000000000001,"15min":2.9079999999999999},"min":{"1min":0.36899999999999999,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":24.843,"5min":82.016000000000005,"15min":82.016000000000005},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 04:21:29 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":2.4220000000000002,"15min":2.5299999999999998},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":3.5990000000000002,"5min":56.561,"15min":114.14},"last":0.501},{"interface":"front","average":{"1min":2.3919999999999999,"5min":2.6120000000000001,"15min":2.6850000000000001},"min":{"1min":0.36199999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":15.456,"5min":56.460999999999999,"15min":114.123},"last":0.54900000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":3.794,"15min":3.1749999999999998},"min":{"1min":0.38,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.738,"5min":101.675,"15min":117.32599999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.79,"5min":4.133,"15min":3.3250000000000002},"min":{"1min":0.432,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":15.406000000000001,"5min":101.72499999999999,"15min":117.28700000000001},"last":0.49099999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762521,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1423704,"kb_used_data":656080,"kb_used_omap":7,"kb_used_meta":767544,"kb_avail":92321448,"statfs":{"total":95995035648,"available":94537162752,"internally_reserved":0,"allocated":671825920,"data_stored":670281881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":785965672},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":3.411,"15min":4.8719999999999999},"min":{"1min":0.26700000000000002,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":10.539,"5min":78.771000000000001,"15min":125.765},"last":0.32300000000000001},{"interface":"front","average":{"1min":4.6210000000000004,"5min":4.4409999999999998,"15min":5.3920000000000003},"min":{"1min":0.254,"5min":0.23799999999999999,"15min":0.193},"max":{"1min":22.446999999999999,"5min":78.853999999999999,"15min":125.46899999999999},"last":6.0259999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":5.5780000000000003,"15min":5.032},"min":{"1min":0.29999999999999999,"5min":0.222,"15min":0.20499999999999999},"max":{"1min":12.792999999999999,"5min":103.143,"15min":186.21299999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":3.7850000000000001,"5min":5.4119999999999999,"15min":5.2229999999999999},"min":{"1min":0.33000000000000002,"5min":0.221,"15min":0.193},"max":{"1min":12.69,"5min":105.19,"15min":191.78},"last":4.1210000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":3.4199999999999999,"15min":3.5299999999999998},"min":{"1min":0.28799999999999998,"5min":0.187,"15min":0.187},"max":{"1min":5.4500000000000002,"5min":127.599,"15min":127.599},"last":0.73999999999999999},{"interface":"front","average":{"1min":3.085,"5min":5.4589999999999996,"15min":4.7229999999999999},"min":{"1min":0.28699999999999998,"5min":0.221,"15min":0.214},"max":{"1min":8.0239999999999991,"5min":129.98599999999999,"15min":129.98599999999999},"last":6.1459999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.9239999999999999,"15min":2.3460000000000001},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.798,"5min":10.013999999999999,"15min":37.652000000000001},"last":6.1200000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":2.097,"15min":2.2919999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.26900000000000002},"max":{"1min":8.6080000000000005,"5min":9.7170000000000005,"15min":37.442},"last":1.972}]},{"osd":5,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.5169999999999999,"15min":2.3399999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5679999999999996,"5min":12.144,"15min":17.901},"last":5.1289999999999996},{"interface":"front","average":{"1min":2.3340000000000001,"5min":1.613,"15min":1.996},"min":{"1min":0.38400000000000001,"5min":0.248,"15min":0.248},"max":{"1min":8.4670000000000005,"5min":8.4670000000000005,"15min":18.068000000000001},"last":4.1680000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.48,"15min":1.8799999999999999},"min":{"1min":0.41699999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.5300000000000002,"5min":5.758,"15min":14.976000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.1259999999999999,"15min":2.27},"min":{"1min":0.39200000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.8479999999999999,"5min":10.071,"15min":15.622},"last":6.0940000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:21:06 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.5590000000000002,"15min":2.4209999999999998},"min":{"1min":0.441,"5min":0.36199999999999999,"15min":0.34399999999999997},"max":{"1min":6.8239999999999998,"5min":28.184000000000001,"15min":28.184000000000001},"last":6.0659999999999998},{"interface":"front","average":{"1min":2.7400000000000002,"5min":2.758,"15min":2.4420000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.6370000000000005,"5min":26.648,"15min":26.648},"last":5.1699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986537,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":801064,"kb_used_data":503776,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":92944088,"statfs":{"total":95995035648,"available":95174746112,"internally_reserved":0,"allocated":515866624,"data_stored":514834925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":6.7859999999999996,"5min":9.9049999999999994,"15min":5.9029999999999996},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.21099999999999999},"max":{"1min":72.759,"5min":154.62100000000001,"15min":154.62100000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":6.7279999999999998,"5min":10.082000000000001,"15min":5.9210000000000003},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23400000000000001},"max":{"1min":72.704999999999998,"5min":154.37299999999999,"15min":154.37299999999999},"last":3.7679999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:21:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":8.2810000000000006,"15min":5.2080000000000002},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":8.1539999999999999,"5min":135.25399999999999,"15min":135.25399999999999},"last":3.3759999999999999},{"interface":"front","average":{"1min":2.282,"5min":8.1039999999999992,"15min":5.1029999999999998},"min":{"1min":0.25700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":8.5749999999999993,"5min":134.44300000000001,"15min":134.44300000000001},"last":2.8300000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.851,"5min":1.8180000000000001,"15min":2.1560000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":8.202,"5min":8.7400000000000002,"15min":123.598},"last":2.9820000000000002},{"interface":"front","average":{"1min":2.5299999999999998,"5min":1.8440000000000001,"15min":2.1160000000000001},"min":{"1min":0.33400000000000002,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":11.064,"5min":11.064,"15min":123.619},"last":3.0139999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:21:05 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":2.0299999999999998,"15min":2.3279999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.2329999999999997,"5min":9.5310000000000006,"15min":125.90300000000001},"last":2.8679999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.339,"15min":2.5649999999999999},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":11.18,"5min":11.18,"15min":127.90000000000001},"last":3.7280000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:21:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.161,"15min":1.645},"min":{"1min":0.27200000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.4249999999999998,"5min":13.599,"15min":111.809},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.5029999999999999,"15min":1.879},"min":{"1min":0.27900000000000003,"5min":0.252,"15min":0.25},"max":{"1min":8.0589999999999993,"5min":9.1530000000000005,"15min":112.002},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:21:05 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.331,"15min":1.454},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":8.1829999999999998,"5min":8.1829999999999998,"15min":76.307000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.867,"5min":1.1899999999999999,"15min":1.3240000000000001},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.6340000000000003,"5min":8.6340000000000003,"15min":78.421999999999997},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:20:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.3149999999999999,"15min":1.079},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.7789999999999999,"5min":8.7789999999999999,"15min":34.039000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.46,"5min":1.508,"15min":1.194},"min":{"1min":0.38300000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.5430000000000001,"5min":10.384,"15min":33.433999999999997},"last":0.64500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724901,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310180,"kb_used_data":724508,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":92434972,"statfs":{"total":95995035648,"available":94653411328,"internally_reserved":0,"allocated":741896192,"data_stored":740864999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":3.25,"15min":5.1600000000000001},"min":{"1min":0.31,"5min":0.193,"15min":0.193},"max":{"1min":9.3529999999999998,"5min":111.312,"15min":151.23599999999999},"last":0.251},{"interface":"front","average":{"1min":2.4199999999999999,"5min":3.552,"15min":5.5739999999999998},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.22},"max":{"1min":5.8890000000000002,"5min":111.096,"15min":151.566},"last":2.4630000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:21:01 2023","interfaces":[{"interface":"back","average":{"1min":4.9359999999999999,"5min":4.5049999999999999,"15min":4.2999999999999998},"min":{"1min":0.25700000000000001,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":48.781999999999996,"5min":111.194,"15min":111.194},"last":0.38300000000000001},{"interface":"front","average":{"1min":4.077,"5min":4.6539999999999999,"15min":4.6440000000000001},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.23300000000000001},"max":{"1min":48.868000000000002,"5min":111.247,"15min":112.226},"last":2.0249999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7250000000000001,"5min":2.4079999999999999,"15min":3.2759999999999998},"min":{"1min":0.28899999999999998,"5min":0.28699999999999998,"15min":0.23499999999999999},"max":{"1min":5.8579999999999997,"5min":11.324,"15min":106.508},"last":2.4300000000000002},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.8979999999999999,"15min":3.0179999999999998},"min":{"1min":0.28199999999999997,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":5.9340000000000002,"5min":8.5280000000000005,"15min":106.462},"last":2.339}]},{"osd":3,"last update":"Mon Oct 9 04:21:01 2023","interfaces":[{"interface":"back","average":{"1min":4.3710000000000004,"5min":2.5739999999999998,"15min":2.5920000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":41,"5min":41,"15min":91.834999999999994},"last":2.7770000000000001},{"interface":"front","average":{"1min":4.5919999999999996,"5min":2.7999999999999998,"15min":2.702},"min":{"1min":0.33500000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":40.988,"5min":40.988,"15min":91.802000000000007},"last":2.681}]},{"osd":4,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.27,"15min":1.0029999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.4370000000000003,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.309},{"interface":"front","average":{"1min":1.897,"5min":1.2829999999999999,"15min":1.0189999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":7.5890000000000004,"5min":11.297000000000001,"15min":11.297000000000001},"last":0.32100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:21:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.214,"15min":1.4099999999999999},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":6.2519999999999998,"5min":12.068,"15min":123.239},"last":0.443},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.204,"15min":1.4830000000000001},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":3.5369999999999999,"5min":9.7720000000000002,"15min":123.014},"last":0.501}]},{"osd":7,"last update":"Mon Oct 9 04:20:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":1.3899999999999999,"15min":1.109},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3800000000000008,"5min":12.096,"15min":12.096},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.5740000000000001,"15min":1.194},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":5.7309999999999999,"5min":9.9760000000000009,"15min":9.9760000000000009},"last":0.55800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":385875968,"data_stored":385875968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":727126016,"data_stored":727121944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":455540736,"data_stored":455540736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":658374656,"data_stored":658374656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":500961280,"data_stored":500957208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":723390464,"data_stored":723386392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":940249088,"data_stored":940245016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:21:36.793 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:21:36.793 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:21:36.793 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:21:36.793 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:21:36.794 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:21:36.794 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:21:36.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:36.942 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:37.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:37.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:37.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:36 smithi163 ceph-mon[221067]: pgmap v1004: 119 pgs: 119 active+clean; 1.1 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:21:37.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:37.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:37.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:36 smithi163 ceph-mon[213841]: pgmap v1004: 119 pgs: 119 active+clean; 1.1 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:21:37.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:37.059 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:37.090 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:37.151 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:37.218 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:37.241 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:37.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:36 smithi181 ceph-mon[196979]: pgmap v1004: 119 pgs: 119 active+clean; 1.1 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 2.1 MiB/s, 32 objects/s recovering 2023-10-09T04:21:37.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:37.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:37.316 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:21:37.420 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:37.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:37.524 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:37.613 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:38.801 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:38.801 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:38.801 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:38.801 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[213841]: pgmap v1005: 119 pgs: 119 active+clean; 984 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s 2023-10-09T04:21:38.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:38.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:38.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:38.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:38 smithi163 ceph-mon[221067]: pgmap v1005: 119 pgs: 119 active+clean; 984 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s 2023-10-09T04:21:38.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:38.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:38.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:21:38.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:38 smithi181 ceph-mon[196979]: pgmap v1005: 119 pgs: 119 active+clean; 984 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 768 op/s 2023-10-09T04:21:39.073 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:21:39.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:39.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:39.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:39.475 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:39.530 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:21:39.530 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "" 2023-10-09T04:21:39.530 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:21:39.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:39.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:39.687 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:39.774 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:39.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:39.904 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:21:39.905 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:21:39.905 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:21:39.905 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:21:39.905 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:21:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:39.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:40.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:40.150 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:40.200 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:40.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:40.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:40.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:40.330 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:40.801 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:40.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:40.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:40.845 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:21:40.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:40.951 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:21:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:21:40] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2023-10-09T04:21:40.972 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:41.014 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:21:41.041 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:41.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:41.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:41.184 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:41.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:40 smithi163 ceph-mon[221067]: pgmap v1006: 119 pgs: 119 active+clean; 939 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T04:21:41.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3185729589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:41.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:40 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:41.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:40 smithi163 ceph-mon[213841]: pgmap v1006: 119 pgs: 119 active+clean; 939 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T04:21:41.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3185729589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:41.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:40 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:41.256 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:41.261 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:40 smithi181 ceph-mon[196979]: pgmap v1006: 119 pgs: 119 active+clean; 939 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 781 op/s 2023-10-09T04:21:41.261 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3185729589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:41.261 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:40 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:21:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:41.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:41.363 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:41.391 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:41.902 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:41.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:41.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:42.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:42.053 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:42.133 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:42.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:42.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:42.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:42.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:21:42.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: osdmap e369: 8 total, 8 up, 7 in 2023-10-09T04:21:42.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.267 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[221067]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: osdmap e369: 8 total, 8 up, 7 in 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.268 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:41 smithi163 ceph-mon[213841]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: osdmap e369: 8 total, 8 up, 7 in 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:41 smithi181 ceph-mon[196979]: failed to encode map e369 with expected crc 2023-10-09T04:21:42.270 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:42.313 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:42.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:42.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:42.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:42.469 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:42.545 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:42.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:42.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:42.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:42.723 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:42.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:42.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:42.843 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:42.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:42.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:42 smithi163 ceph-mon[213841]: pgmap v1008: 119 pgs: 119 active+clean; 799 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T04:21:42.977 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:42.981 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:42 smithi163 ceph-mon[221067]: pgmap v1008: 119 pgs: 119 active+clean; 799 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T04:21:43.054 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:43.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:43.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:43.214 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:42 smithi181 ceph-mon[196979]: pgmap v1008: 119 pgs: 119 active+clean; 799 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T04:21:43.348 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:43.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:43.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:43.513 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:43.551 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:43.567 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:43.590 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:43.619 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:43.635 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:43.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:43.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:43.680 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:43.729 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:43.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:43.775 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:43.779 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.784 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:21:43.825 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:43.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:43.933 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:43.958 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:44.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:44.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:43 smithi181 ceph-mon[196979]: 1.1b scrub starts 2023-10-09T04:21:44.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:43 smithi181 ceph-mon[196979]: 1.1b scrub ok 2023-10-09T04:21:44.059 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3219822832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:44.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:43 smithi163 ceph-mon[213841]: 1.1b scrub starts 2023-10-09T04:21:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:43 smithi163 ceph-mon[213841]: 1.1b scrub ok 2023-10-09T04:21:44.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3219822832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:44.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:43 smithi163 ceph-mon[221067]: 1.1b scrub starts 2023-10-09T04:21:44.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:43 smithi163 ceph-mon[221067]: 1.1b scrub ok 2023-10-09T04:21:44.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3219822832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:44.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:21:44.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:44.278 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:44.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:44.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:44.686 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:44.711 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:44.797 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:44.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:44.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:44.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:44.982 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:45.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:44 smithi163 ceph-mon[213841]: pgmap v1009: 119 pgs: 119 active+clean; 690 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 794 op/s 2023-10-09T04:21:45.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:45 smithi163 ceph-mon[221067]: pgmap v1009: 119 pgs: 119 active+clean; 690 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 794 op/s 2023-10-09T04:21:45.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:45.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:45.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:44 smithi181 ceph-mon[196979]: pgmap v1009: 119 pgs: 119 active+clean; 690 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 794 op/s 2023-10-09T04:21:45.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:45.310 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:45.374 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:45.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:45.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:45.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:45.547 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:21:45.548 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:21:45.548 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T04:21:45.548 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:21:45.548 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:21:45.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T04:21:45.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:45.644 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:45.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:45.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:45.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:45.808 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:45.827 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:21:45.827 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:21:45.902 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:46.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:46 smithi163 ceph-mon[213841]: 1.1 scrub starts 2023-10-09T04:21:46.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:46 smithi163 ceph-mon[213841]: 1.1 scrub ok 2023-10-09T04:21:46.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:46 smithi163 ceph-mon[221067]: 1.1 scrub starts 2023-10-09T04:21:46.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:46 smithi163 ceph-mon[221067]: 1.1 scrub ok 2023-10-09T04:21:46.122 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:46.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:46.228 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:46.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:46 smithi181 ceph-mon[196979]: 1.1 scrub starts 2023-10-09T04:21:46.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:46 smithi181 ceph-mon[196979]: 1.1 scrub ok 2023-10-09T04:21:46.279 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:46.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:46.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:46.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:47.006 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:47 smithi163 ceph-mon[213841]: pgmap v1010: 119 pgs: 119 active+clean; 674 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T04:21:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:47 smithi163 ceph-mon[221067]: pgmap v1010: 119 pgs: 119 active+clean; 674 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T04:21:47.099 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:47.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:47.175 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:21:47.213 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:47.259 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:47 smithi181 ceph-mon[196979]: pgmap v1010: 119 pgs: 119 active+clean; 674 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T04:21:47.262 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:47.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:21:47.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:47.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:47.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:47.477 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:47.496 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:47.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:47.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:47.659 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:47.700 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:47.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:47.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:47.889 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:21:47.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:47.959 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:48.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:48 smithi163 ceph-mon[221067]: 1.17 scrub starts 2023-10-09T04:21:48.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:48 smithi163 ceph-mon[221067]: 1.17 scrub ok 2023-10-09T04:21:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:48 smithi163 ceph-mon[213841]: 1.17 scrub starts 2023-10-09T04:21:48.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:48 smithi163 ceph-mon[213841]: 1.17 scrub ok 2023-10-09T04:21:48.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:48 smithi181 ceph-mon[196979]: 1.17 scrub starts 2023-10-09T04:21:48.335 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:48 smithi181 ceph-mon[196979]: 1.17 scrub ok 2023-10-09T04:21:48.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:48.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:48.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:48.523 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:48.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:48.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:48.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:48.785 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:48.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:48.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:48.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:48.993 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:49.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[213841]: 1.f scrub starts 2023-10-09T04:21:49.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[213841]: 1.f scrub ok 2023-10-09T04:21:49.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[213841]: pgmap v1011: 119 pgs: 119 active+clean; 483 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 800 op/s 2023-10-09T04:21:49.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3191038005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:49.067 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:49.134 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:49.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:49.218 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:49.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:49.264 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:49.304 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:49.306 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:49 smithi181 ceph-mon[196979]: 1.f scrub starts 2023-10-09T04:21:49.306 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:49 smithi181 ceph-mon[196979]: 1.f scrub ok 2023-10-09T04:21:49.306 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:49 smithi181 ceph-mon[196979]: pgmap v1011: 119 pgs: 119 active+clean; 483 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 800 op/s 2023-10-09T04:21:49.306 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3191038005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:49.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:49.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[221067]: 1.f scrub starts 2023-10-09T04:21:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[221067]: 1.f scrub ok 2023-10-09T04:21:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[221067]: pgmap v1011: 119 pgs: 119 active+clean; 483 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 800 op/s 2023-10-09T04:21:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3191038005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:21:49.408 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:49.455 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:49.610 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:21:49.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:49.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:49.774 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:21:49.775 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T04:21:49.775 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:21:49.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:49.908 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:21:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:21:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: failed to encode map e370 with expected crc 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3191038005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: osdmap e370: 8 total, 8 up, 7 in 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]: dispatch 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]: dispatch 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]: dispatch 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:50.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: failed to encode map e370 with expected crc 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3191038005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: osdmap e370: 8 total, 8 up, 7 in 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]: dispatch 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]: dispatch 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]: dispatch 2023-10-09T04:21:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-10-09T04:21:50.384 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:21:50.384 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: failed to encode map e370 with expected crc 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3191038005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: osdmap e370: 8 total, 8 up, 7 in 2023-10-09T04:21:50.410 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]: dispatch 2023-10-09T04:21:50.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]: dispatch 2023-10-09T04:21:50.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]: dispatch 2023-10-09T04:21:50.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]: dispatch 2023-10-09T04:21:50.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-10-09T04:21:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:21:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:21:50] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-10-09T04:21:50.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:50.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:50.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:51.033 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:51.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: failed to encode map e370 with expected crc 2023-10-09T04:21:51.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: pgmap v1013: 119 pgs: 119 active+clean; 431 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T04:21:51.310 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]': finished 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]': finished 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]': finished 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: osdmap e371: 8 total, 8 up, 7 in 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: failed to encode map e371 with expected crc 2023-10-09T04:21:51.311 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:51 smithi181 ceph-mon[196979]: failed to encode map e371 with expected crc 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: failed to encode map e370 with expected crc 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: pgmap v1013: 119 pgs: 119 active+clean; 431 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]': finished 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]': finished 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]': finished 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: osdmap e371: 8 total, 8 up, 7 in 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: failed to encode map e371 with expected crc 2023-10-09T04:21:51.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[213841]: failed to encode map e371 with expected crc 2023-10-09T04:21:51.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:21:51.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: failed to encode map e370 with expected crc 2023-10-09T04:21:51.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: pgmap v1013: 119 pgs: 119 active+clean; 431 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T04:21:51.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.2", "id": [7, 2]}]': finished 2023-10-09T04:21:51.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.4", "id": [5, 0]}]': finished 2023-10-09T04:21:51.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2023-10-09T04:21:51.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1c", "id": [7, 2]}]': finished 2023-10-09T04:21:51.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-10-09T04:21:51.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: osdmap e371: 8 total, 8 up, 7 in 2023-10-09T04:21:51.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: failed to encode map e371 with expected crc 2023-10-09T04:21:51.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:51 smithi163 ceph-mon[221067]: failed to encode map e371 with expected crc 2023-10-09T04:21:51.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:51.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:51.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:51.445 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:51.504 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:51.555 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:51.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:51.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:51.698 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:51.847 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:51.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:51.916 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:21:51.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:52.017 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:52.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:52.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:52.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:52.246 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:52.329 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:52.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:52.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:52.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:52.649 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:52.728 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:52.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:52.834 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:52.860 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: osdmap e372: 8 total, 8 up, 7 in 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[221067]: pgmap v1016: 119 pgs: 119 active+clean; 294 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: osdmap e372: 8 total, 8 up, 7 in 2023-10-09T04:21:52.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: failed to encode map e372 with expected crc 2023-10-09T04:21:52.862 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:52 smithi163 ceph-mon[213841]: pgmap v1016: 119 pgs: 119 active+clean; 294 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T04:21:52.864 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:52.876 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:52.897 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:52.909 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:52.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:52.917 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:52.936 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:52.954 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:53.007 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:53.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: failed to encode map e372 with expected crc 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: failed to encode map e372 with expected crc 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: osdmap e372: 8 total, 8 up, 7 in 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: failed to encode map e372 with expected crc 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: failed to encode map e372 with expected crc 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: failed to encode map e372 with expected crc 2023-10-09T04:21:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:52 smithi181 ceph-mon[196979]: pgmap v1016: 119 pgs: 119 active+clean; 294 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T04:21:53.045 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:53.077 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:53.083 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:53.090 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.090 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.091 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.091 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.091 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.091 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.091 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.091 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:21:53.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:21:53.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:53.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:21:53.376 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:53.422 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:53.499 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:53.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:53.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:53.645 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:53.686 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:53.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:53.709 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:21:53.790 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:53.853 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:53.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:53.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:53 smithi181 ceph-mon[196979]: failed to encode map e373 with expected crc 2023-10-09T04:21:53.950 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:53 smithi181 ceph-mon[196979]: failed to encode map e373 with expected crc 2023-10-09T04:21:53.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:53 smithi181 ceph-mon[196979]: osdmap e373: 8 total, 8 up, 7 in 2023-10-09T04:21:53.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:53 smithi181 ceph-mon[196979]: failed to encode map e373 with expected crc 2023-10-09T04:21:53.951 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1271704644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:53.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:53.961 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:21:53.961 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:21:53.961 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:21:53.961 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:21:53.961 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:21:54.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:54.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[221067]: failed to encode map e373 with expected crc 2023-10-09T04:21:54.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[221067]: failed to encode map e373 with expected crc 2023-10-09T04:21:54.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[221067]: osdmap e373: 8 total, 8 up, 7 in 2023-10-09T04:21:54.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[221067]: failed to encode map e373 with expected crc 2023-10-09T04:21:54.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1271704644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:54.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[213841]: failed to encode map e373 with expected crc 2023-10-09T04:21:54.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[213841]: failed to encode map e373 with expected crc 2023-10-09T04:21:54.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[213841]: osdmap e373: 8 total, 8 up, 7 in 2023-10-09T04:21:54.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[213841]: failed to encode map e373 with expected crc 2023-10-09T04:21:54.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1271704644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:54.161 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:54.259 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:54.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:54.363 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:54.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:54.423 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:54.539 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:54.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:54.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:54.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:54.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:54 smithi181 ceph-mon[196979]: failed to encode map e374 with expected crc 2023-10-09T04:21:54.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:54 smithi181 ceph-mon[196979]: osdmap e374: 8 total, 8 up, 7 in 2023-10-09T04:21:54.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:54 smithi181 ceph-mon[196979]: failed to encode map e374 with expected crc 2023-10-09T04:21:54.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:54 smithi181 ceph-mon[196979]: failed to encode map e374 with expected crc 2023-10-09T04:21:54.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:54 smithi181 ceph-mon[196979]: failed to encode map e374 with expected crc 2023-10-09T04:21:54.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:54 smithi181 ceph-mon[196979]: pgmap v1019: 119 pgs: 1 active+undersized+remapped, 1 remapped+peering, 117 active+clean; 183 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:21:54.701 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:54.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:54.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:55.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:21:55.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:55.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[221067]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[221067]: osdmap e374: 8 total, 8 up, 7 in 2023-10-09T04:21:55.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[221067]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[221067]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[221067]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.054 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[221067]: pgmap v1019: 119 pgs: 1 active+undersized+remapped, 1 remapped+peering, 117 active+clean; 183 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:21:55.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[213841]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[213841]: osdmap e374: 8 total, 8 up, 7 in 2023-10-09T04:21:55.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[213841]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[213841]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[213841]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:54 smithi163 ceph-mon[213841]: pgmap v1019: 119 pgs: 1 active+undersized+remapped, 1 remapped+peering, 117 active+clean; 183 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:21:55.106 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:55.107 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 98723 objects 2023-10-09T04:21:55.107 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :130.224 2023-10-09T04:21:55.150 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:55.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:55.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:55.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:55.280 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:55.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:55.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:55.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:55.425 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:21:55.450 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:55.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:55.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:55.649 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:55.675 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:55.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:55.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:55.930 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:55 smithi181 ceph-mon[196979]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:55 smithi181 ceph-mon[196979]: osdmap e375: 8 total, 8 up, 7 in 2023-10-09T04:21:55.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:55 smithi181 ceph-mon[196979]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:55 smithi181 ceph-mon[196979]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.931 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:55 smithi181 ceph-mon[196979]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[221067]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[221067]: osdmap e375: 8 total, 8 up, 7 in 2023-10-09T04:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[221067]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[221067]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.968 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[221067]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.968 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[213841]: failed to encode map e374 with expected crc 2023-10-09T04:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[213841]: osdmap e375: 8 total, 8 up, 7 in 2023-10-09T04:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[213841]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[213841]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.969 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:55 smithi163 ceph-mon[213841]: failed to encode map e375 with expected crc 2023-10-09T04:21:55.970 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:56.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:56.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:56.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:21:56.172 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:56.200 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:56.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:56.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:56.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:21:56.373 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:56.470 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:56.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:21:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:56.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:21:56.663 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:21:56.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:21:56.719 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:21:56.719 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:21:56.719 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:21:56.719 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.158252 2023-10-09T04:21:56.720 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.15825169245324644 2023-10-09T04:21:56.741 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:56.836 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:56 smithi181 ceph-mon[196979]: pgmap v1021: 119 pgs: 2 peering, 117 active+clean; 168 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 3.3 KiB/s rd, 8.6 KiB/s wr, 474 op/s; 6.1 KiB/s, 0 objects/s recovering 2023-10-09T04:21:56.836 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:56 smithi181 ceph-mon[196979]: 1.1c deep-scrub starts 2023-10-09T04:21:56.836 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:56 smithi181 ceph-mon[196979]: 1.1c deep-scrub ok 2023-10-09T04:21:56.836 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2000303745' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:56.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:21:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:56.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:21:56.940 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:21:56.964 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[221067]: pgmap v1021: 119 pgs: 2 peering, 117 active+clean; 168 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 3.3 KiB/s rd, 8.6 KiB/s wr, 474 op/s; 6.1 KiB/s, 0 objects/s recovering 2023-10-09T04:21:56.965 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[221067]: 1.1c deep-scrub starts 2023-10-09T04:21:56.965 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[221067]: 1.1c deep-scrub ok 2023-10-09T04:21:56.965 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2000303745' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:56.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[213841]: pgmap v1021: 119 pgs: 2 peering, 117 active+clean; 168 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 3.3 KiB/s rd, 8.6 KiB/s wr, 474 op/s; 6.1 KiB/s, 0 objects/s recovering 2023-10-09T04:21:56.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[213841]: 1.1c deep-scrub starts 2023-10-09T04:21:56.965 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[213841]: 1.1c deep-scrub ok 2023-10-09T04:21:56.966 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2000303745' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:21:56.967 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:57.073 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:21:57.073 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:38929 9.c 9:3774ead0:::benchmark_data_smithi163_464650_object38928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:37.798770+0000", 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 199.27239676100001, 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10792890500000001, 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.074 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38929 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798770+0000", 2023-10-09T04:21:57.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798770+0000", 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798768+0000", 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798826+0000", 2023-10-09T04:21:57.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8443999999999998e-05 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798842+0000", 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5053e-05 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798848+0000", 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2009999999999996e-06 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.798899+0000", 2023-10-09T04:21:57.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1659999999999997e-05 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.799042+0000", 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014276499999999999 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.799190+0000", 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014819799999999999 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.800645+0000", 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014543399999999999 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.801886+0000", 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001241027 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.906605+0000", 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10471944399999999 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.906659+0000", 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3653000000000003e-05 2023-10-09T04:21:57.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:37.906698+0000", 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9570000000000002e-05 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:72489 9.c 9:3680aebc:::benchmark_data_smithi163_464650_object72488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:09.567138+0000", 2023-10-09T04:21:57.083 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.504028393, 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15767242400000001, 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72489 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.084 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567138+0000", 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567138+0000", 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567136+0000", 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567177+0000", 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1226999999999997e-05 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567193+0000", 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5841999999999998e-05 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567201+0000", 2023-10-09T04:21:57.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0819999999999999e-06 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567258+0000", 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6674e-05 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567424+0000", 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016615099999999999 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.567603+0000", 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178467 2023-10-09T04:21:57.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.570436+0000", 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00283358 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573203+0000", 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027669740000000002 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.724713+0000", 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.151509703 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.724770+0000", 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7485000000000003e-05 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.724810+0000", 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0048e-05 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:72498 9.c 9:39f48ce2:::benchmark_data_smithi163_464650_object72497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-10-09T04:21:57.093 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:09.573382+0000", 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 167.49778436599999, 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15170962700000001, 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 72498 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573382+0000", 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573382+0000", 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573381+0000", 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573430+0000", 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9427e-05 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573440+0000", 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0009e-05 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573445+0000", 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8029999999999996e-06 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573490+0000", 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5427999999999999e-05 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573634+0000", 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014404500000000001 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.573736+0000", 2023-10-09T04:21:57.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101561 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.575906+0000", 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021696910000000001 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.576350+0000", 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00044416399999999998 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.725020+0000", 2023-10-09T04:21:57.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14867 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.725056+0000", 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6068000000000001e-05 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:09.725092+0000", 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5540999999999997e-05 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:108780 9.c 9:3a83a127:::benchmark_data_smithi163_464650_object10054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:58.004448+0000", 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 119.066718232, 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26132886500000002, 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.105 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 108780 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004448+0000", 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004448+0000", 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004447+0000", 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004454+0000", 2023-10-09T04:21:57.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9260000000000003e-06 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004471+0000", 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7771e-05 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004478+0000", 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5529999999999998e-06 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004558+0000", 2023-10-09T04:21:57.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0350000000000001e-05 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004664+0000", 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010596599999999999 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.004767+0000", 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010290599999999999 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.007758+0000", 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002990941 2023-10-09T04:21:57.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011875+0000", 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041168719999999997 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.265688+0000", 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25381269299999998 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.265739+0000", 2023-10-09T04:21:57.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1054999999999998e-05 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.265777+0000", 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8141e-05 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.115 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:108783 9.c 9:3eb5faa7:::benchmark_data_smithi163_464650_object10057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:58.011830+0000", 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 119.059336632, 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.26106785100000002, 2023-10-09T04:21:57.116 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 108783 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011830+0000", 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011830+0000", 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011828+0000", 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011835+0000", 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5919999999999997e-06 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011847+0000", 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2425999999999999e-05 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011858+0000", 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0226e-05 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011893+0000", 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5716000000000002e-05 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.011999+0000", 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105548 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.012080+0000", 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1318999999999998e-05 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.014443+0000", 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002362477 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.015079+0000", 2023-10-09T04:21:57.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00063615899999999996 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.272840+0000", 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25776143200000001 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.272873+0000", 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2925000000000001e-05 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.272897+0000", 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4309999999999999e-05 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:108788 9.c 9:310e0d56:::benchmark_data_smithi163_464650_object10062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:58.017051+0000", 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 119.054115647, 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25591207799999999, 2023-10-09T04:21:57.125 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 108788 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017051+0000", 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017051+0000", 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017050+0000", 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017054+0000", 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2930000000000002e-06 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017067+0000", 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2670000000000001e-05 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017071+0000", 2023-10-09T04:21:57.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6120000000000001e-06 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017113+0000", 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1705999999999997e-05 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017188+0000", 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4956000000000005e-05 2023-10-09T04:21:57.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.017257+0000", 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9151000000000003e-05 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.019408+0000", 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021511870000000002 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.019650+0000", 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000241723 2023-10-09T04:21:57.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.272917+0000", 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.25326713000000001 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.272938+0000", 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0619999999999999e-05 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:58.272963+0000", 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4833000000000001e-05 2023-10-09T04:21:57.133 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:109841 9.c 9:36408c6f:::benchmark_data_smithi163_464650_object11115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:59.674530+0000", 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 117.396635778, 2023-10-09T04:21:57.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.128972262, 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 109841 2023-10-09T04:21:57.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674530+0000", 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674530+0000", 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674529+0000", 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674536+0000", 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1029999999999996e-06 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674550+0000", 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3723000000000001e-05 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674556+0000", 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6380000000000001e-06 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674595+0000", 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9341000000000002e-05 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674691+0000", 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6430999999999999e-05 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.674770+0000", 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8413999999999998e-05 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.677218+0000", 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024482369999999998 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.678387+0000", 2023-10-09T04:21:57.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0011693140000000001 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.803388+0000", 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.125000891 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.803450+0000", 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1284000000000005e-05 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:59.803503+0000", 2023-10-09T04:21:57.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3143999999999998e-05 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:116998 9.c 9:3cf9272f:::benchmark_data_smithi163_464650_object18272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:12.348425+0000", 2023-10-09T04:21:57.144 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 104.722740864, 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.220198329, 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116998 2023-10-09T04:21:57.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348425+0000", 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348425+0000", 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348424+0000", 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:21:57.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348429+0000", 2023-10-09T04:21:57.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4859999999999999e-06 2023-10-09T04:21:57.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348438+0000", 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0289999999999994e-06 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348442+0000", 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5519999999999998e-06 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348471+0000", 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8819999999999999e-05 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348552+0000", 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0986e-05 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.348621+0000", 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8570000000000002e-05 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.350878+0000", 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022575130000000001 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.355265+0000", 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043862839999999998 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.568537+0000", 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.213272827 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.568587+0000", 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9678000000000001e-05 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:12.568624+0000", 2023-10-09T04:21:57.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6566999999999997e-05 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:120924 9.c 9:39b1ff0b:::benchmark_data_smithi163_464650_object22198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:17.318704+0000", 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 99.752462446999999, 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11030688, 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.155 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120924 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.318704+0000", 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.318704+0000", 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.318703+0000", 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.318709+0000", 2023-10-09T04:21:57.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5130000000000002e-06 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.318726+0000", 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7419999999999999e-05 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.318733+0000", 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8990000000000004e-06 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320130+0000", 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001396216 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320253+0000", 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000123652 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320370+0000", 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116295 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.325629+0000", 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052589969999999996 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.326152+0000", 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00052376400000000002 2023-10-09T04:21:57.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.428934+0000", 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.102781688 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.428980+0000", 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6306e-05 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.429011+0000", 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.04e-05 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:120925 9.c 9:3538d421:::benchmark_data_smithi163_464650_object22199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:17.320067+0000", 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 99.751099167000007, 2023-10-09T04:21:57.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.112015541, 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120925 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320067+0000", 2023-10-09T04:21:57.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320067+0000", 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320066+0000", 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320072+0000", 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1549999999999998e-06 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320086+0000", 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4858000000000001e-05 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.320093+0000", 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1970000000000001e-06 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.323233+0000", 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003140632 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.323348+0000", 2023-10-09T04:21:57.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114774 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.323418+0000", 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9776000000000004e-05 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.325760+0000", 2023-10-09T04:21:57.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023423459999999999 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.326238+0000", 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047789799999999998 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.431989+0000", 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10575119199999999 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.432041+0000", 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1612e-05 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:17.432083+0000", 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1807000000000001e-05 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:121556 9.c 9:302e5e92:::benchmark_data_smithi163_464650_object22830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:18.218549+0000", 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 98.852616783000002, 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15008998300000001, 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121556 2023-10-09T04:21:57.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218549+0000", 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218549+0000", 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218549+0000", 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218553+0000", 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5199999999999999e-06 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218563+0000", 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9429999999999995e-06 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218568+0000", 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4959999999999998e-06 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218603+0000", 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5407000000000002e-05 2023-10-09T04:21:57.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218671+0000", 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7601999999999994e-05 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.218737+0000", 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6241999999999994e-05 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.220945+0000", 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002207868 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.222148+0000", 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012029740000000001 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.368564+0000", 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14641633400000001 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.368601+0000", 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6820999999999999e-05 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:18.368639+0000", 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8529999999999999e-05 2023-10-09T04:21:57.176 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:122495 9.c 9:3b151cbd:::benchmark_data_smithi163_464650_object23769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:19.574606+0000", 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 97.496559986999998, 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.097344256000000004, 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.177 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 122495 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.574606+0000", 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.574606+0000", 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.574606+0000", 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.574609+0000", 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1729999999999999e-06 2023-10-09T04:21:57.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.574615+0000", 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9540000000000001e-06 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.574619+0000", 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.861e-06 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.577722+0000", 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031036390000000001 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.577847+0000", 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012477500000000001 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.577955+0000", 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000107663 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.583719+0000", 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057643210000000002 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.585069+0000", 2023-10-09T04:21:57.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013501260000000001 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.671872+0000", 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.086803138000000002 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.671913+0000", 2023-10-09T04:21:57.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0617000000000001e-05 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:19.671951+0000", 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7460999999999999e-05 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:122806 9.c 9:302a14ff:::benchmark_data_smithi163_464650_object24080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-10-09T04:21:57.184 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:20.099767+0000", 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 96.971399446999996, 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.190332273, 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 122806 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099767+0000", 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099767+0000", 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099766+0000", 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099771+0000", 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5999999999999997e-06 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099795+0000", 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4318e-05 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099800+0000", 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8210000000000001e-06 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099852+0000", 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1564999999999998e-05 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099931+0000", 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9509000000000001e-05 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.099999+0000", 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7626000000000006e-05 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.102258+0000", 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022593130000000002 2023-10-09T04:21:57.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.103059+0000", 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00080039100000000001 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.290017+0000", 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.18695835 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.290064+0000", 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7070999999999997e-05 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:20.290099+0000", 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4968000000000001e-05 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:123807 9.c 9:38149e3a:::benchmark_data_smithi163_464650_object25081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:21.671652+0000", 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 95.399513838000004, 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.208920409, 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 123807 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.192 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671652+0000", 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671652+0000", 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671651+0000", 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671661+0000", 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.007e-05 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671671+0000", 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9110000000000005e-06 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671676+0000", 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.237e-06 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671719+0000", 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2466000000000003e-05 2023-10-09T04:21:57.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671837+0000", 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011786999999999999 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.671950+0000", 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113239 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.677664+0000", 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057143519999999998 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.678600+0000", 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00093630799999999995 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.880478+0000", 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.20187749899999999 2023-10-09T04:21:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.880535+0000", 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7195999999999999e-05 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:21.880573+0000", 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7691000000000002e-05 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.198 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:154335 9.c 9:338e1649:::benchmark_data_smithi163_464650_object55609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:00.567530+0000", 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 56.503636116000003, 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15481107199999999, 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.199 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 154335 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567530+0000", 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567530+0000", 2023-10-09T04:21:57.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567529+0000", 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567534+0000", 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3519999999999997e-06 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567549+0000", 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4391e-05 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567554+0000", 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4160000000000003e-06 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567602+0000", 2023-10-09T04:21:57.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7849000000000002e-05 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567738+0000", 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013620099999999999 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.567875+0000", 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000136649 2023-10-09T04:21:57.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.570682+0000", 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028067249999999999 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.571418+0000", 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00073691800000000003 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.722258+0000", 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.150839535 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.722306+0000", 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7975000000000001e-05 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:00.722341+0000", 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5212000000000001e-05 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:154814 9.c 9:347405fb:::benchmark_data_smithi163_464650_object56088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e362)", 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:01.189770+0000", 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 55.881396375000001, 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.125489093, 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.207 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 154814 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189770+0000", 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189770+0000", 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189769+0000", 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189774+0000", 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5799999999999999e-06 2023-10-09T04:21:57.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189789+0000", 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4545999999999999e-05 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189802+0000", 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3033e-05 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189832+0000", 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0360000000000001e-05 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.189946+0000", 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011421699999999999 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.190027+0000", 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0298999999999998e-05 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.194442+0000", 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044154060000000002 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.194469+0000", 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6998000000000001e-05 2023-10-09T04:21:57.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.315165+0000", 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.120696267 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.315216+0000", 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0257999999999999e-05 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:01.315259+0000", 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3322999999999998e-05 2023-10-09T04:21:57.215 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:166471 9.c 9:32d8a513:::benchmark_data_smithi163_464650_object67745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:15.967060+0000", 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 41.104106016000003, 2023-10-09T04:21:57.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.1237473, 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 166471 2023-10-09T04:21:57.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.967060+0000", 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.967060+0000", 2023-10-09T04:21:57.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.967060+0000", 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.967062+0000", 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4880000000000001e-06 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.967068+0000", 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8749999999999997e-06 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.967072+0000", 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6959999999999998e-06 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.972788+0000", 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057164049999999999 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.972921+0000", 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013297299999999999 2023-10-09T04:21:57.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.973032+0000", 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111097 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975807+0000", 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002774838 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.980490+0000", 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046831590000000001 2023-10-09T04:21:57.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.090721+0000", 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.110230933 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.090768+0000", 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6513e-05 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.090808+0000", 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9715000000000002e-05 2023-10-09T04:21:57.225 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:166516 9.c 9:34519eea:::benchmark_data_smithi163_464650_object67790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:16.118581+0000", 2023-10-09T04:21:57.226 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 40.952584846000001, 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13158829699999999, 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 166516 2023-10-09T04:21:57.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118581+0000", 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118581+0000", 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118579+0000", 2023-10-09T04:21:57.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999976 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118591+0000", 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2079000000000001e-05 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118624+0000", 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2339999999999999e-05 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118636+0000", 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2816000000000001e-05 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118725+0000", 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8539000000000001e-05 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.118983+0000", 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00025811600000000002 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.119058+0000", 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4412999999999994e-05 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.121863+0000", 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028058010000000001 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.121867+0000", 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0049999999999998e-06 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.250062+0000", 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.128194533 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.250122+0000", 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9957999999999998e-05 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.250170+0000", 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7880000000000002e-05 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:185165 9.c 9:3b93fef9:::benchmark_data_smithi163_464650_object86439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:40.023061+0000", 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.048105727999999, 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.115144182, 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185165 2023-10-09T04:21:57.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023061+0000", 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023061+0000", 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023059+0000", 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023068+0000", 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9069999999999996e-06 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023084+0000", 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6113e-05 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023090+0000", 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.229e-06 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023132+0000", 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1924e-05 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023245+0000", 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112576 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.023333+0000", 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8183999999999995e-05 2023-10-09T04:21:57.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.026454+0000", 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031211199999999998 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035399+0000", 2023-10-09T04:21:57.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089449639999999997 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.138113+0000", 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.10271441100000001 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.138165+0000", 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1245999999999997e-05 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.138205+0000", 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0092000000000002e-05 2023-10-09T04:21:57.244 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:185175 9.c 9:322621ab:::benchmark_data_smithi163_464650_object86449:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:40.035841+0000", 2023-10-09T04:21:57.245 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 17.035324865, 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.102459644, 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 185175 2023-10-09T04:21:57.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035841+0000", 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035841+0000", 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035840+0000", 2023-10-09T04:21:57.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035848+0000", 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8180000000000004e-06 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035879+0000", 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1448000000000001e-05 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035886+0000", 2023-10-09T04:21:57.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8109999999999997e-06 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.035923+0000", 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7258000000000001e-05 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.036016+0000", 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3177999999999995e-05 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.036096+0000", 2023-10-09T04:21:57.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9721999999999993e-05 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.038485+0000", 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023888479999999998 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.038784+0000", 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000299155 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.138237+0000", 2023-10-09T04:21:57.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.099452809000000003 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.138271+0000", 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3485000000000002e-05 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:21:57.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:40.138301+0000", 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0524000000000001e-05 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:21:57.256 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:21:57.257 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:21:57.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:21:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:57.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:21:57.721 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:57.750 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:57.804 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:21:57.897 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:57.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:57.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[221067]: failed to encode map e376 with expected crc 2023-10-09T04:21:57.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[221067]: failed to encode map e376 with expected crc 2023-10-09T04:21:57.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2000303745' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:21:57.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[221067]: osdmap e376: 8 total, 8 up, 7 in 2023-10-09T04:21:57.899 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[221067]: failed to encode map e376 with expected crc 2023-10-09T04:21:57.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:57.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[213841]: failed to encode map e376 with expected crc 2023-10-09T04:21:57.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[213841]: failed to encode map e376 with expected crc 2023-10-09T04:21:57.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2000303745' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:21:57.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[213841]: osdmap e376: 8 total, 8 up, 7 in 2023-10-09T04:21:57.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:57 smithi163 ceph-mon[213841]: failed to encode map e376 with expected crc 2023-10-09T04:21:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:57.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:58.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:58.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:57 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:21:58.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:57 smithi181 ceph-mon[196979]: failed to encode map e376 with expected crc 2023-10-09T04:21:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:57 smithi181 ceph-mon[196979]: failed to encode map e376 with expected crc 2023-10-09T04:21:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2000303745' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:21:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:57 smithi181 ceph-mon[196979]: osdmap e376: 8 total, 8 up, 7 in 2023-10-09T04:21:58.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:57 smithi181 ceph-mon[196979]: failed to encode map e376 with expected crc 2023-10-09T04:21:58.028 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:58.151 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:21:58.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:58.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:58.302 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:21:58.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:21:58.382 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:58.427 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:21:58.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:58.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:21:58.532 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:58.561 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:58.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:58.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:21:58.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:58 smithi163 ceph-mon[221067]: pgmap v1023: 119 pgs: 2 peering, 117 active+clean; 16 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 2.7 KiB/s rd, 7.0 KiB/s wr, 659 op/s; 5.0 KiB/s, 0 objects/s recovering 2023-10-09T04:21:58.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:58 smithi163 ceph-mon[213841]: pgmap v1023: 119 pgs: 2 peering, 117 active+clean; 16 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 2.7 KiB/s rd, 7.0 KiB/s wr, 659 op/s; 5.0 KiB/s, 0 objects/s recovering 2023-10-09T04:21:58.862 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:21:58.889 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:58.906 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:21:58.925 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:21:58.944 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:21:58.963 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:21:58.981 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:21:58.999 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:21:59.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:21:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:58 smithi181 ceph-mon[196979]: pgmap v1023: 119 pgs: 2 peering, 117 active+clean; 16 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 2.7 KiB/s rd, 7.0 KiB/s wr, 659 op/s; 5.0 KiB/s, 0 objects/s recovering 2023-10-09T04:21:59.039 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.041 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:21:59.113 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:21:59.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:59.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:21:59.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:21:59.246 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:59.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:21:59.617 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:21:59.617 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:21:59.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:21:59.794 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:21:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/636399274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:59.794 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:21:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/636399274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:59.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:21:59.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:21:59.922 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:21:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/636399274' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:21:59.926 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:21:59.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:21:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:21:59.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:00.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:00.085 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:00.099 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:00.210 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T04:22:00.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T04:22:00.343 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:00.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:00.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:00.353 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:22:00.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:00.477 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:00.543 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:00.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:00.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:00.679 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:01.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:00 smithi163 ceph-mon[221067]: pgmap v1024: 119 pgs: 119 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 464 op/s 2023-10-09T04:22:01.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:00 smithi163 ceph-mon[213841]: pgmap v1024: 119 pgs: 119 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 464 op/s 2023-10-09T04:22:01.065 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:22:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:22:00] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-10-09T04:22:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:01.065 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:01.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:01.066 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:01.066 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:00 smithi181 ceph-mon[196979]: pgmap v1024: 119 pgs: 119 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 464 op/s 2023-10-09T04:22:01.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:02.937 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2372528135' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.15825169245324644}]: dispatch 2023-10-09T04:22:02.937 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:22:02.938 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:22:02.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: failed to encode map e377 with expected crc 2023-10-09T04:22:02.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: failed to encode map e377 with expected crc 2023-10-09T04:22:02.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2372528135' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.15825169245324644}]': finished 2023-10-09T04:22:02.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: osdmap e377: 8 total, 8 up, 7 in 2023-10-09T04:22:02.938 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: pgmap v1026: 119 pgs: 119 active+clean; 582 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 401 op/s 2023-10-09T04:22:02.939 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3388590626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:02.939 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:02 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:03.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:22:03.098 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:03.098 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.1 primary-affinity to 0.158252 (8103712) 2023-10-09T04:22:03.099 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2372528135' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.15825169245324644}]: dispatch 2023-10-09T04:22:03.099 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2372528135' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.15825169245324644}]: dispatch 2023-10-09T04:22:03.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:22:03.102 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: failed to encode map e377 with expected crc 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: failed to encode map e377 with expected crc 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2372528135' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.15825169245324644}]': finished 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: osdmap e377: 8 total, 8 up, 7 in 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: pgmap v1026: 119 pgs: 119 active+clean; 582 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 401 op/s 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3388590626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:03.103 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: failed to encode map e377 with expected crc 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: failed to encode map e377 with expected crc 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2372528135' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.15825169245324644}]': finished 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: osdmap e377: 8 total, 8 up, 7 in 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: pgmap v1026: 119 pgs: 119 active+clean; 582 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 401 op/s 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3388590626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:03.104 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:02 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:03.126 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:03.171 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:03.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:03.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:03.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:03.309 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:03.471 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:03.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:03.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:03.704 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:03.732 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:03.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:03.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: osdmap e378: 8 total, 8 up, 7 in 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.875 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[213841]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: osdmap e378: 8 total, 8 up, 7 in 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.876 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:03 smithi163 ceph-mon[221067]: failed to encode map e378 with expected crc 2023-10-09T04:22:03.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:03.907 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:03.940 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:04.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: osdmap e378: 8 total, 8 up, 7 in 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:03 smithi181 ceph-mon[196979]: failed to encode map e378 with expected crc 2023-10-09T04:22:04.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:04.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:04.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:04.236 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:04.481 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:04.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:04.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:04.611 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:04.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:04.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:04.694 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_3' removed 2023-10-09T04:22:04.764 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:04.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:04 smithi163 ceph-mon[221067]: pgmap v1028: 119 pgs: 2 peering, 117 active+clean; 591 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 129 op/s 2023-10-09T04:22:04.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3627118049' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:22:04.768 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:04.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:04 smithi163 ceph-mon[213841]: pgmap v1028: 119 pgs: 2 peering, 117 active+clean; 591 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 129 op/s 2023-10-09T04:22:04.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3627118049' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:22:04.768 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:04.791 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:04.830 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:04 smithi181 ceph-mon[196979]: pgmap v1028: 119 pgs: 2 peering, 117 active+clean; 591 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 129 op/s 2023-10-09T04:22:04.830 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3627118049' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:22:04.830 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:04.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:04.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:04.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:05.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:05.098 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:22:05.098 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:22:05.122 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:05.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:05.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:05.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:05.401 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:05.444 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:05.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:05.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:05.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:05.565 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:22:05.566 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:22:05.566 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T04:22:05.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T04:22:05.580 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:05.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:22:05.802 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:05.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[221067]: failed to encode map e379 with expected crc 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[221067]: failed to encode map e379 with expected crc 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3627118049' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[221067]: osdmap e379: 8 total, 8 up, 7 in 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[221067]: failed to encode map e379 with expected crc 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:22:05.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[213841]: failed to encode map e379 with expected crc 2023-10-09T04:22:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[213841]: failed to encode map e379 with expected crc 2023-10-09T04:22:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3627118049' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:22:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[213841]: osdmap e379: 8 total, 8 up, 7 in 2023-10-09T04:22:05.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:05 smithi163 ceph-mon[213841]: failed to encode map e379 with expected crc 2023-10-09T04:22:05.860 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:22:05.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:22:05.906 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:05.926 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:22:05.943 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:05.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:22:05.988 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:22:05.998 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:06.010 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:22:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:05 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:22:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:05 smithi181 ceph-mon[196979]: failed to encode map e379 with expected crc 2023-10-09T04:22:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:05 smithi181 ceph-mon[196979]: failed to encode map e379 with expected crc 2023-10-09T04:22:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3627118049' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:22:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:05 smithi181 ceph-mon[196979]: osdmap e379: 8 total, 8 up, 7 in 2023-10-09T04:22:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:05 smithi181 ceph-mon[196979]: failed to encode map e379 with expected crc 2023-10-09T04:22:06.026 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:06.032 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:22:06.050 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:06.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:06.057 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:22:06.077 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:22:06.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:06.187 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:06.210 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:06.234 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:06.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:06.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:06.365 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:06.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:06.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:06.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:06.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:06.590 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:06 smithi181 ceph-mon[196979]: failed to encode map e379 with expected crc 2023-10-09T04:22:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4192927629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:06 smithi181 ceph-mon[196979]: pgmap v1030: 103 pgs: 2 peering, 101 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 10 KiB/s rd, 5 op/s 2023-10-09T04:22:06.955 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:06 smithi163 ceph-mon[221067]: failed to encode map e379 with expected crc 2023-10-09T04:22:06.955 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4192927629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:06.955 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:06 smithi163 ceph-mon[221067]: pgmap v1030: 103 pgs: 2 peering, 101 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 10 KiB/s rd, 5 op/s 2023-10-09T04:22:06.955 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:06 smithi163 ceph-mon[213841]: failed to encode map e379 with expected crc 2023-10-09T04:22:06.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4192927629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:06.956 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:06 smithi163 ceph-mon[213841]: pgmap v1030: 103 pgs: 2 peering, 101 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 10 KiB/s rd, 5 op/s 2023-10-09T04:22:07.308 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:07.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:07.411 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:07.441 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:07.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:07.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:07.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:07.668 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:07.710 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:22:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:22:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:22:07.710 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:22:08.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:08.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:08.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:08.103 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:22:08.103 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:22:08.103 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T04:22:08.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T04:22:08.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:08.174 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:08.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:08.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:08.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:22:08.491 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:08.518 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:08.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:08.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:08.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:08.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:08.696 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:08 smithi163 ceph-mon[221067]: pgmap v1031: 103 pgs: 103 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 9.8 KiB/s rd, 4 op/s 2023-10-09T04:22:08.696 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:08 smithi163 ceph-mon[213841]: pgmap v1031: 103 pgs: 103 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 9.8 KiB/s rd, 4 op/s 2023-10-09T04:22:08.700 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:08.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:08 smithi181 ceph-mon[196979]: pgmap v1031: 103 pgs: 103 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 9.8 KiB/s rd, 4 op/s 2023-10-09T04:22:08.774 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:08.802 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:08.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:08.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:08.923 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:08.934 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:08.978 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:08.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:09.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:09.110 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:09.133 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:22:09.165 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:09.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:09.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:09.298 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:09.392 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:09.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:09.495 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:09.522 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:09.591 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_4' created 2023-10-09T04:22:09.591 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:22:09.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:22:09.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1589551985' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:22:09.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:22:09.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3700996056' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:09.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:22:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1589551985' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:22:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:22:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3700996056' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:09.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:09 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:22:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1589551985' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:22:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:22:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3700996056' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:10.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:10.062 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:10.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:10.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:10.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:10.193 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:10.284 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:22:10.284 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:22:10.284 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:22:10.284 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:22:10.284 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:22:10.339 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:10.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:10.444 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:10.473 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:10.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:10.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: failed to encode map e380 with expected crc 2023-10-09T04:22:10.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:22:10.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:22:10.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: osdmap e380: 8 total, 8 up, 7 in 2023-10-09T04:22:10.578 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: failed to encode map e380 with expected crc 2023-10-09T04:22:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:22:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:22:10] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.43.0" 2023-10-09T04:22:10.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[213841]: pgmap v1033: 119 pgs: 16 unknown, 103 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 6.5 KiB/s rd, 3 op/s 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: failed to encode map e380 with expected crc 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: osdmap e380: 8 total, 8 up, 7 in 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: failed to encode map e380 with expected crc 2023-10-09T04:22:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:10 smithi163 ceph-mon[221067]: pgmap v1033: 119 pgs: 16 unknown, 103 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 6.5 KiB/s rd, 3 op/s 2023-10-09T04:22:10.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T04:22:11.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:11.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: failed to encode map e380 with expected crc 2023-10-09T04:22:11.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:22:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:22:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: osdmap e380: 8 total, 8 up, 7 in 2023-10-09T04:22:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: failed to encode map e380 with expected crc 2023-10-09T04:22:11.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:10 smithi181 ceph-mon[196979]: pgmap v1033: 119 pgs: 16 unknown, 103 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 6.5 KiB/s rd, 3 op/s 2023-10-09T04:22:11.052 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:11.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:11.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:22:11.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:11.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:11.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:11.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:11.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:11.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:11.394 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:11.427 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:11.441 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:11.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:11.480 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:11.494 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:11.547 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:11.581 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:11.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T04:22:11.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.669 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.670 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.670 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: 1.b scrub starts 2023-10-09T04:22:11.670 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:11 smithi181 ceph-mon[196979]: 1.b scrub ok 2023-10-09T04:22:11.689 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:11.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:11.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:11.822 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: 1.b scrub starts 2023-10-09T04:22:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[221067]: 1.b scrub ok 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: osdmap e381: 8 total, 8 up, 7 in 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: failed to encode map e381 with expected crc 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: 1.b scrub starts 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 ceph-mon[213841]: 1.b scrub ok 2023-10-09T04:22:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:11 smithi163 conmon[213817]: 2023-10-09T04:22:11.656+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e382 definitely_dead 0 2023-10-09T04:22:12.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:12.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:12.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:12.198 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:12.458 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:12.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:12.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:12.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:12.605 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:12.618 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.6. 2023-10-09T04:22:12.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:12 smithi181 ceph-mon[196979]: failed to encode map e382 with expected crc 2023-10-09T04:22:12.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:12 smithi181 ceph-mon[196979]: failed to encode map e382 with expected crc 2023-10-09T04:22:12.704 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:12 smithi181 ceph-mon[196979]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T04:22:12.705 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4092341787' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:22:12.705 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:12 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:22:12.705 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:12 smithi181 ceph-mon[196979]: pgmap v1036: 119 pgs: 1 creating+activating, 5 creating+peering, 7 unknown, 106 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:22:12.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:12.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[221067]: failed to encode map e382 with expected crc 2023-10-09T04:22:12.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[221067]: failed to encode map e382 with expected crc 2023-10-09T04:22:12.877 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[221067]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4092341787' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[221067]: pgmap v1036: 119 pgs: 1 creating+activating, 5 creating+peering, 7 unknown, 106 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[213841]: failed to encode map e382 with expected crc 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[213841]: failed to encode map e382 with expected crc 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[213841]: osdmap e382: 8 total, 8 up, 7 in 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4092341787' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:22:12.878 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:22:12.879 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:12 smithi163 ceph-mon[213841]: pgmap v1036: 119 pgs: 1 creating+activating, 5 creating+peering, 7 unknown, 106 active+clean; 621 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:22:12.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:12.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:12.933 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:22:12.933 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:70984 8.5 8:a6ee6452:::benchmark_data_smithi163_404719_object70983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.698687+0000", 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.23223645799999, 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.04172793, 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.934 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 70984 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698687+0000", 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698687+0000", 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698685+0000", 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698722+0000", 2023-10-09T04:22:12.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7101999999999998e-05 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698736+0000", 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4136e-05 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698744+0000", 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8310000000000001e-06 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698839+0000", 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4968000000000003e-05 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.698852+0000", 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3225e-05 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.714871+0000", 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0160188429999999 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.714980+0000", 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109119 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.715089+0000", 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010936499999999999 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722342+0000", 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0072527260000000001 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740274+0000", 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017932251999999999 2023-10-09T04:22:12.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740329+0000", 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4815000000000003e-05 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740389+0000", 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9481999999999997e-05 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740415+0000", 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5911e-05 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:70992 8.5 8:ad8cc80d:::benchmark_data_smithi163_404719_object70991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.709201+0000", 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.22172168100002, 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.031350838, 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.942 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 70992 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709201+0000", 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709201+0000", 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709200+0000", 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709245+0000", 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5425e-05 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709259+0000", 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3689999999999999e-05 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709264+0000", 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0710000000000002e-06 2023-10-09T04:22:12.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709315+0000", 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1563000000000001e-05 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.709322+0000", 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1890000000000004e-06 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.716865+0000", 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.007543503 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.717004+0000", 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013937699999999999 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.717178+0000", 2023-10-09T04:22:12.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017350699999999999 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725150+0000", 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079721570000000005 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740456+0000", 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015305536999999999 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740498+0000", 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2141999999999997e-05 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740530+0000", 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1856000000000001e-05 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740552+0000", 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2501999999999999e-05 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:70994 8.5 8:ac3f1792:::benchmark_data_smithi163_404719_object70993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.711696+0000", 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.21922676100002, 2023-10-09T04:22:12.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.029056387, 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 70994 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711696+0000", 2023-10-09T04:22:12.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711696+0000", 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711695+0000", 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711742+0000", 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7809999999999998e-05 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711757+0000", 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4721e-05 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711763+0000", 2023-10-09T04:22:12.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7080000000000002e-06 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711829+0000", 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.567e-05 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.711840+0000", 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1269e-05 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.719011+0000", 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0071713149999999 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.719112+0000", 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101012 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.719209+0000", 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6763999999999997e-05 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725161+0000", 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059519120000000002 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740595+0000", 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015433733999999999 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740711+0000", 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011607699999999999 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740734+0000", 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3340999999999999e-05 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740753+0000", 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8683999999999999e-05 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71006 8.5 8:a0595674:::benchmark_data_smithi163_404719_object71005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.721327+0000", 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.20959633699999, 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.019527788, 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.958 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71006 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721327+0000", 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721327+0000", 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721325+0000", 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721358+0000", 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2787999999999998e-05 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721368+0000", 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.747e-06 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721373+0000", 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.972e-06 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721444+0000", 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1616e-05 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.721450+0000", 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7810000000000002e-06 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.720425+0000", 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99897466800000001 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.720564+0000", 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138923 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.720733+0000", 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016904999999999999 2023-10-09T04:22:12.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726337+0000", 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005604263 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740663+0000", 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014326271999999999 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740787+0000", 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012382499999999999 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740821+0000", 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3371999999999998e-05 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740854+0000", 2023-10-09T04:22:12.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3926e-05 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71026 8.5 8:ae6fe286:::benchmark_data_smithi163_404719_object71025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.745442+0000", 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.18548059400001, 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99553465500000005, 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.966 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71026 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745442+0000", 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745442+0000", 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745441+0000", 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745496+0000", 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5269999999999998e-05 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745515+0000", 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8793000000000001e-05 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745524+0000", 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2459999999999992e-06 2023-10-09T04:22:12.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745623+0000", 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8352999999999998e-05 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.745638+0000", 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5029e-05 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722389+0000", 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97675139099999997 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722520+0000", 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013034200000000001 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.722669+0000", 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014938199999999999 2023-10-09T04:22:12.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726350+0000", 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036812239999999999 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740688+0000", 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014338138 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740917+0000", 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022848700000000001 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740945+0000", 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8660999999999999e-05 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740977+0000", 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1699000000000003e-05 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71032 8.5 8:a57fa36f:::benchmark_data_smithi163_404719_object71031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.755072+0000", 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.17585153200002, 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98602111000000003, 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.974 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71032 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755072+0000", 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755072+0000", 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755070+0000", 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755156+0000", 2023-10-09T04:22:12.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5607999999999998e-05 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755176+0000", 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9936e-05 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755183+0000", 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4549999999999998e-06 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755257+0000", 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4425000000000007e-05 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.755268+0000", 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0352e-05 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.723798+0000", 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.96853069899999999 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.723932+0000", 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013312900000000001 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.724093+0000", 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016132000000000001 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.729917+0000", 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0058235860000000004 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740889+0000", 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010972812 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741018+0000", 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012877900000000001 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741054+0000", 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6196999999999999e-05 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741093+0000", 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8302e-05 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.981 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71064 8.5 8:a22115a5:::benchmark_data_smithi163_404719_object71063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.795912+0000", 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.13501104800002, 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94542275099999995, 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.982 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71064 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795912+0000", 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795912+0000", 2023-10-09T04:22:12.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795910+0000", 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.795983+0000", 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3238999999999995e-05 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796003+0000", 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9522999999999999e-05 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796012+0000", 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6719999999999993e-06 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796072+0000", 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9982999999999998e-05 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.796081+0000", 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.611e-06 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725178+0000", 2023-10-09T04:22:12.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92909712700000002 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725247+0000", 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8887999999999995e-05 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.725329+0000", 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2138999999999994e-05 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733054+0000", 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077248730000000002 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741159+0000", 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081048920000000007 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741241+0000", 2023-10-09T04:22:12.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2134999999999999e-05 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741297+0000", 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6091999999999998e-05 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741335+0000", 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7395999999999999e-05 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.989 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71093 8.5 8:aad31fa0:::benchmark_data_smithi163_404719_object71092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.832717+0000", 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.098206285, 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.908734229, 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.990 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71093 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832717+0000", 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832717+0000", 2023-10-09T04:22:12.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832715+0000", 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832756+0000", 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0484000000000001e-05 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832777+0000", 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1484999999999998e-05 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832785+0000", 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6380000000000004e-06 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832872+0000", 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6444000000000001e-05 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.832879+0000", 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1500000000000002e-06 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726395+0000", 2023-10-09T04:22:12.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89351601599999997 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726520+0000", 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125439 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.726687+0000", 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166446 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733064+0000", 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0063777189999999996 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741213+0000", 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081485700000000008 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:12.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741367+0000", 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015453900000000001 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741419+0000", 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1666999999999998e-05 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741451+0000", 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1931000000000003e-05 2023-10-09T04:22:12.997 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71111 8.5 8:a84d3fc2:::benchmark_data_smithi163_404719_object71110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.852904+0000", 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 442.07801855600002, 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88869594600000001, 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:12.998 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71111 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852904+0000", 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:12.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852904+0000", 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852896+0000", 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999914 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852939+0000", 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3155000000000003e-05 2023-10-09T04:22:13.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852956+0000", 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6302000000000001e-05 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.852962+0000", 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6309999999999997e-06 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.853035+0000", 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3224000000000002e-05 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.853047+0000", 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1993e-05 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.728640+0000", 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87559244700000005 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.728728+0000", 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8270999999999996e-05 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.728824+0000", 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6330999999999996e-05 2023-10-09T04:22:13.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736127+0000", 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0073024980000000001 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741496+0000", 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053695189999999997 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741531+0000", 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4558999999999999e-05 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741571+0000", 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9604000000000002e-05 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741600+0000", 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9832000000000002e-05 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71156 8.5 8:af4bd142:::benchmark_data_smithi163_404719_object71155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.932452+0000", 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 441.99847136699998, 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80928035600000003, 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.006 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71156 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932452+0000", 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932452+0000", 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932450+0000", 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932530+0000", 2023-10-09T04:22:13.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0531000000000005e-05 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932547+0000", 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.679e-05 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932554+0000", 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5010000000000001e-06 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932616+0000", 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2164999999999999e-05 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.932622+0000", 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5529999999999998e-06 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.729963+0000", 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79734061300000003 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.730088+0000", 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012476699999999999 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.730195+0000", 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010767 2023-10-09T04:22:13.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736133+0000", 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059373910000000002 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741630+0000", 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0054977200000000002 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741682+0000", 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1603e-05 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741710+0000", 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7708999999999998e-05 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741732+0000", 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2283999999999999e-05 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71164 8.5 8:a61e31e7:::benchmark_data_smithi163_404719_object71163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.938922+0000", 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 441.99200083599999, 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80291156200000002, 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.014 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71164 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938922+0000", 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938922+0000", 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938921+0000", 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938963+0000", 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2756999999999997e-05 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938981+0000", 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7232e-05 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.938988+0000", 2023-10-09T04:22:13.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5429999999999996e-06 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.939059+0000", 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0704000000000006e-05 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.939071+0000", 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1817000000000001e-05 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.731693+0000", 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79262240100000003 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.731779+0000", 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5703000000000003e-05 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.731887+0000", 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108614 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737400+0000", 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055126380000000003 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741663+0000", 2023-10-09T04:22:13.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0042629709999999999 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741767+0000", 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103897 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741805+0000", 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8052000000000001e-05 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741834+0000", 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8707000000000001e-05 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71188 8.5 8:a7ef5618:::benchmark_data_smithi163_404719_object71187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:50.981878+0000", 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 441.949045528, 2023-10-09T04:22:13.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.76008435900000004, 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71188 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981878+0000", 2023-10-09T04:22:13.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981878+0000", 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981876+0000", 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981907+0000", 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0701999999999998e-05 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981923+0000", 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5843000000000001e-05 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981929+0000", 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981979+0000", 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0614e-05 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:50.981987+0000", 2023-10-09T04:22:13.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5399999999999998e-06 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733108+0000", 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.751121025 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733221+0000", 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000112824 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.733319+0000", 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8542999999999995e-05 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737429+0000", 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041098580000000001 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741863+0000", 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044344629999999996 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741896+0000", 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2789000000000001e-05 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741933+0000", 2023-10-09T04:22:13.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6572999999999997e-05 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.741962+0000", 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9087000000000001e-05 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.030 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71212 8.5 8:a8a8c0bd:::benchmark_data_smithi163_404719_object71211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:51.023176+0000", 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 441.90774659800002, 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.71901980099999996, 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71212 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023176+0000", 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023176+0000", 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023175+0000", 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023241+0000", 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6810000000000006e-05 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023258+0000", 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6670000000000001e-05 2023-10-09T04:22:13.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023265+0000", 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.939e-06 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023330+0000", 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4519999999999999e-05 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.023340+0000", 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0241000000000001e-05 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.734849+0000", 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.71150922500000002 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.734932+0000", 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2621999999999996e-05 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.735027+0000", 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5351000000000002e-05 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.738674+0000", 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036465289999999999 2023-10-09T04:22:13.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742078+0000", 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034042859999999999 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742149+0000", 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1057999999999996e-05 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742177+0000", 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7776999999999998e-05 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742196+0000", 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9616e-05 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71222 8.5 8:a9184906:::benchmark_data_smithi163_404719_object71221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:51.037493+0000", 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 441.893429948, 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.70486150800000003, 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71222 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037493+0000", 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037493+0000", 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037491+0000", 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037563+0000", 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1388000000000002e-05 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037578+0000", 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5000999999999999e-05 2023-10-09T04:22:13.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037585+0000", 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7549999999999997e-06 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037641+0000", 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6799999999999998e-05 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.037650+0000", 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3629999999999994e-06 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736153+0000", 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.69850320200000005 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736236+0000", 2023-10-09T04:22:13.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2647999999999999e-05 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.736354+0000", 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118429 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.739932+0000", 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003577724 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742128+0000", 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021960819999999998 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742281+0000", 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152984 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742324+0000", 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3430999999999998e-05 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742355+0000", 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0292999999999999e-05 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:71234 8.5 8:acb150b2:::benchmark_data_smithi163_404719_object71233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:14:51.067468+0000", 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 441.86345549800001, 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.67751995300000001, 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 71234 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.047 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067468+0000", 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067468+0000", 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067466+0000", 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067516+0000", 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9421e-05 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067528+0000", 2023-10-09T04:22:13.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2018000000000001e-05 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067533+0000", 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7140000000000001e-06 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067589+0000", 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6066000000000002e-05 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.067598+0000", 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0299999999999999e-06 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737452+0000", 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.66985455400000005 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737525+0000", 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3143000000000001e-05 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.737618+0000", 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3046000000000003e-05 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.740018+0000", 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002400063 2023-10-09T04:22:13.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.742236+0000", 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022179439999999999 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.744908+0000", 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026718050000000002 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.744946+0000", 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7642999999999999e-05 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:14:51.744987+0000", 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1659000000000002e-05 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:153543 8.5 8:a8336751:::benchmark_data_smithi163_404719_object62959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:02.565871+0000", 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 310.36505217799998, 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.136344084, 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 153543 2023-10-09T04:22:13.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565871+0000", 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565871+0000", 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565864+0000", 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999938 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565876+0000", 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.163e-05 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565892+0000", 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6181999999999999e-05 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565899+0000", 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.973e-06 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.565949+0000", 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9293999999999999e-05 2023-10-09T04:22:13.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.566071+0000", 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012286 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.566201+0000", 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012923599999999999 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.571813+0000", 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056126839999999997 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.582645+0000", 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010831845 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.583005+0000", 2023-10-09T04:22:13.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00035997600000000002 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.702039+0000", 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.11903359600000001 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.702097+0000", 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8232000000000001e-05 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.702215+0000", 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117991 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:153548 8.5 8:aef0c451:::benchmark_data_smithi163_404719_object62964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:02.572497+0000", 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 310.35842652899998, 2023-10-09T04:22:13.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13459079700000001, 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 153548 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572497+0000", 2023-10-09T04:22:13.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572497+0000", 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572495+0000", 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572502+0000", 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4520000000000002e-06 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572513+0000", 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1762e-05 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572519+0000", 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.161e-06 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572596+0000", 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6977999999999994e-05 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572716+0000", 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119546 2023-10-09T04:22:13.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.572845+0000", 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012866599999999999 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.581558+0000", 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0087131129999999998 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.588731+0000", 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007173268 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.593288+0000", 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045568199999999996 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.706918+0000", 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.113630197 2023-10-09T04:22:13.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.706972+0000", 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3810000000000001e-05 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:02.707087+0000", 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011543400000000001 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.069 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:155118 8.5 8:a02aec34:::benchmark_data_smithi163_404719_object64534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:05.291508+0000", 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 307.63941489199999, 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14030984799999999, 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.070 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 155118 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291508+0000", 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291508+0000", 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291507+0000", 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291513+0000", 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0739999999999998e-06 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291525+0000", 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2241e-05 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291531+0000", 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2560000000000002e-06 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291586+0000", 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5575999999999998e-05 2023-10-09T04:22:13.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291733+0000", 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014699400000000001 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291892+0000", 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015912500000000001 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.297510+0000", 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056172419999999997 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.297563+0000", 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3143000000000003e-05 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.297627+0000", 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4435000000000002e-05 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.431635+0000", 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13400812100000001 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.431696+0000", 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1189e-05 2023-10-09T04:22:13.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.431818+0000", 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121581 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:155119 8.5 8:ae15b907:::benchmark_data_smithi163_404719_object64535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:05.291545+0000", 2023-10-09T04:22:13.077 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 307.63937829899999, 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14083680600000001, 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 155119 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291545+0000", 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291545+0000", 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291544+0000", 2023-10-09T04:22:13.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291547+0000", 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9550000000000001e-06 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291554+0000", 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.319e-06 2023-10-09T04:22:13.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.291558+0000", 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4150000000000001e-06 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.294352+0000", 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027939369999999998 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.294499+0000", 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014693900000000001 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,4,7", 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.294640+0000", 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014142000000000001 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.297554+0000", 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029136320000000002 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.297740+0000", 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018557600000000001 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.297751+0000", 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1494e-05 2023-10-09T04:22:13.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.432219+0000", 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.13446810000000001 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.432272+0000", 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2889999999999997e-05 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:05.432382+0000", 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109486 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.35475.0:170410 8.7 e290/241 8:e492b28b:::benchmark_data_smithi163_404719_object79826:head v 290'10733, mlcod=290'10731)", 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:17:31.636554+0000", 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 281.29436919400001, 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12633328799999999, 2023-10-09T04:22:13.085 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636554+0000", 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636554+0000", 2023-10-09T04:22:13.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636553+0000", 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636560+0000", 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6789999999999999e-06 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636569+0000", 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7100000000000002e-06 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636574+0000", 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5680000000000001e-06 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:13.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636623+0000", 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8776000000000002e-05 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.636644+0000", 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0781999999999999e-05 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.762852+0000", 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.12620795900000001 2023-10-09T04:22:13.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:17:31.762887+0000", 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5571999999999998e-05 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:13.090 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:22:13.091 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:13.120 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:13.252 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:13.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:13.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:13.283 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:22:13.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:13.387 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T04:22:13.676 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.677 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:13 smithi181 ceph-mon[196979]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:13.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:22:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:13.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:13.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:13.826 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:13.845 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:22:13.845 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:22:13.845 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:22:13.862 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:22:13.881 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:22:13.908 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:22:13.928 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:22:13.947 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:22:13.948 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.948 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.948 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.948 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.948 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.948 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.949 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.949 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[221067]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: failed to encode map e383 with expected crc 2023-10-09T04:22:13.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:13 smithi163 ceph-mon[213841]: failed to encode map e383 with expected crc 2023-10-09T04:22:14.079 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:14.538 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T04:22:14.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:14.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:14.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:14.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:14.725 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:14.788 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.788 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T04:22:14.788 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.788 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3445278166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[221067]: pgmap v1039: 119 pgs: 1 creating+activating, 5 creating+peering, 3 unknown, 110 active+clean; 612 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T04:22:14.789 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: failed to encode map e384 with expected crc 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3445278166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:14.790 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:14 smithi163 ceph-mon[213841]: pgmap v1039: 119 pgs: 1 creating+activating, 5 creating+peering, 3 unknown, 110 active+clean; 612 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:22:14.871 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:14.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:14.907 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:14.975 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:15.003 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:15.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: failed to encode map e384 with expected crc 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3445278166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:15.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:14 smithi181 ceph-mon[196979]: pgmap v1039: 119 pgs: 1 creating+activating, 5 creating+peering, 3 unknown, 110 active+clean; 612 KiB data, 4.4 GiB used, 621 GiB / 626 GiB avail 2023-10-09T04:22:15.147 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:15.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:15.252 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:15.284 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:15.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:15.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:15.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:15.493 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:15.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:15.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:15.625 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T04:22:15.641 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:15.670 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:15.682 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:22:15.682 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:22:15.716 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:15.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:15.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/437944525' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:22:15.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:15.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/437944525' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: failed to encode map e385 with expected crc 2023-10-09T04:22:15.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/437944525' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:22:15.849 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:16.007 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:16.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:16.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:16.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:16.140 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:16.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:16.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:16.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:22:16.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:16.477 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:16.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:16.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:16.627 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/437944525' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.661 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: failed to encode map e386 with expected crc 2023-10-09T04:22:16.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: pgmap v1042: 119 pgs: 1 remapped+peering, 1 creating+activating, 5 active+undersized+remapped, 112 active+clean; 626 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 4.5 KiB/s wr, 1 op/s; 84 KiB/s, 1 objects/s recovering 2023-10-09T04:22:16.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: 1.1d scrub starts 2023-10-09T04:22:16.662 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:16 smithi181 ceph-mon[196979]: 1.1d scrub ok 2023-10-09T04:22:16.664 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:16.768 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:16.813 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:16.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:22:16.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:16.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:16.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:16.945 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:16.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:16.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:16.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:17.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/437944525' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:22:17.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T04:22:17.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: pgmap v1042: 119 pgs: 1 remapped+peering, 1 creating+activating, 5 active+undersized+remapped, 112 active+clean; 626 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 4.5 KiB/s wr, 1 op/s; 84 KiB/s, 1 objects/s recovering 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: 1.1d scrub starts 2023-10-09T04:22:17.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[221067]: 1.1d scrub ok 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/437944525' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: failed to encode map e386 with expected crc 2023-10-09T04:22:17.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: pgmap v1042: 119 pgs: 1 remapped+peering, 1 creating+activating, 5 active+undersized+remapped, 112 active+clean; 626 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 1.7 KiB/s rd, 4.5 KiB/s wr, 1 op/s; 84 KiB/s, 1 objects/s recovering 2023-10-09T04:22:17.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: 1.1d scrub starts 2023-10-09T04:22:17.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:16 smithi163 ceph-mon[213841]: 1.1d scrub ok 2023-10-09T04:22:17.093 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:17.120 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:17.167 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:17.414 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:17.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:17.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:17.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:17.545 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:17.630 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:22:17.649 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T04:22:17.651 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:22:17.651 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:22:17.699 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T04:22:17.699 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:22:17.699 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_523108 2023-10-09T04:22:17.699 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:22:17.700 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:22:17.753 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:17.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:17.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:17.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T04:22:17.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.883 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: 1.3 scrub starts 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: 1.3 scrub ok 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3868774991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:17.884 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.885 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: failed to encode map e387 with expected crc 2023-10-09T04:22:17.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: 1.3 scrub starts 2023-10-09T04:22:17.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: 1.3 scrub ok 2023-10-09T04:22:17.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3868774991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:17.886 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:17 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:17.886 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T04:22:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: failed to encode map e387 with expected crc 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: 1.3 scrub starts 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: 1.3 scrub ok 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3868774991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:18.022 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:17 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:18.384 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:22:18.385 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:22:18.699 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 2750 2734 170.865 170.875 0.00112517 0.00525508 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: failed to encode map e388 with expected crc 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: failed to encode map e388 with expected crc 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: failed to encode map e388 with expected crc 2023-10-09T04:22:18.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:18 smithi181 ceph-mon[196979]: pgmap v1045: 119 pgs: 6 peering, 1 remapped+peering, 5 active+undersized+remapped, 107 active+clean; 626 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 4 op/s; 91 KiB/s, 2 objects/s recovering 2023-10-09T04:22:19.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:22:19.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: failed to encode map e388 with expected crc 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: failed to encode map e388 with expected crc 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: failed to encode map e388 with expected crc 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[221067]: pgmap v1045: 119 pgs: 6 peering, 1 remapped+peering, 5 active+undersized+remapped, 107 active+clean; 626 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 4 op/s; 91 KiB/s, 2 objects/s recovering 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:22:19.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: failed to encode map e388 with expected crc 2023-10-09T04:22:19.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: failed to encode map e388 with expected crc 2023-10-09T04:22:19.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:19.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T04:22:19.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: failed to encode map e388 with expected crc 2023-10-09T04:22:19.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:18 smithi163 ceph-mon[213841]: pgmap v1045: 119 pgs: 6 peering, 1 remapped+peering, 5 active+undersized+remapped, 107 active+clean; 626 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 4 op/s; 91 KiB/s, 2 objects/s recovering 2023-10-09T04:22:19.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:19.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:19.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:19.263 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:19.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:19.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:19.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:19.477 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:19.539 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 5, 7, 4, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:22:19.540 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:22:19.540 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 5, 7, 4, 3, 6] 2023-10-09T04:22:19.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T04:22:19.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 4317 4301 134.108 97.9375 0.00485111 0.00743542 2023-10-09T04:22:19.726 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:19.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:19 smithi181 ceph-mon[196979]: 1.1e deep-scrub starts 2023-10-09T04:22:19.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:19 smithi181 ceph-mon[196979]: 1.1e deep-scrub ok 2023-10-09T04:22:19.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:19.730 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:19.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:19.755 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:22:19.792 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:22:19.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:22:19.828 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:22:19.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[213841]: 1.1e deep-scrub starts 2023-10-09T04:22:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[213841]: 1.1e deep-scrub ok 2023-10-09T04:22:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[221067]: 1.1e deep-scrub starts 2023-10-09T04:22:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[221067]: 1.1e deep-scrub ok 2023-10-09T04:22:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:19.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:19.875 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:22:19.909 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:19.912 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:22:19.948 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:22:19.984 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:22:20.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:20.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:20.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:20.024 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:22:20.031 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.032 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:22:20.117 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:20.172 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:20.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:20.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:20.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:20.387 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:20.435 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:20.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:20.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:20.528 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:22:20.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:20.656 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/889453946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:20.656 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:20 smithi181 ceph-mon[196979]: pgmap v1046: 119 pgs: 6 peering, 5 active+undersized+remapped, 108 active+clean; 38 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 7.0 MiB/s wr, 115 op/s; 69 KiB/s, 2 objects/s recovering 2023-10-09T04:22:20.676 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:20.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 7360 7344 152.769 190.188 0.00298659 0.00653611 2023-10-09T04:22:20.772 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:20.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:20.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/889453946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:20.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:20 smithi163 ceph-mon[213841]: pgmap v1046: 119 pgs: 6 peering, 5 active+undersized+remapped, 108 active+clean; 38 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 7.0 MiB/s wr, 115 op/s; 69 KiB/s, 2 objects/s recovering 2023-10-09T04:22:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/889453946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:20.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:20 smithi163 ceph-mon[221067]: pgmap v1046: 119 pgs: 6 peering, 5 active+undersized+remapped, 108 active+clean; 38 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 7.0 MiB/s wr, 115 op/s; 69 KiB/s, 2 objects/s recovering 2023-10-09T04:22:20.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:22:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:22:20] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2023-10-09T04:22:20.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:20.924 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:21.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:22:21.314 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:21.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:21.417 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:21.466 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:21.488 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:21.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:21.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:21.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:21.661 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:21.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 10285 10269 160.268 182.812 0.00336624 0.00623066 2023-10-09T04:22:21.714 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:21.863 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:21.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:21.971 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:22.027 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:22.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 13194 13178 164.568 181.812 0.00133535 0.00601012 2023-10-09T04:22:23.093 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:23.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:23.204 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:23.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:22 smithi163 ceph-mon[221067]: pgmap v1047: 119 pgs: 119 active+clean; 318 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 805 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:23.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:22 smithi163 ceph-mon[213841]: pgmap v1047: 119 pgs: 119 active+clean; 318 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 805 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:23.243 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:23.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:22 smithi181 ceph-mon[196979]: pgmap v1047: 119 pgs: 119 active+clean; 318 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 805 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:23.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:23.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:23.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:23.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:22:23.561 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:23.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 15 15099 15084 156.998 119.125 0.00536678 0.00636299 2023-10-09T04:22:23.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:24.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:24.190 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:24.273 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28475 9.1 9:8009e758:::benchmark_data_smithi163_464650_object28474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 236.464605519, 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.096956874, 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:22:24.279 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28475 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801180+0000", 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801209+0000", 2023-10-09T04:22:24.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8646e-05 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801226+0000", 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.715e-05 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801234+0000", 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3459999999999996e-06 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801291+0000", 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7438999999999997e-05 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801409+0000", 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011781399999999999 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:22:24.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801670+0000", 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026062299999999997 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809884+0000", 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082140670000000002 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809890+0000", 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6460000000000004e-06 2023-10-09T04:22:24.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812682+0000", 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002791615 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812694+0000", 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.171e-05 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.884929+0000", 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0722356550000001 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.885962+0000", 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010329860000000001 2023-10-09T04:22:24.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898005+0000", 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012042361999999999 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898138+0000", 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013387100000000001 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.287 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28492 9.1 9:80d8c0c4:::benchmark_data_smithi163_464650_object28491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 236.353610503, 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9871532940000001, 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.288 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28492 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912175+0000", 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912211+0000", 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6192000000000002e-05 2023-10-09T04:22:24.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912224+0000", 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2965999999999999e-05 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912230+0000", 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6540000000000004e-06 2023-10-09T04:22:24.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912283+0000", 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3115999999999998e-05 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912289+0000", 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2700000000000001e-06 2023-10-09T04:22:24.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812717+0000", 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90042778800000001 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812726+0000", 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5189999999999992e-06 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886022+0000", 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.073296604 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886114+0000", 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1397999999999995e-05 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886285+0000", 2023-10-09T04:22:24.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171807 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891737+0000", 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005451661 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898177+0000", 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064403500000000001 2023-10-09T04:22:24.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898550+0000", 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037258199999999999 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898598+0000", 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8214000000000003e-05 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.296 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899330+0000", 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000731593 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28496 9.1 9:834685fc:::benchmark_data_smithi163_464650_object28495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 236.34595974600001, 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9805368130000001, 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.297 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28496 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919880+0000", 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3053000000000002e-05 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919893+0000", 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3439e-05 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919899+0000", 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.161e-06 2023-10-09T04:22:24.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919957+0000", 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8118999999999997e-05 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919969+0000", 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2065000000000001e-05 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812748+0000", 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89277856099999997 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812756+0000", 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6079999999999998e-06 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887444+0000", 2023-10-09T04:22:24.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.074688672 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887596+0000", 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000151533 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887736+0000", 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013991200000000001 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891747+0000", 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040110479999999997 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898230+0000", 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064832809999999996 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899391+0000", 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001160863 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899443+0000", 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1969000000000003e-05 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.900364+0000", 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00092131999999999999 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.305 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.306 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87402 9.d 9:b58a4af0:::benchmark_data_smithi163_464650_object87401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.36918243, 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9327528470000002, 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:22:24.330 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87402 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896603+0000", 2023-10-09T04:22:24.581 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896662+0000", 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8721999999999998e-05 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896672+0000", 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0342e-05 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.582 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896678+0000", 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5890000000000002e-06 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897746+0000", 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001067901 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.583 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897884+0000", 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013837699999999999 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.898027+0000", 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014309899999999999 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.903780+0000", 2023-10-09T04:22:24.584 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057525900000000001 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913854+0000", 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010074411 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913865+0000", 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0667e-05 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.585 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904284+0000", 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99041863600000002 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904295+0000", 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1622e-05 2023-10-09T04:22:24.586 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829288+0000", 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9249930480000002 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829357+0000", 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9139000000000003e-05 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.587 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87427 9.d 9:b85b518c:::benchmark_data_smithi163_464650_object87426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.33604342000001, 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9083351629999998, 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87427 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.590 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:22:24.591 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929742+0000", 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929812+0000", 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9973000000000002e-05 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.592 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929833+0000", 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0372000000000001e-05 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929840+0000", 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0380000000000002e-06 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929955+0000", 2023-10-09T04:22:24.593 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115658 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929964+0000", 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.02e-06 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904451+0000", 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97448704799999997 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.594 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904459+0000", 2023-10-09T04:22:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:22:24.609 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829570+0000", 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9251109199999998 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829657+0000", 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6693000000000005e-05 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.610 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829772+0000", 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115109 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837839+0000", 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080666330000000001 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837847+0000", 2023-10-09T04:22:24.611 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5720000000000003e-06 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837892+0000", 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4297e-05 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837929+0000", 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7622e-05 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.612 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837980+0000", 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0816000000000003e-05 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838079+0000", 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8758000000000006e-05 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.613 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87460 9.d 9:be691015:::benchmark_data_smithi163_464650_object87459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.29511580499999, 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8737964319999998, 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87460 2023-10-09T04:22:24.614 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.615 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970670+0000", 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970704+0000", 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3708999999999999e-05 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.616 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970712+0000", 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2910000000000007e-06 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970718+0000", 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2909999999999998e-06 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971579+0000", 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086131599999999999 2023-10-09T04:22:24.617 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971589+0000", 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0108e-05 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904489+0000", 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93289962699999995 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.618 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904496+0000", 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3939999999999998e-06 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832602+0000", 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9281065480000001 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832830+0000", 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022731300000000001 2023-10-09T04:22:24.619 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.833032+0000", 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202256 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837853+0000", 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048209000000000004 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.620 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838118+0000", 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000265083 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838174+0000", 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5485999999999997e-05 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844377+0000", 2023-10-09T04:22:24.621 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062031949999999999 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844410+0000", 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3266999999999998e-05 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844468+0000", 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7698000000000002e-05 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.622 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87469 9.d 9:b0d8b7f7:::benchmark_data_smithi163_464650_object87468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.291907151, 2023-10-09T04:22:24.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8824357369999998, 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87469 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973879+0000", 2023-10-09T04:22:24.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973909+0000", 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0063999999999999e-05 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973921+0000", 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.204e-05 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973927+0000", 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2049999999999999e-06 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974432+0000", 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050552799999999999 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974441+0000", 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904518+0000", 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93007710099999996 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904525+0000", 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9280000000000004e-06 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835034+0000", 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9305090869999999 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835189+0000", 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015497800000000001 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835319+0000", 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012983499999999999 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838122+0000", 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028034420000000002 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838149+0000", 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6636999999999999e-05 2023-10-09T04:22:24.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838193+0000", 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4202999999999997e-05 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856143+0000", 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017949975 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856198+0000", 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4968e-05 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856316+0000", 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117589 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.649 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87510 9.d 9:b1e25812:::benchmark_data_smithi163_464650_object87509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.24254973800001, 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8391414909999999, 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87510 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.650 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023236+0000", 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023264+0000", 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7639000000000001e-05 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023280+0000", 2023-10-09T04:22:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6189999999999999e-05 2023-10-09T04:22:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023285+0000", 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4380000000000003e-06 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023338+0000", 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2713999999999997e-05 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023350+0000", 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1912999999999999e-05 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904548+0000", 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88119805299999998 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904555+0000", 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1690000000000003e-06 2023-10-09T04:22:24.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838206+0000", 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9336506519999999 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838292+0000", 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6051999999999995e-05 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838426+0000", 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013363900000000001 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844098+0000", 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056724699999999998 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844218+0000", 2023-10-09T04:22:24.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012015000000000001 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844519+0000", 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000300842 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862218+0000", 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017698935999999998 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862272+0000", 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4085999999999997e-05 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862379+0000", 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87520 9.d 9:b10313ad:::benchmark_data_smithi163_464650_object87519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.23392191299999, 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.830771683, 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87520 2023-10-09T04:22:24.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031864+0000", 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031940+0000", 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6348999999999997e-05 2023-10-09T04:22:24.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031956+0000", 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5733e-05 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031962+0000", 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1330000000000004e-06 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032020+0000", 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8168999999999998e-05 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032031+0000", 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0546e-05 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904577+0000", 2023-10-09T04:22:24.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87254678500000005 2023-10-09T04:22:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904584+0000", 2023-10-09T04:22:24.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8279999999999997e-06 2023-10-09T04:22:24.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841092+0000", 2023-10-09T04:22:24.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.936507684 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841230+0000", 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138245 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841385+0000", 2023-10-09T04:22:24.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015503399999999999 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844512+0000", 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031265519999999999 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850232+0000", 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057197300000000001 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850519+0000", 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000287342 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862488+0000", 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011968807999999999 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862529+0000", 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1087999999999997e-05 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862637+0000", 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108094 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.740 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87523 9.d 9:b2b1ff3b:::benchmark_data_smithi163_464650_object87522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.228073163, 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8252149690000001, 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87523 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037713+0000", 2023-10-09T04:22:24.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037758+0000", 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5336000000000001e-05 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037769+0000", 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1139e-05 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037774+0000", 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6249999999999998e-06 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037811+0000", 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7234999999999998e-05 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037819+0000", 2023-10-09T04:22:24.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8599999999999993e-06 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904623+0000", 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86680350299999998 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904630+0000", 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5780000000000001e-06 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844552+0000", 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9399218949999999 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844685+0000", 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013337900000000001 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844852+0000", 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166682 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850445+0000", 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055933379999999998 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850487+0000", 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1508000000000003e-05 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856104+0000", 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056174129999999999 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862788+0000", 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066833480000000004 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862832+0000", 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000002e-05 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862929+0000", 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7040999999999995e-05 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87524 9.d 9:bf5a5ce2:::benchmark_data_smithi163_464650_object87523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.749 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.22534905200001, 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8227152589999998, 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87524 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040437+0000", 2023-10-09T04:22:24.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040544+0000", 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010666399999999999 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040556+0000", 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697999999999999e-05 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.752 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040562+0000", 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0839999999999997e-06 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040647+0000", 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4971999999999999e-05 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040656+0000", 2023-10-09T04:22:24.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9479999999999997e-06 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904660+0000", 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86400389600000005 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904667+0000", 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847526+0000", 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9428587660000001 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847651+0000", 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125136 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847838+0000", 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018702700000000001 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850552+0000", 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002713853 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856113+0000", 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055611580000000001 2023-10-09T04:22:24.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862179+0000", 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060656729999999997 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863021+0000", 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000842759 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863060+0000", 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8272000000000002e-05 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863153+0000", 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3684000000000006e-05 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.758 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87531 9.d 9:b4d1cf98:::benchmark_data_smithi163_464650_object87530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.759 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:22:24.759 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.22432154099999, 2023-10-09T04:22:24.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.836533765, 2023-10-09T04:22:24.759 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.763 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.763 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.763 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87531 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041465+0000", 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041513+0000", 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8566999999999998e-05 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041522+0000", 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9020000000000005e-06 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041528+0000", 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041602+0000", 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4530000000000006e-05 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041611+0000", 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2819999999999996e-06 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904688+0000", 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86307752000000004 2023-10-09T04:22:24.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904695+0000", 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850572+0000", 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9458773759999999 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850685+0000", 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011277699999999999 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850827+0000", 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014244099999999999 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862146+0000", 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011318701 2023-10-09T04:22:24.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862451+0000", 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030508599999999999 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862741+0000", 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029011400000000001 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877820+0000", 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015078775000000001 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877869+0000", 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9407999999999999e-05 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877999+0000", 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129868 2023-10-09T04:22:24.771 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87536 9.d 9:b13d2384:::benchmark_data_smithi163_464650_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.21383912799999, 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.826285253, 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.772 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87536 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:22:24.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051947+0000", 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052032+0000", 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5885999999999997e-05 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052049+0000", 2023-10-09T04:22:24.792 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6867e-05 2023-10-09T04:22:24.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.792 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052056+0000", 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9349999999999997e-06 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052117+0000", 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0758999999999999e-05 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052127+0000", 2023-10-09T04:22:24.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301e-05 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904719+0000", 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85259186799999997 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904726+0000", 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7989999999999996e-06 2023-10-09T04:22:24.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853627+0000", 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9489006930000001 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853769+0000", 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142619 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853901+0000", 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205399999999999 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862421+0000", 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085194190000000003 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862735+0000", 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031399200000000003 2023-10-09T04:22:24.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862747+0000", 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2238e-05 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878064+0000", 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015316585000000001 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878120+0000", 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6338999999999997e-05 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878233+0000", 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113268 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87542 9.d 9:b8ae8bf6:::benchmark_data_smithi163_464650_object87541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.20433754800001, 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.817112244, 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87542 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.799 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061448+0000", 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061495+0000", 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6932999999999999e-05 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061515+0000", 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:22:24.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061522+0000", 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8499999999999996e-06 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061580+0000", 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8037e-05 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061591+0000", 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1099e-05 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904748+0000", 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84315744100000001 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904755+0000", 2023-10-09T04:22:24.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7880000000000001e-06 2023-10-09T04:22:24.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856351+0000", 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9515959619999999 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856435+0000", 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3497000000000004e-05 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856590+0000", 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015579299999999999 2023-10-09T04:22:24.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862697+0000", 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0061066360000000004 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862753+0000", 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6106000000000001e-05 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862991+0000", 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023799100000000001 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878381+0000", 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015390452000000001 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878432+0000", 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0779999999999998e-05 2023-10-09T04:22:24.809 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878562+0000", 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012949399999999999 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87544 9.d 9:b2959611:::benchmark_data_smithi163_464650_object87543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:22:24.810 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.20419252299999, 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8172143809999999, 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87544 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.811 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.812 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.857 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.857 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.857 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061594+0000", 2023-10-09T04:22:24.858 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:24.858 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.858 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.858 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.858 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061631+0000", 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6940000000000002e-05 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061641+0000", 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5650000000000007e-06 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061645+0000", 2023-10-09T04:22:24.859 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9400000000000001e-06 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061692+0000", 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6638000000000003e-05 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061705+0000", 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2525e-05 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.860 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904777+0000", 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84307196799999995 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904783+0000", 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859252+0000", 2023-10-09T04:22:24.861 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.954468756 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859377+0000", 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124423 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859542+0000", 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016522800000000001 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.862 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862758+0000", 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032164730000000001 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862963+0000", 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000205105 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.863 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871931+0000", 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089675880000000003 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878651+0000", 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067196959999999998 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878706+0000", 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5047999999999999e-05 2023-10-09T04:22:24.864 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878809+0000", 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103266 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87579 9.d 9:b6955889:::benchmark_data_smithi163_464650_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:22:24.865 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.14689551999999, 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.760215788, 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87579 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.866 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118890+0000", 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:24.867 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118943+0000", 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2917000000000002e-05 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.868 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118957+0000", 2023-10-09T04:22:24.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3553000000000001e-05 2023-10-09T04:22:24.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118963+0000", 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9329999999999998e-06 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119027+0000", 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4450999999999997e-05 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119039+0000", 2023-10-09T04:22:24.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1657999999999999e-05 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904808+0000", 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78576894799999997 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904814+0000", 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.472e-06 2023-10-09T04:22:24.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863179+0000", 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.958364789 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863297+0000", 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011818 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863439+0000", 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000141836 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871995+0000", 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085565290000000002 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872008+0000", 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2779000000000001e-05 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878306+0000", 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062979710000000003 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878882+0000", 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057614199999999997 2023-10-09T04:22:24.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878977+0000", 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4944000000000004e-05 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879107+0000", 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013005199999999999 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87597 9.d 9:ba722c6f:::benchmark_data_smithi163_464650_object87596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.118516804, 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.741504747, 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87597 2023-10-09T04:22:24.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147306+0000", 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6227999999999999e-05 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147315+0000", 2023-10-09T04:22:24.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4420000000000004e-06 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147321+0000", 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3900000000000001e-06 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147373+0000", 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1922000000000002e-05 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147383+0000", 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8160000000000005e-06 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904836+0000", 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75745349399999995 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904843+0000", 2023-10-09T04:22:24.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9360000000000002e-06 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866166+0000", 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9613231419999999 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866285+0000", 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011870399999999999 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866435+0000", 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015030299999999999 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.875034+0000", 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085985039999999999 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878346+0000", 2023-10-09T04:22:24.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033125379999999999 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878607+0000", 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000261158 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888598+0000", 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099901480000000008 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888652+0000", 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4688999999999998e-05 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888775+0000", 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012282100000000001 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87606 9.d 9:bd98b1c5:::benchmark_data_smithi163_464650_object87605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.097667398, 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7216060710000001, 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87606 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.911 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:22:24.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:24.913 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 17011 16995 151.634 119.438 0.00174818 0.00652902 2023-10-09T04:22:24.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168119+0000", 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168170+0000", 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1724000000000001e-05 2023-10-09T04:22:24.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168183+0000", 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2322e-05 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168188+0000", 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.011e-06 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168245+0000", 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7197999999999997e-05 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168257+0000", 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1742999999999999e-05 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904867+0000", 2023-10-09T04:22:24.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.73660977999999999 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904873+0000", 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3450000000000002e-06 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.868939+0000", 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9640662180000001 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869059+0000", 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012025 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869200+0000", 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014040799999999999 2023-10-09T04:22:24.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878017+0000", 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0088170209999999995 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878849+0000", 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00083247799999999999 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879166+0000", 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000316373 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889551+0000", 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010385553 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889609+0000", 2023-10-09T04:22:24.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-05 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889726+0000", 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116743 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87628 9.d 9:b09d3eb4:::benchmark_data_smithi163_464650_object87627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:22:24.948 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 178.013206747, 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6374180369999998, 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87628 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252579+0000", 2023-10-09T04:22:24.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252624+0000", 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5071999999999998e-05 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252637+0000", 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.272e-05 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252642+0000", 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8999999999999997e-06 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252685+0000", 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3056000000000002e-05 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252691+0000", 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3790000000000003e-06 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905181+0000", 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.65248916000000001 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905191+0000", 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0231000000000001e-05 2023-10-09T04:22:24.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872040+0000", 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.966849243 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872156+0000", 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115611 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872308+0000", 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152707 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878257+0000", 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059484389999999998 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879226+0000", 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096947500000000002 2023-10-09T04:22:24.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888556+0000", 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093301310000000002 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889825+0000", 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001268948 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889882+0000", 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6885999999999997e-05 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889998+0000", 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011613899999999999 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.962 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:166481 9.3 9:ce187e50:::benchmark_data_smithi163_464650_object67755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:21:15.975004+0000", 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 68.290783340000004, 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28756461, 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 166481 2023-10-09T04:22:24.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975004+0000", 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975004+0000", 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:22:24.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975003+0000", 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975006+0000", 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4940000000000002e-06 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:22:24.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975011+0000", 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6869999999999999e-06 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:15.975016+0000", 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1500000000000001e-06 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.089897+0000", 2023-10-09T04:22:24.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.114881733 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.090007+0000", 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109947 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.090114+0000", 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106437 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.098694+0000", 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085805250000000003 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.098804+0000", 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109411 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.262510+0000", 2023-10-09T04:22:24.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16370647499999999 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.262542+0000", 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2141000000000003e-05 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:21:16.262568+0000", 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6055999999999999e-05 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.969 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.970 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.970 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:22:24.970 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:22:24.970 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:22:25.057 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:24 smithi163 ceph-mon[213841]: pgmap v1048: 119 pgs: 119 active+clean; 496 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:24 smithi163 ceph-mon[221067]: pgmap v1048: 119 pgs: 119 active+clean; 496 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:25.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:25.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:25.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:25.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:24 smithi181 ceph-mon[196979]: pgmap v1048: 119 pgs: 119 active+clean; 496 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:25.302 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:25.380 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:25.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:25.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:25.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:22:25.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:25.554 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:25.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 18378 18362 143.363 85.4375 0.00321464 0.0069711 2023-10-09T04:22:25.825 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:25 smithi163 conmon[213817]: 2023-10-09T04:22:25.412+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e388 definitely_dead 0 2023-10-09T04:22:25.846 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:25.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:25.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:25.865 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:25.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:25.996 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.0. 2023-10-09T04:22:25.998 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:26.044 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:26.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:26.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:26.074 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:26.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1593001480' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:22:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:25 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:22:26.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:25 smithi163 ceph-mon[221067]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:22:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1593001480' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:22:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:25 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:22:26.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:25 smithi163 ceph-mon[213841]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:22:26.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:26.180 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1593001480' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:22:26.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:25 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:22:26.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:25 smithi181 ceph-mon[196979]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:22:26.704 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 18951 18935 131.418 35.8125 0.00186207 0.00700782 2023-10-09T04:22:26.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:26.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:26.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:27.006 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:27.056 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:27.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:27.152 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[221067]: pgmap v1049: 119 pgs: 119 active+clean; 863 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:27.152 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[221067]: failed to encode map e389 with expected crc 2023-10-09T04:22:27.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:22:27.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[221067]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T04:22:27.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[213841]: pgmap v1049: 119 pgs: 119 active+clean; 863 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:27.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[213841]: failed to encode map e389 with expected crc 2023-10-09T04:22:27.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:22:27.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:26 smithi163 ceph-mon[213841]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T04:22:27.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:27.193 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:27.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:26 smithi181 ceph-mon[196979]: pgmap v1049: 119 pgs: 119 active+clean; 863 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:22:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:26 smithi181 ceph-mon[196979]: failed to encode map e389 with expected crc 2023-10-09T04:22:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:26 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:22:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:26 smithi181 ceph-mon[196979]: osdmap e389: 8 total, 8 up, 7 in 2023-10-09T04:22:27.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:27.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:27.600 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:27.632 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:27.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:27.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:27.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:27.705 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 18951 18935 118.281 0 - 0.00700782 2023-10-09T04:22:27.792 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:27.820 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.998 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: failed to encode map e390 with expected crc 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2051495429' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:22:27.999 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:27 smithi181 ceph-mon[196979]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T04:22:28.012 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:22:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T04:22:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2051495429' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:22:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[213841]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: osdmap e390: 8 total, 8 up, 7 in 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: failed to encode map e390 with expected crc 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2051495429' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:22:28.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:22:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:22:28.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:27 smithi163 ceph-mon[221067]: osdmap e391: 8 total, 8 up, 7 in 2023-10-09T04:22:28.707 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 16 19748 19732 112.041 24.9062 0.0122025 0.00891854 2023-10-09T04:22:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: pgmap v1052: 119 pgs: 119 active+clean; 1.0 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T04:22:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:29 smithi181 ceph-mon[196979]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T04:22:29.282 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:29.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: pgmap v1052: 119 pgs: 119 active+clean; 1.0 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.290 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.291 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[213841]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T04:22:29.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: pgmap v1052: 119 pgs: 119 active+clean; 1.0 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: failed to encode map e391 with expected crc 2023-10-09T04:22:29.292 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:29 smithi163 ceph-mon[221067]: osdmap e392: 8 total, 8 up, 7 in 2023-10-09T04:22:29.393 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:29.459 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:29.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:29.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:29.665 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:29.706 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 21251 21235 110.532 93.9375 0.00686958 0.00903948 2023-10-09T04:22:29.730 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:29.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:29.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:29.854 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:21460 10.b 10:d7908384:::benchmark_data_smithi163_523108_object21459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e392)", 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:29.847522+0000", 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0044487249999999997, 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044870209999999999, 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:22:29.855 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 21460 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847522+0000", 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:29.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847522+0000", 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847520+0000", 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:29.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847583+0000", 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3047999999999997e-05 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847605+0000", 2023-10-09T04:22:29.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1641000000000002e-05 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847616+0000", 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1083e-05 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:29.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847678+0000", 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2725999999999995e-05 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.847877+0000", 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019896099999999999 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:22:29.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.848066+0000", 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018857800000000001 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.849990+0000", 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001924409 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:29.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46073.0:21460 10.b e392/390)", 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:29.851724+0000", 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.00024712600000000002, 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00037743400000000001, 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:29.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.851724+0000", 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.851724+0000", 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.851723+0000", 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.851730+0000", 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6210000000000004e-06 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.851749+0000", 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8355999999999998e-05 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.851757+0000", 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8009999999999993e-06 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:29.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:29.852097+0000", 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00033941400000000001 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 2 2023-10-09T04:22:29.866 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:22:29.945 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.015 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.026 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[221067]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: failed to encode map e392 with expected crc 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:30.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:30 smithi163 ceph-mon[213841]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T04:22:30.173 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:22:30.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:30.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:30.207 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:22:30.243 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:22:30.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:30.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:30 smithi181 ceph-mon[196979]: osdmap e393: 8 total, 8 up, 7 in 2023-10-09T04:22:30.276 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:22:30.300 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:30.310 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:22:30.350 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:22:30.363 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:30.381 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:22:30.410 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.411 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.412 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:22:30.432 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:30.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:30.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:30.564 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:30.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:30.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:30.707 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 22377 22361 107.444 70.375 0.0115385 0.00930083 2023-10-09T04:22:30.711 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:30.770 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:30.831 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:22:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:22:30] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2023-10-09T04:22:30.851 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:30.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:30.960 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:31.014 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:31.162 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:31.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: pgmap v1055: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 115 active+clean; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 3604/53556 objects degraded (6.729%) 2023-10-09T04:22:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1595552003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:31.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:31 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 3604/53556 objects degraded (6.729%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:31.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:31.303 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: pgmap v1055: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 115 active+clean; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 3604/53556 objects degraded (6.729%) 2023-10-09T04:22:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1595552003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 3604/53556 objects degraded (6.729%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: pgmap v1055: 119 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 115 active+clean; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 3604/53556 objects degraded (6.729%) 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: failed to encode map e393 with expected crc 2023-10-09T04:22:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1595552003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:31 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 3604/53556 objects degraded (6.729%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:31.352 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:31.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:31.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:31.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:22:31.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:31.490 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:31.710 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 23505 23489 104.784 70.5 0.0102266 0.00953604 2023-10-09T04:22:31.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:22:31.892 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:31.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:31.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:31.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:32.067 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:32.100 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:32.148 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:32.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:32.154 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 4, 3, 6] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:22:32.155 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:22:32.155 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T04:22:32.155 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:22:32.155 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:22:32.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T04:22:32.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:32.320 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:32.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:32.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:32.567 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:32.618 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:32.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:32.710 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 24822 24806 103.285 82.3125 0.0139184 0.00967368 2023-10-09T04:22:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:32.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:32.816 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:32.865 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:32.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:32.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:33.095 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:33.150 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:33.231 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:33.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:33.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:33 smithi181 ceph-mon[196979]: pgmap v1057: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 109 active+clean; 1.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 20 KiB/s rd, 67 MiB/s wr, 1.09k op/s; 8306/65139 objects degraded (12.751%); 1843/65139 objects misplaced (2.829%); 6.7 MiB/s, 104 objects/s recovering 2023-10-09T04:22:33.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:33 smithi163 ceph-mon[221067]: pgmap v1057: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 109 active+clean; 1.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 20 KiB/s rd, 67 MiB/s wr, 1.09k op/s; 8306/65139 objects degraded (12.751%); 1843/65139 objects misplaced (2.829%); 6.7 MiB/s, 104 objects/s recovering 2023-10-09T04:22:33.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:33 smithi163 ceph-mon[213841]: pgmap v1057: 119 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 109 active+clean; 1.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 20 KiB/s rd, 67 MiB/s wr, 1.09k op/s; 8306/65139 objects degraded (12.751%); 1843/65139 objects misplaced (2.829%); 6.7 MiB/s, 104 objects/s recovering 2023-10-09T04:22:33.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:33.407 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:33.522 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:33.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:33.626 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:33.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:22:33.682 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:33.710 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 26420 26404 103.07 99.875 0.0107647 0.00969529 2023-10-09T04:22:33.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:22:33.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:33.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:22:33.797 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:33.886 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:33.947 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:33.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:22:34.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:34.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:34.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:34.344 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:34.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 27797 27781 102.069 86.0625 0.0103708 0.00978853 2023-10-09T04:22:34.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:34.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:34.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:34.969 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:35.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[213841]: pgmap v1058: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 16 KiB/s rd, 63 MiB/s wr, 1.03k op/s; 8306/67593 objects degraded (12.288%); 3022/67593 objects misplaced (4.471%); 5.5 MiB/s, 85 objects/s recovering 2023-10-09T04:22:35.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:35.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:35.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3503945507' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:35.038 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:35.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:35.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:35.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:35.263 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:35 smithi181 ceph-mon[196979]: pgmap v1058: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 16 KiB/s rd, 63 MiB/s wr, 1.03k op/s; 8306/67593 objects degraded (12.288%); 3022/67593 objects misplaced (4.471%); 5.5 MiB/s, 85 objects/s recovering 2023-10-09T04:22:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:35 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3503945507' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:35 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[221067]: pgmap v1058: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 16 KiB/s rd, 63 MiB/s wr, 1.03k op/s; 8306/67593 objects degraded (12.288%); 3022/67593 objects misplaced (4.471%); 5.5 MiB/s, 85 objects/s recovering 2023-10-09T04:22:35.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:35.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:35.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3503945507' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:35.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:35 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:35.356 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:22:35.710 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 29277 29261 101.537 92.5 0.00252287 0.00978701 2023-10-09T04:22:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:36 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:36 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:36.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:36 smithi163 ceph-mon[213841]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T04:22:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:36 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:36 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:36.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:36 smithi163 ceph-mon[221067]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T04:22:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:36 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:36 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:22:36.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:36 smithi181 ceph-mon[196979]: osdmap e394: 8 total, 8 up, 7 in 2023-10-09T04:22:36.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 30476 30460 100.138 74.9375 0.0020559 0.0099704 2023-10-09T04:22:36.766 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:36.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:36.871 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:36.929 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:37.032 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:37.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:37.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:37.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:37 smithi181 ceph-mon[196979]: failed to encode map e394 with expected crc 2023-10-09T04:22:37.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:37 smithi181 ceph-mon[196979]: failed to encode map e394 with expected crc 2023-10-09T04:22:37.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:37 smithi181 ceph-mon[196979]: pgmap v1060: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.6 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 14 KiB/s rd, 86 MiB/s wr, 1.39k op/s; 8306/78612 objects degraded (10.566%); 3022/78612 objects misplaced (3.844%); 4.8 MiB/s, 74 objects/s recovering 2023-10-09T04:22:37.190 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 8306/78612 objects degraded (10.566%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:37.192 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:37.291 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:37.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:37.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:37.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[221067]: failed to encode map e394 with expected crc 2023-10-09T04:22:37.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[221067]: failed to encode map e394 with expected crc 2023-10-09T04:22:37.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[221067]: pgmap v1060: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.6 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 14 KiB/s rd, 86 MiB/s wr, 1.39k op/s; 8306/78612 objects degraded (10.566%); 3022/78612 objects misplaced (3.844%); 4.8 MiB/s, 74 objects/s recovering 2023-10-09T04:22:37.328 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 8306/78612 objects degraded (10.566%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:37.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[213841]: failed to encode map e394 with expected crc 2023-10-09T04:22:37.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[213841]: failed to encode map e394 with expected crc 2023-10-09T04:22:37.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[213841]: pgmap v1060: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.6 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 14 KiB/s rd, 86 MiB/s wr, 1.39k op/s; 8306/78612 objects degraded (10.566%); 3022/78612 objects misplaced (3.844%); 4.8 MiB/s, 74 objects/s recovering 2023-10-09T04:22:37.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 8306/78612 objects degraded (10.566%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:37.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:37.466 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:37.474 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:37.562 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:37.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:37.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:37.696 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:37.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:22:37.711337+0000 min lat: 0.00100671 max lat: 2.11443 avg lat: 0.0100121 2023-10-09T04:22:37.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:22:37.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 31541 31525 98.4593 66.5625 0.00210105 0.0100121 2023-10-09T04:22:37.738 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:37.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:37.844 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:37.878 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:37.921 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:22:37.921 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T04:22:37.921 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:22:38.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:38.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:38.166 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:38.199 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:38.475 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:22:38.475 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:22:38.623 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[221067]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T04:22:38.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[221067]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[221067]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[221067]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:38.624 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[221067]: pgmap v1062: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 12 KiB/s rd, 87 MiB/s wr, 1.39k op/s; 8306/86415 objects degraded (9.612%); 1646/86415 objects misplaced (1.905%); 15 MiB/s, 239 objects/s recovering 2023-10-09T04:22:38.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[213841]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T04:22:38.625 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[213841]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.626 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[213841]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.626 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[213841]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.626 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:38.626 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:38 smithi163 ceph-mon[213841]: pgmap v1062: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 12 KiB/s rd, 87 MiB/s wr, 1.39k op/s; 8306/86415 objects degraded (9.612%); 1646/86415 objects misplaced (1.905%); 15 MiB/s, 239 objects/s recovering 2023-10-09T04:22:38.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 31781 31765 94.4866 15 0.00203041 0.0105052 2023-10-09T04:22:38.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:38 smithi181 ceph-mon[196979]: osdmap e395: 8 total, 8 up, 7 in 2023-10-09T04:22:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:38 smithi181 ceph-mon[196979]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:38 smithi181 ceph-mon[196979]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:38 smithi181 ceph-mon[196979]: failed to encode map e395 with expected crc 2023-10-09T04:22:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:38 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:38.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:38 smithi181 ceph-mon[196979]: pgmap v1062: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 12 KiB/s rd, 87 MiB/s wr, 1.39k op/s; 8306/86415 objects degraded (9.612%); 1646/86415 objects misplaced (1.905%); 15 MiB/s, 239 objects/s recovering 2023-10-09T04:22:39.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[213841]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[213841]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[213841]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[213841]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[221067]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[221067]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T04:22:39.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[221067]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:39 smithi163 ceph-mon[221067]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 32255 32239 91.5395 29.625 0.00730442 0.0109174 2023-10-09T04:22:39.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:39 smithi181 ceph-mon[196979]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:39 smithi181 ceph-mon[196979]: osdmap e396: 8 total, 8 up, 7 in 2023-10-09T04:22:39.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:39 smithi181 ceph-mon[196979]: failed to encode map e396 with expected crc 2023-10-09T04:22:39.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:39 smithi181 ceph-mon[196979]: failed to encode map e396 with expected crc 2023-10-09T04:22:40.639 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.639 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.639 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T04:22:40.639 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.639 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.639 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: pgmap v1065: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 7099/89994 objects degraded (7.888%); 2286/89994 objects misplaced (2.540%); 27 MiB/s, 426 objects/s recovering 2023-10-09T04:22:40.640 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:40.643 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: pgmap v1065: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 7099/89994 objects degraded (7.888%); 2286/89994 objects misplaced (2.540%); 27 MiB/s, 426 objects/s recovering 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:40 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:40.644 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:22:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:22:40] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.43.0" 2023-10-09T04:22:40.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 33485 33469 90.9019 76.875 0.00204737 0.0109877 2023-10-09T04:22:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: osdmap e397: 8 total, 8 up, 7 in 2023-10-09T04:22:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: failed to encode map e397 with expected crc 2023-10-09T04:22:40.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: pgmap v1065: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.8 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 7099/89994 objects degraded (7.888%); 2286/89994 objects misplaced (2.540%); 27 MiB/s, 426 objects/s recovering 2023-10-09T04:22:40.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:40 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:41.711 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 34620 34604 90.0698 70.9375 0.00956322 0.011097 2023-10-09T04:22:41.818 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:22:41.863 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:41.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:41.980 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:42.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 7099/98577 objects degraded (7.201%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:42.046 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:42.078 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:42.169 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:42.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:42.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 7099/98577 objects degraded (7.201%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:42.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:42.290 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 7099/98577 objects degraded (7.201%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:42.363 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:42.646 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:42.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:42.712 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 35763 35747 89.3209 71.4375 0.009556 0.011189 2023-10-09T04:22:42.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:42.766 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:22:42.806 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:22:42.844 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:22:42.846 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:42.883 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:22:42.936 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:22:42.978 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:22:43.012 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:43.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:43.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:22:43.076 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:22:43.112 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[221067]: pgmap v1066: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 7099/98577 objects degraded (7.201%); 1819/98577 objects misplaced (1.845%); 25 MiB/s, 401 objects/s recovering 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1563838877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[213841]: pgmap v1066: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 7099/98577 objects degraded (7.201%); 1819/98577 objects misplaced (1.845%); 25 MiB/s, 401 objects/s recovering 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:43.113 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1563838877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:43.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:43.204 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:43.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:43 smithi181 ceph-mon[196979]: pgmap v1066: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 7099/98577 objects degraded (7.201%); 1819/98577 objects misplaced (1.845%); 25 MiB/s, 401 objects/s recovering 2023-10-09T04:22:43.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:43.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:43 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:43.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1563838877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:43.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 15 37102 37087 89.098 83.75 0.00715767 0.0112171 2023-10-09T04:22:44.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 38165 38149 88.2566 66.375 0.00249135 0.0113081 2023-10-09T04:22:45.041 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:45 smithi163 ceph-mon[213841]: pgmap v1067: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 7099/100368 objects degraded (7.073%); 1141/100368 objects misplaced (1.137%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:22:45.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:45.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:45.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:45 smithi163 ceph-mon[221067]: pgmap v1067: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 7099/100368 objects degraded (7.073%); 1141/100368 objects misplaced (1.137%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:22:45.384 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:45.411 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:45 smithi181 ceph-mon[196979]: pgmap v1067: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 7099/100368 objects degraded (7.073%); 1141/100368 objects misplaced (1.137%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:22:45.414 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:45.415 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:45.463 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:45.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:45.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:45.611 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:45.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 38633 38617 86.1501 29.25 0.00323752 0.0113593 2023-10-09T04:22:45.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:45.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:45.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:45.894 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:45.929 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:46.155 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:46.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:46.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:46.305 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:46.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[213841]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T04:22:46.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[213841]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[213841]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[213841]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[221067]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T04:22:46.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[221067]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[221067]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:46 smithi163 ceph-mon[221067]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.375 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:46.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:46 smithi181 ceph-mon[196979]: osdmap e398: 8 total, 8 up, 7 in 2023-10-09T04:22:46.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:46 smithi181 ceph-mon[196979]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:46 smithi181 ceph-mon[196979]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:46 smithi181 ceph-mon[196979]: failed to encode map e398 with expected crc 2023-10-09T04:22:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:46.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:46.483 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:46.513 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:46.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 38971 38955 83.9088 21.125 0.00301686 0.0117248 2023-10-09T04:22:46.793 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 4, 3, 6] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:22:46.793 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:22:46.793 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T04:22:46.793 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:22:46.793 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:22:46.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T04:22:47.071 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:22:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: pgmap v1069: 119 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 5968/109788 objects degraded (5.436%); 1695/109788 objects misplaced (1.544%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:22:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T04:22:47.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2420658299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:47.083 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: pgmap v1069: 119 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 5968/109788 objects degraded (5.436%); 1695/109788 objects misplaced (1.544%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:22:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T04:22:47.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2420658299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:47.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:47.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:47.188 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:47.270 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:47.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:22:47.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:47.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:47.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: pgmap v1069: 119 pgs: 1 active+recovering+remapped, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s; 5968/109788 objects degraded (5.436%); 1695/109788 objects misplaced (1.544%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:22:47.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: osdmap e399: 8 total, 8 up, 7 in 2023-10-09T04:22:47.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: failed to encode map e399 with expected crc 2023-10-09T04:22:47.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2420658299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:47.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:22:47.533 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:47.565 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:47.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 39749 39733 82.7331 48.625 0.00699858 0.0120824 2023-10-09T04:22:48.051 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:48.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:48.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:48.157 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:48.218 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:48.312 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:48.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:48.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:48.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:22:48.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T04:22:48.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.327 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5968/109788 objects degraded (5.436%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:48 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5968/109788 objects degraded (5.436%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:48.418 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:48.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:22:48.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:22:48.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: osdmap e400: 8 total, 8 up, 7 in 2023-10-09T04:22:48.466 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: failed to encode map e400 with expected crc 2023-10-09T04:22:48.467 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:48 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5968/109788 objects degraded (5.436%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:48.470 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:48.486 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:48.542 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:48.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:48.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:48.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97814 9.8 9:1e2e17f2:::benchmark_data_smithi163_464650_object97813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.94470729899999, 2023-10-09T04:22:48.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0107446179999999, 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97814 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:22:48.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710188+0000", 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:48.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710223+0000", 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5763000000000003e-05 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710243+0000", 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9536000000000001e-05 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710260+0000", 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7271000000000001e-05 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710304+0000", 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3745000000000002e-05 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710446+0000", 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014181999999999999 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710552+0000", 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010643500000000001 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.713530+0000", 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029773289999999999 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717690+0000", 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041607049999999998 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.742122+0000", 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024431905 2023-10-09T04:22:48.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.742134+0000", 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1542000000000001e-05 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506765+0000", 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76463178499999995 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506780+0000", 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4603000000000001e-05 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.720842+0000", 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2140615079999999 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.720934+0000", 2023-10-09T04:22:48.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2258999999999999e-05 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97818 9.8 9:11ed80ac:::benchmark_data_smithi163_464650_object97817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.94027614199999, 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0067211999999999, 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.668 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97818 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714618+0000", 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714643+0000", 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4170999999999999e-05 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714655+0000", 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2646000000000001e-05 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714667+0000", 2023-10-09T04:22:48.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1525000000000001e-05 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717718+0000", 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003051649 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717853+0000", 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013490299999999999 2023-10-09T04:22:48.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717991+0000", 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137861 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.723501+0000", 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055102470000000002 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.726686+0000", 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031843050000000001 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.743057+0000", 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016371198999999999 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.743067+0000", 2023-10-09T04:22:48.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0241000000000001e-05 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506817+0000", 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76374998100000002 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506828+0000", 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0964999999999999e-05 2023-10-09T04:22:48.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721244+0000", 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2144160310000001 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721342+0000", 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7343999999999995e-05 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.676 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97832 9.8 9:1e55ad63:::benchmark_data_smithi163_464650_object97831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.924220303, 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.99079312, 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.677 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97832 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730695+0000", 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8825999999999999e-05 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730702+0000", 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1600000000000001e-06 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730707+0000", 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8840000000000002e-06 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730736+0000", 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9824000000000001e-05 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730895+0000", 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015838500000000001 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.731058+0000", 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016319699999999999 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.736028+0000", 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049698779999999996 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745215+0000", 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091867310000000001 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745227+0000", 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1897e-05 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507474+0000", 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76224778000000004 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507486+0000", 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1707e-05 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721391+0000", 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.213904777 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721469+0000", 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8479999999999994e-05 2023-10-09T04:22:48.684 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97836 9.8 9:11733f2c:::benchmark_data_smithi163_464650_object97835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.910804404, 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.996319304, 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.685 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97836 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744090+0000", 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744115+0000", 2023-10-09T04:22:48.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4522999999999999e-05 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744128+0000", 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3434999999999999e-05 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744138+0000", 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0118e-05 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745249+0000", 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011104089999999999 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745254+0000", 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8379999999999998e-06 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507512+0000", 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.762257929 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507520+0000", 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9240000000000007e-06 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721492+0000", 2023-10-09T04:22:48.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.213971412 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721620+0000", 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000128085 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721817+0000", 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000196839 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725055+0000", 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032383199999999998 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732136+0000", 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070812619999999996 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732296+0000", 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015994499999999999 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740237+0000", 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079407409999999994 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740292+0000", 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5081999999999999e-05 2023-10-09T04:22:48.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740411+0000", 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119461 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97838 9.8 9:1b793ee6:::benchmark_data_smithi163_464650_object97837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:22:48.694 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.910701899, 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0048248380000002, 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97838 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744194+0000", 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744242+0000", 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8263999999999997e-05 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744253+0000", 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0642000000000001e-05 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744259+0000", 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3040000000000001e-06 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745272+0000", 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010125170000000001 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745278+0000", 2023-10-09T04:22:48.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.127e-06 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507541+0000", 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76226340000000004 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507549+0000", 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6620000000000005e-06 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725084+0000", 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2175345179999999 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725227+0000", 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000143406 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725417+0000", 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189956 2023-10-09T04:22:48.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732176+0000", 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067589599999999996 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732240+0000", 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4319000000000005e-05 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740463+0000", 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082230529999999993 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.748849+0000", 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083855000000000006 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.748896+0000", 2023-10-09T04:22:48.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7463000000000003e-05 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749019+0000", 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012314499999999999 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97866 9.8 9:1cea5b31:::benchmark_data_smithi163_464650_object97865:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:22:48.703 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.87919449099999, 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.973609446, 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97866 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775722+0000", 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0279999999999999e-05 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775729+0000", 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.707 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:22:48.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775734+0000", 2023-10-09T04:22:48.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1359999999999996e-06 2023-10-09T04:22:48.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775796+0000", 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2526000000000004e-05 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775811+0000", 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4799e-05 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507569+0000", 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.73175752299999997 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507576+0000", 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728482+0000", 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2209053270000001 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728643+0000", 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016166299999999999 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728783+0000", 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000139185 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732184+0000", 2023-10-09T04:22:48.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034017660000000001 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732369+0000", 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018494 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740499+0000", 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0081298159999999998 2023-10-09T04:22:48.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749147+0000", 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086481760000000005 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749200+0000", 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2543999999999997e-05 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749311+0000", 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111702 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97872 9.8 9:10a54521:::benchmark_data_smithi163_464650_object97871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:22:48.715 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.86919127199999, 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.963828709, 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97872 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:22:48.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785703+0000", 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785769+0000", 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6533999999999997e-05 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785789+0000", 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785799+0000", 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5719999999999992e-06 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785910+0000", 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111128 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785917+0000", 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3359999999999997e-06 2023-10-09T04:22:48.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507597+0000", 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.72167993600000002 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507606+0000", 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1589999999999993e-06 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732411+0000", 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.224805452 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732576+0000", 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016478 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732767+0000", 2023-10-09T04:22:48.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019163200000000001 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740425+0000", 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076572660000000002 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740534+0000", 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109341 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749083+0000", 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085489380000000007 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749377+0000", 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029441100000000001 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749423+0000", 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5676000000000001e-05 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749534+0000", 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110911 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97874 9.8 9:191c8b34:::benchmark_data_smithi163_464650_object97873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:22:48.724 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.86908175400001, 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9835905089999999, 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97874 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785814+0000", 2023-10-09T04:22:48.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785855+0000", 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0932e-05 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785866+0000", 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0762e-05 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785871+0000", 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7550000000000002e-06 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785935+0000", 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4851999999999995e-05 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785940+0000", 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2119999999999997e-06 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507627+0000", 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.72168717900000001 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507635+0000", 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3699999999999995e-06 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736110+0000", 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2284743970000001 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736281+0000", 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017120699999999999 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736480+0000", 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019965799999999999 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740574+0000", 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0040938049999999998 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749102+0000", 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085278049999999994 2023-10-09T04:22:48.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753493+0000", 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043906470000000001 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769243+0000", 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015749984000000002 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769290+0000", 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7129000000000002e-05 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769405+0000", 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115487 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.733 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97892 9.8 9:1aaa6a58:::benchmark_data_smithi163_464650_object97891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.846266415, 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9609627970000001, 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97892 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808629+0000", 2023-10-09T04:22:48.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808672+0000", 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3235000000000002e-05 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808684+0000", 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2139e-05 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808692+0000", 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6420000000000007e-06 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808752+0000", 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9707999999999998e-05 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808760+0000", 2023-10-09T04:22:48.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3569999999999998e-06 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507656+0000", 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.69889576899999994 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507663+0000", 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6120000000000001e-06 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740602+0000", 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2329382529999999 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740744+0000", 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000142769 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740917+0000", 2023-10-09T04:22:48.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017210500000000001 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753421+0000", 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012504678999999999 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753524+0000", 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102972 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.761433+0000", 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079086420000000004 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769451+0000", 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080184650000000007 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769498+0000", 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6409999999999998e-05 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769593+0000", 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5168999999999996e-05 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97904 9.8 9:1009d856:::benchmark_data_smithi163_464650_object97903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 186.833251413, 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9481932799999999, 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97904 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.743 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821643+0000", 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821692+0000", 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9016e-05 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821706+0000", 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3781000000000001e-05 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821713+0000", 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3730000000000004e-06 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821759+0000", 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5587999999999998e-05 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821769+0000", 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9369999999999999e-06 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507684+0000", 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.68591571799999995 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507693+0000", 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1689999999999992e-06 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745049+0000", 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2373563329999999 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745227+0000", 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017849599999999999 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745426+0000", 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019907700000000001 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753455+0000", 2023-10-09T04:22:48.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080286279999999995 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753558+0000", 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010336399999999999 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.761441+0000", 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078828909999999995 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769701+0000", 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082594929999999997 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769746+0000", 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4821999999999999e-05 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769838+0000", 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2744000000000005e-05 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181583 9.5 9:accfeffe:::benchmark_data_smithi163_464650_object82857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.378559+0000", 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 73.276337943000001, 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25827423100000002, 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.753 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181583 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378559+0000", 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378559+0000", 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378558+0000", 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378561+0000", 2023-10-09T04:22:48.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7750000000000001e-06 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378567+0000", 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7100000000000004e-06 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378572+0000", 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6509999999999998e-06 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382048+0000", 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034762149999999999 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382122+0000", 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3170999999999995e-05 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382195+0000", 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3726000000000005e-05 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.385552+0000", 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003356637 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.387701+0000", 2023-10-09T04:22:48.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021486640000000002 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.636653+0000", 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.248952109 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.636786+0000", 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013360600000000001 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.636833+0000", 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6417e-05 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:14585 10.a 10:5fd7ae88:::benchmark_data_smithi163_523108_object14584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:23.225293+0000", 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.429603767, 2023-10-09T04:22:48.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25724578300000001, 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 14585 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225293+0000", 2023-10-09T04:22:48.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225293+0000", 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225291+0000", 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225382+0000", 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0706999999999997e-05 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225394+0000", 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2336e-05 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225402+0000", 2023-10-09T04:22:48.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.994e-06 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225485+0000", 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2954999999999994e-05 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225630+0000", 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000144766 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.225776+0000", 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014596800000000001 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.227290+0000", 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015145709999999999 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482362+0000", 2023-10-09T04:22:48.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25507187100000001 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482432+0000", 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9574999999999997e-05 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482487+0000", 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5483999999999999e-05 2023-10-09T04:22:48.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482539+0000", 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1416999999999999e-05 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46073.0:14584 10.d e388/383 10:bc753fe0:::benchmark_data_smithi163_523108_object14583:head v 388'874, mlcod=388'872)", 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:23.226586+0000", 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.428310051, 2023-10-09T04:22:48.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25796582499999998, 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226586+0000", 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226586+0000", 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226586+0000", 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226621+0000", 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5036999999999997e-05 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226631+0000", 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0178e-05 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226638+0000", 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6320000000000002e-06 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226692+0000", 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4338000000000001e-05 2023-10-09T04:22:48.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.226711+0000", 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8502e-05 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.484522+0000", 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25781129600000002 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.484552+0000", 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0476000000000001e-05 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:14592 10.8 10:1262ca6e:::benchmark_data_smithi163_523108_object14591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:23.228458+0000", 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 25.426438635, 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25737765499999998, 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.772 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 14592 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.228458+0000", 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.228458+0000", 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.228456+0000", 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.228478+0000", 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1342999999999999e-05 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482292+0000", 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25381468400000001 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482314+0000", 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1807999999999999e-05 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482376+0000", 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1879999999999997e-05 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482591+0000", 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021490499999999999 2023-10-09T04:22:48.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.482750+0000", 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015925499999999999 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.484562+0000", 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018118399999999999 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.484649+0000", 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7374999999999996e-05 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.485727+0000", 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001077288 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.485786+0000", 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9213999999999999e-05 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:23.485836+0000", 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9577000000000003e-05 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:17168 10.a 10:5ed8f31d:::benchmark_data_smithi163_523108_object17167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:24.790716+0000", 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.864180772000001, 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26102962000000002, 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 17168 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.780 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790716+0000", 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790716+0000", 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790715+0000", 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790739+0000", 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4661999999999999e-05 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790753+0000", 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4110999999999999e-05 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790761+0000", 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1000000000000004e-06 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790831+0000", 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9368000000000004e-05 2023-10-09T04:22:48.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791006+0000", 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017549800000000001 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791147+0000", 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014113099999999999 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.792531+0000", 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013835519999999999 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.045571+0000", 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25304050900000002 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.051646+0000", 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0060741709999999997 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.051700+0000", 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4342999999999998e-05 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.051745+0000", 2023-10-09T04:22:48.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5393999999999999e-05 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:17169 10.4 10:2492ba14:::benchmark_data_smithi163_523108_object17168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:24.790791+0000", 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.864105713000001, 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27507624000000003, 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.787 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 17169 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790791+0000", 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790791+0000", 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790790+0000", 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790819+0000", 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8547e-05 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790828+0000", 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1090000000000004e-06 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790834+0000", 2023-10-09T04:22:48.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5409999999999997e-06 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.790877+0000", 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2701999999999999e-05 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791004+0000", 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012729700000000001 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791147+0000", 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014299000000000001 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.793484+0000", 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0023361300000000001 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.793994+0000", 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00051001799999999999 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.065771+0000", 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27177721199999999 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.065825+0000", 2023-10-09T04:22:48.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4486e-05 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.065867+0000", 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1755000000000003e-05 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.794 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:17173 10.4 10:27ca85a2:::benchmark_data_smithi163_523108_object17172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:24.791362+0000", 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 23.863534491999999, 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.274603241, 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 17173 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791362+0000", 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791362+0000", 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791361+0000", 2023-10-09T04:22:48.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791382+0000", 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1285e-05 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791394+0000", 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1785e-05 2023-10-09T04:22:48.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.791405+0000", 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0933e-05 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.792215+0000", 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080982700000000005 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.792382+0000", 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016709200000000001 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.792525+0000", 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014347700000000001 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.793568+0000", 2023-10-09T04:22:48.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010429219999999999 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:24.794163+0000", 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00059440800000000005 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.065902+0000", 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27173970600000003 2023-10-09T04:22:48.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.065936+0000", 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3655999999999998e-05 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.065965+0000", 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9136999999999999e-05 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.801 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18935 10.b 10:da4c922f:::benchmark_data_smithi163_523108_object18934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.674481673999999, 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0962053709999999, 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.802 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18935 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:22:48.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980413+0000", 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980478+0000", 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4868000000000003e-05 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980501+0000", 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3396000000000001e-05 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980530+0000", 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9082999999999999e-05 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980605+0000", 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5334999999999996e-05 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980613+0000", 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6529999999999994e-06 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985700+0000", 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050872310000000002 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985710+0000", 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0077e-05 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.054356+0000", 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0686453579999999 2023-10-09T04:22:48.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.054995+0000", 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00063910900000000001 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.055173+0000", 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000178314 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058584+0000", 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034106679999999999 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058630+0000", 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6400999999999998e-05 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060716+0000", 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020859139999999999 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.075527+0000", 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014810489 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.075578+0000", 2023-10-09T04:22:48.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0995999999999998e-05 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076620+0000", 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001042608 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18945 10.b 10:db72c492:::benchmark_data_smithi163_523108_object18944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:22:48.811 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.674340177000001, 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.097054776, 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18945 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980610+0000", 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4863e-05 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980622+0000", 2023-10-09T04:22:48.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1056000000000001e-05 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980628+0000", 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4520000000000002e-06 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980665+0000", 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7508000000000001e-05 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980671+0000", 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1889999999999998e-06 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985732+0000", 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005061767 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985739+0000", 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8000000000000001e-06 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056187+0000", 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0704480949999999 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056380+0000", 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019281100000000001 2023-10-09T04:22:48.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056554+0000", 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017387899999999999 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058639+0000", 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002084776 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058684+0000", 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5627000000000002e-05 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060847+0000", 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002162298 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076681+0000", 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015834559000000002 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076730+0000", 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8708000000000002e-05 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077611+0000", 2023-10-09T04:22:48.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088110100000000002 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18948 10.b 10:d3e53c2e:::benchmark_data_smithi163_523108_object18947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 21.67426163, 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0978110240000001, 2023-10-09T04:22:48.821 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18948 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:22:48.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980634+0000", 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980675+0000", 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0608999999999997e-05 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980682+0000", 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7939999999999997e-06 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980686+0000", 2023-10-09T04:22:48.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0589999999999996e-06 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.981834+0000", 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011482910000000001 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.981840+0000", 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3459999999999999e-06 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985759+0000", 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039191210000000002 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985767+0000", 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4109999999999999e-06 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057486+0000", 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.071719098 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057617+0000", 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013104999999999999 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:22:48.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057754+0000", 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137408 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058774+0000", 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001020137 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058886+0000", 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111945 2023-10-09T04:22:48.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060884+0000", 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001997619 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077700+0000", 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016816296000000001 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077760+0000", 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0031999999999997e-05 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.078446+0000", 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068554800000000002 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.830 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:22:48.831 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:22:48.831 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:22:48.831 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 41026 41010 82.6387 79.8125 0.0122466 0.0120947 2023-10-09T04:22:48.894 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:49.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:22:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:49.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:22:49.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:49.176 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:49.256 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:49 smithi163 ceph-mon[213841]: pgmap v1072: 119 pgs: 1 active+recovering+remapped, 3 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s; 4834/114561 objects degraded (4.220%); 2714/114561 objects misplaced (2.369%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:22:49.258 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:49 smithi163 ceph-mon[221067]: pgmap v1072: 119 pgs: 1 active+recovering+remapped, 3 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s; 4834/114561 objects degraded (4.220%); 2714/114561 objects misplaced (2.369%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:22:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:49 smithi181 ceph-mon[196979]: pgmap v1072: 119 pgs: 1 active+recovering+remapped, 3 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 887 op/s; 4834/114561 objects degraded (4.220%); 2714/114561 objects misplaced (2.369%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:22:49.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 42261 42245 82.4682 77.1875 0.00215905 0.0120662 2023-10-09T04:22:50.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:22:50.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:50.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:50 smithi163 ceph-mon[213841]: pgmap v1073: 119 pgs: 1 active+recovering+remapped, 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s; 3626/117654 objects degraded (3.082%); 3180/117654 objects misplaced (2.703%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:22:50.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:50.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:50.487 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:50 smithi163 ceph-mon[221067]: pgmap v1073: 119 pgs: 1 active+recovering+remapped, 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s; 3626/117654 objects degraded (3.082%); 3180/117654 objects misplaced (2.703%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:22:50.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:50 smithi181 ceph-mon[196979]: pgmap v1073: 119 pgs: 1 active+recovering+remapped, 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s; 3626/117654 objects degraded (3.082%); 3180/117654 objects misplaced (2.703%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:22:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:50.716 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 43476 43460 82.2702 75.9375 0.0097973 0.0121499 2023-10-09T04:22:50.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:22:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:22:50] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-10-09T04:22:51.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:22:51.235 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:51.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:51.344 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:51.445 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:51.554 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:51.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:51.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:51.685 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:51.715 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 44607 44591 81.9295 70.6875 0.00958205 0.0121515 2023-10-09T04:22:52.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:52.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:52.173 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:22:52.264 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:52.716 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 45869 45853 81.8421 78.875 0.00588667 0.0122129 2023-10-09T04:22:52.748 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:53.034 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:22:53.034 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T04:22:53.034 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:22:53.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:52 smithi181 ceph-mon[196979]: pgmap v1074: 119 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 3626/129114 objects degraded (2.808%); 1210/129114 objects misplaced (0.937%); 26 MiB/s, 414 objects/s recovering 2023-10-09T04:22:53.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:52 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 3626/129114 objects degraded (2.808%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:52 smithi163 ceph-mon[221067]: pgmap v1074: 119 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 3626/129114 objects degraded (2.808%); 1210/129114 objects misplaced (0.937%); 26 MiB/s, 414 objects/s recovering 2023-10-09T04:22:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:52 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 3626/129114 objects degraded (2.808%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:52 smithi163 ceph-mon[213841]: pgmap v1074: 119 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 3626/129114 objects degraded (2.808%); 1210/129114 objects misplaced (0.937%); 26 MiB/s, 414 objects/s recovering 2023-10-09T04:22:53.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:52 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 3626/129114 objects degraded (2.808%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:53.524 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:22:53.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:53.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:22:53.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:53.680 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:53.717 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 46935 46919 81.4171 66.625 0.0095435 0.0122769 2023-10-09T04:22:53.762 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:53.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:53.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:53.869 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:53.899 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:54.354 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:54.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:54.374 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:22:54.426 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:22:54.426 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:22:54.445 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:22:54.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:54.467 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:22:54.488 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:54.488 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:22:54.512 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:22:54.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:54.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:54.538 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:54.544 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.546 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:22:54.637 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:54.685 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:54.717 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 47143 47127 79.5687 13 0.00698098 0.0123305 2023-10-09T04:22:55.034 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:55.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:55.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:55.166 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:55.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: pgmap v1075: 119 pgs: 4 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 2437/130164 objects degraded (1.872%); 2374/130164 objects misplaced (1.824%); 22 MiB/s, 356 objects/s recovering 2023-10-09T04:22:55.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: osdmap e401: 8 total, 8 up, 7 in 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1764596519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: pgmap v1075: 119 pgs: 4 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 2437/130164 objects degraded (1.872%); 2374/130164 objects misplaced (1.824%); 22 MiB/s, 356 objects/s recovering 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: osdmap e401: 8 total, 8 up, 7 in 2023-10-09T04:22:55.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1764596519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:55.212 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:55.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:55.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:55.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: pgmap v1075: 119 pgs: 4 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 2437/130164 objects degraded (1.872%); 2374/130164 objects misplaced (1.824%); 22 MiB/s, 356 objects/s recovering 2023-10-09T04:22:55.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: osdmap e401: 8 total, 8 up, 7 in 2023-10-09T04:22:55.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: failed to encode map e401 with expected crc 2023-10-09T04:22:55.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1764596519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:22:55.318 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:55.348 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:55.717 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 47341 47325 77.8011 12.375 0.00276542 0.0126371 2023-10-09T04:22:56.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:55 smithi181 ceph-mon[196979]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:55 smithi181 ceph-mon[196979]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:55 smithi181 ceph-mon[196979]: osdmap e402: 8 total, 8 up, 7 in 2023-10-09T04:22:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:55 smithi181 ceph-mon[196979]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:55 smithi181 ceph-mon[196979]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:55 smithi181 ceph-mon[196979]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[213841]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[213841]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[213841]: osdmap e402: 8 total, 8 up, 7 in 2023-10-09T04:22:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[213841]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[213841]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[213841]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[221067]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[221067]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[221067]: osdmap e402: 8 total, 8 up, 7 in 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[221067]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[221067]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:55 smithi163 ceph-mon[221067]: failed to encode map e402 with expected crc 2023-10-09T04:22:56.717 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 48178 48162 77.1476 52.3125 0.00342815 0.0129475 2023-10-09T04:22:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:56 smithi181 ceph-mon[196979]: pgmap v1078: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 2437/138681 objects degraded (1.757%); 1486/138681 objects misplaced (1.072%); 28 MiB/s, 453 objects/s recovering 2023-10-09T04:22:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:56 smithi181 ceph-mon[196979]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:56 smithi181 ceph-mon[196979]: osdmap e403: 8 total, 8 up, 7 in 2023-10-09T04:22:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:56 smithi181 ceph-mon[196979]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:56 smithi181 ceph-mon[196979]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:56 smithi181 ceph-mon[196979]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[213841]: pgmap v1078: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 2437/138681 objects degraded (1.757%); 1486/138681 objects misplaced (1.072%); 28 MiB/s, 453 objects/s recovering 2023-10-09T04:22:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[213841]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[213841]: osdmap e403: 8 total, 8 up, 7 in 2023-10-09T04:22:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[213841]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[213841]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[213841]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[221067]: pgmap v1078: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 2437/138681 objects degraded (1.757%); 1486/138681 objects misplaced (1.072%); 28 MiB/s, 453 objects/s recovering 2023-10-09T04:22:57.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[221067]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[221067]: osdmap e403: 8 total, 8 up, 7 in 2023-10-09T04:22:57.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[221067]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[221067]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:56 smithi163 ceph-mon[221067]: failed to encode map e403 with expected crc 2023-10-09T04:22:57.719 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:22:57.717797+0000 min lat: 0.00100671 max lat: 2.11443 avg lat: 0.0129542 2023-10-09T04:22:57.719 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:22:57.719 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 49421 49405 77.1608 77.6875 0.0221162 0.0129542 2023-10-09T04:22:57.781 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:57.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:57.890 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:58.036 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[213841]: 1.11 scrub starts 2023-10-09T04:22:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2437/138681 objects degraded (1.757%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:58.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:58.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:58.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[221067]: 1.11 scrub starts 2023-10-09T04:22:58.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2437/138681 objects degraded (1.757%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:58.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:58.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:58.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:58.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:58.255 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:22:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:57 smithi181 ceph-mon[196979]: 1.11 scrub starts 2023-10-09T04:22:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2437/138681 objects degraded (1.757%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:22:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:58.322 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:58.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:22:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:58.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:22:58.497 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:58.542 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:58.642 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:58.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:58.693 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:22:58.694 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:22:58.717 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 50639 50623 77.1353 76.125 0.0175296 0.012957 2023-10-09T04:22:58.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:22:58.803 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:58.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:22:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:58.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:22:59.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[213841]: 1.b scrub starts 2023-10-09T04:22:59.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[213841]: pgmap v1080: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 2437/141876 objects degraded (1.718%); 1164/141876 objects misplaced (0.820%); 13 MiB/s, 205 objects/s recovering 2023-10-09T04:22:59.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:59.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:59.007 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:59.008 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1350934797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:59.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[221067]: 1.b scrub starts 2023-10-09T04:22:59.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[221067]: pgmap v1080: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 2437/141876 objects degraded (1.718%); 1164/141876 objects misplaced (0.820%); 13 MiB/s, 205 objects/s recovering 2023-10-09T04:22:59.008 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:59.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:59.065 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:59.145 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:59.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:59.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:22:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:59 smithi181 ceph-mon[196979]: 1.b scrub starts 2023-10-09T04:22:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:59 smithi181 ceph-mon[196979]: pgmap v1080: 119 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 2437/141876 objects degraded (1.718%); 1164/141876 objects misplaced (0.820%); 13 MiB/s, 205 objects/s recovering 2023-10-09T04:22:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:22:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1350934797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:59.272 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:59.272 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:22:59.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:22:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1350934797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:22:59.275 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:59.316 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:22:59.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:59.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:22:59.419 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:22:59.448 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:59.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:22:59.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:59.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:22:59.576 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 4, 3, 6] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:22:59.576 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:22:59.576 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T04:22:59.577 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T04:22:59.643 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:22:59.719 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 51641 51625 76.7869 62.625 0.0140878 0.0130179 2023-10-09T04:22:59.721 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:22:59.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:22:59.758 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:22:59.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:22:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:22:59.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:22:59.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:23:00.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:00.126 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:00.204 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:00.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:00.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:00 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:23:00.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1350934797' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:00.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:00 smithi181 ceph-mon[196979]: osdmap e404: 8 total, 8 up, 7 in 2023-10-09T04:23:00.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:00.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:00.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:23:00.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1350934797' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:00.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[213841]: osdmap e404: 8 total, 8 up, 7 in 2023-10-09T04:23:00.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:00.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:23:00.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1350934797' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:00.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[221067]: osdmap e404: 8 total, 8 up, 7 in 2023-10-09T04:23:00.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:00.369 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:00.449 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:00.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:00.561 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:00.620 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:00.720 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 52827 52811 76.7222 74.125 0.00480274 0.0130281 2023-10-09T04:23:00.732 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:00.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:00.796 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:00.829 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:00.840 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:00.870 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:00.904 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:23:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:23:00] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-10-09T04:23:00.911 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:01.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:01.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:01.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:01.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[213841]: failed to encode map e404 with expected crc 2023-10-09T04:23:01.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[213841]: failed to encode map e404 with expected crc 2023-10-09T04:23:01.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[213841]: 1.11 scrub starts 2023-10-09T04:23:01.243 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[213841]: pgmap v1082: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s; 2437/144861 objects degraded (1.682%); 386/144861 objects misplaced (0.266%); 21 MiB/s, 334 objects/s recovering 2023-10-09T04:23:01.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[221067]: failed to encode map e404 with expected crc 2023-10-09T04:23:01.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[221067]: failed to encode map e404 with expected crc 2023-10-09T04:23:01.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[221067]: 1.11 scrub starts 2023-10-09T04:23:01.244 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:01 smithi163 ceph-mon[221067]: pgmap v1082: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s; 2437/144861 objects degraded (1.682%); 386/144861 objects misplaced (0.266%); 21 MiB/s, 334 objects/s recovering 2023-10-09T04:23:01.251 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:01 smithi181 ceph-mon[196979]: failed to encode map e404 with expected crc 2023-10-09T04:23:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:01 smithi181 ceph-mon[196979]: failed to encode map e404 with expected crc 2023-10-09T04:23:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:01 smithi181 ceph-mon[196979]: 1.11 scrub starts 2023-10-09T04:23:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:01 smithi181 ceph-mon[196979]: pgmap v1082: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s; 2437/144861 objects degraded (1.682%); 386/144861 objects misplaced (0.266%); 21 MiB/s, 334 objects/s recovering 2023-10-09T04:23:01.344 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:01.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:01.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:01.520 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:01.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:01.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:01.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:01.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 54121 54105 76.8164 80.875 0.00528036 0.0130138 2023-10-09T04:23:01.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:01.790 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:02.119 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:02.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:02.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:02.250 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:02.659 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:23:02.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 54726 54710 75.9499 37.8125 0.00279107 0.012998 2023-10-09T04:23:02.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:02.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:02.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:03.002 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:03.032 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:03.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:03.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:03.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:03.226 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:03.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[221067]: pgmap v1083: 119 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 2437/157017 objects degraded (1.552%); 386/157017 objects misplaced (0.246%); 13 MiB/s, 209 objects/s recovering 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[221067]: osdmap e405: 8 total, 8 up, 7 in 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[221067]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[221067]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[221067]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2437/157017 objects degraded (1.552%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[213841]: pgmap v1083: 119 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 2437/157017 objects degraded (1.552%); 386/157017 objects misplaced (0.246%); 13 MiB/s, 209 objects/s recovering 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[213841]: osdmap e405: 8 total, 8 up, 7 in 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[213841]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.253 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[213841]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[213841]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:03 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2437/157017 objects degraded (1.552%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:03.257 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:03 smithi181 ceph-mon[196979]: pgmap v1083: 119 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 2437/157017 objects degraded (1.552%); 386/157017 objects misplaced (0.246%); 13 MiB/s, 209 objects/s recovering 2023-10-09T04:23:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:03 smithi181 ceph-mon[196979]: osdmap e405: 8 total, 8 up, 7 in 2023-10-09T04:23:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:03 smithi181 ceph-mon[196979]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:03 smithi181 ceph-mon[196979]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:03 smithi181 ceph-mon[196979]: failed to encode map e405 with expected crc 2023-10-09T04:23:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:03 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2437/157017 objects degraded (1.552%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:03.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 55444 55428 75.2746 44.875 0.0107443 0.0132798 2023-10-09T04:23:03.940 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:03.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:04.051 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:04.108 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:04.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[221067]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[221067]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[221067]: osdmap e406: 8 total, 8 up, 7 in 2023-10-09T04:23:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[221067]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[221067]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[213841]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[213841]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[213841]: osdmap e406: 8 total, 8 up, 7 in 2023-10-09T04:23:04.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[213841]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.139 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:04 smithi163 ceph-mon[213841]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.198 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:04.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:04.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:04.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:04 smithi181 ceph-mon[196979]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:04 smithi181 ceph-mon[196979]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:04 smithi181 ceph-mon[196979]: osdmap e406: 8 total, 8 up, 7 in 2023-10-09T04:23:04.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:04 smithi181 ceph-mon[196979]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:04 smithi181 ceph-mon[196979]: failed to encode map e406 with expected crc 2023-10-09T04:23:04.359 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:04.382 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:04.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:04.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:04.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:04.579 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:04.648 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:04.713 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:04.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:04.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:04.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 56623 56607 75.2407 73.6875 0.0313772 0.0132836 2023-10-09T04:23:04.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:04.877 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:04.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:04.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:04.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:05.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:05.098 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:23:05.098 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:23:05.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[221067]: pgmap v1086: 119 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 2437/157947 objects degraded (1.543%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:23:05.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:05.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:05.150 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/236339299' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:23:05.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:04 smithi163 conmon[213817]: 2023-10-09T04:23:04.885+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e406 definitely_dead 0 2023-10-09T04:23:05.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[213841]: pgmap v1086: 119 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 2437/157947 objects degraded (1.543%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:23:05.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:05.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:05.151 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/236339299' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:23:05.192 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:05.203 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:23:05.246 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:23:05.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:05 smithi181 ceph-mon[196979]: pgmap v1086: 119 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 2437/157947 objects degraded (1.543%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:23:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/236339299' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:23:05.284 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:23:05.326 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:23:05.350 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.0. 2023-10-09T04:23:05.373 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:23:05.393 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:23:05.412 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:23:05.435 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:23:05.436 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.436 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.436 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.437 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.437 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.437 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.437 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.437 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:23:05.447 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:05.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:05.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:05.578 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:05.620 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:05.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:05.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 57335 57319 74.6004 44.5 0.00288911 0.0132943 2023-10-09T04:23:05.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:05.759 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:06.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/225373141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:06.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/236339299' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:23:06.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[221067]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T04:23:06.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[221067]: failed to encode map e407 with expected crc 2023-10-09T04:23:06.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[221067]: failed to encode map e407 with expected crc 2023-10-09T04:23:06.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/225373141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:06.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/236339299' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:23:06.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[213841]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T04:23:06.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[213841]: failed to encode map e407 with expected crc 2023-10-09T04:23:06.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:06 smithi163 ceph-mon[213841]: failed to encode map e407 with expected crc 2023-10-09T04:23:06.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/225373141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/236339299' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:23:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:06 smithi181 ceph-mon[196979]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T04:23:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:06 smithi181 ceph-mon[196979]: failed to encode map e407 with expected crc 2023-10-09T04:23:06.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:06 smithi181 ceph-mon[196979]: failed to encode map e407 with expected crc 2023-10-09T04:23:06.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 57338 57322 73.0822 0.1875 1.11884 0.0133331 2023-10-09T04:23:07.053 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:23:07.103 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:07.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:07.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:07.234 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:07.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:07.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:07.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: failed to encode map e407 with expected crc 2023-10-09T04:23:07.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: failed to encode map e407 with expected crc 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: pgmap v1088: 119 pgs: 2 remapped+peering, 6 peering, 111 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2437/157947 objects degraded (1.543%), 2 pgs degraded) 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[213841]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: failed to encode map e407 with expected crc 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: failed to encode map e407 with expected crc 2023-10-09T04:23:07.351 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: pgmap v1088: 119 pgs: 2 remapped+peering, 6 peering, 111 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:23:07.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2437/157947 objects degraded (1.543%), 2 pgs degraded) 2023-10-09T04:23:07.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:23:07.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T04:23:07.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.352 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:07 smithi163 ceph-mon[221067]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.389 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: failed to encode map e407 with expected crc 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: failed to encode map e407 with expected crc 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: pgmap v1088: 119 pgs: 2 remapped+peering, 6 peering, 111 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2437/157947 objects degraded (1.543%), 2 pgs degraded) 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T04:23:07.413 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.414 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:07 smithi181 ceph-mon[196979]: failed to encode map e408 with expected crc 2023-10-09T04:23:07.415 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:07.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:07.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:07.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:07.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:07.638 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:07.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 57579 57563 71.9222 15.0625 0.0107171 0.0138973 2023-10-09T04:23:07.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:07.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:07.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:07.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:07.906 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:07.994 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:08.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:08.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:08.180 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:08.273 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:08.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:08.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e409 with expected crc 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: 10.2 starting backfill to osd.0 from (0'0,388'1189] MIN to 406'3579 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e409 with expected crc 2023-10-09T04:23:08.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: 10.5 starting backfill to osd.0 from (0'0,388'1207] MIN to 406'3683 2023-10-09T04:23:08.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: 10.c starting backfill to osd.0 from (0'0,388'1137] MIN to 406'3597 2023-10-09T04:23:08.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: 10.8 starting backfill to osd.0 from (0'0,388'1208] MIN to 406'3569 2023-10-09T04:23:08.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: 10.a starting backfill to osd.0 from (0'0,388'1225] MIN to 406'3599 2023-10-09T04:23:08.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T04:23:08.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e410 with expected crc 2023-10-09T04:23:08.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[213841]: failed to encode map e410 with expected crc 2023-10-09T04:23:08.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T04:23:08.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e409 with expected crc 2023-10-09T04:23:08.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: 10.2 starting backfill to osd.0 from (0'0,388'1189] MIN to 406'3579 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e409 with expected crc 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: 10.5 starting backfill to osd.0 from (0'0,388'1207] MIN to 406'3683 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: 10.c starting backfill to osd.0 from (0'0,388'1137] MIN to 406'3597 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: 10.8 starting backfill to osd.0 from (0'0,388'1208] MIN to 406'3569 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: 10.a starting backfill to osd.0 from (0'0,388'1225] MIN to 406'3599 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e410 with expected crc 2023-10-09T04:23:08.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:08 smithi163 ceph-mon[221067]: failed to encode map e410 with expected crc 2023-10-09T04:23:08.380 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T04:23:08.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e408 with expected crc 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e409 with expected crc 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: 10.2 starting backfill to osd.0 from (0'0,388'1189] MIN to 406'3579 2023-10-09T04:23:08.438 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e409 with expected crc 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: 10.5 starting backfill to osd.0 from (0'0,388'1207] MIN to 406'3683 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: 10.c starting backfill to osd.0 from (0'0,388'1137] MIN to 406'3597 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: 10.8 starting backfill to osd.0 from (0'0,388'1208] MIN to 406'3569 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: 10.a starting backfill to osd.0 from (0'0,388'1225] MIN to 406'3599 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e410 with expected crc 2023-10-09T04:23:08.439 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:08 smithi181 ceph-mon[196979]: failed to encode map e410 with expected crc 2023-10-09T04:23:08.447 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:08.485 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:08.554 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:08.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:08.663 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:08.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 58562 58546 71.7165 61.4375 0.0155268 0.013938 2023-10-09T04:23:08.734 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:08.829 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:08.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:08.947 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:09.014 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:09.177 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:09.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:09.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:09.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: 10.6 starting backfill to osd.0 from (0'0,388'1134] MIN to 406'3487 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: 10.b starting backfill to osd.0 from (0'0,398'2424] MIN to 406'3637 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: pgmap v1092: 119 pgs: 2 remapped+peering, 17 peering, 100 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:23:09.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2024355129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: 10.6 starting backfill to osd.0 from (0'0,388'1134] MIN to 406'3487 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: 10.b starting backfill to osd.0 from (0'0,398'2424] MIN to 406'3637 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: pgmap v1092: 119 pgs: 2 remapped+peering, 17 peering, 100 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2024355129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:09.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e409 with expected crc 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: 10.6 starting backfill to osd.0 from (0'0,388'1134] MIN to 406'3487 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: 10.b starting backfill to osd.0 from (0'0,398'2424] MIN to 406'3637 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.357 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e410 with expected crc 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: pgmap v1092: 119 pgs: 2 remapped+peering, 17 peering, 100 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2024355129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:09.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:09.363 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:09.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:09.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:09.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:09.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:23:09.518 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:09.545 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:09.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:09.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:09.662 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:23:09.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:09.721 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 59369 59353 71.3074 50.4375 0.0220683 0.0140181 2023-10-09T04:23:09.767 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:10.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:23:10.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T04:23:10.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:10.404 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:10.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:10.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:10.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: failed to encode map e411 with expected crc 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T04:23:10.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:10.570 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:10.722 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 60321 60305 71.0836 59.5 0.0213521 0.0140625 2023-10-09T04:23:10.796 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:23:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:23:10] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-10-09T04:23:10.815 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:10.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:10.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:10.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:10.986 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:11.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:11 smithi163 ceph-mon[221067]: failed to encode map e412 with expected crc 2023-10-09T04:23:11.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:11 smithi163 ceph-mon[221067]: failed to encode map e412 with expected crc 2023-10-09T04:23:11.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:11 smithi163 ceph-mon[221067]: pgmap v1095: 119 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 remapped+peering, 18 peering, 95 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 MiB/s wr, 307 op/s; 14854/173655 objects misplaced (8.554%); 67 KiB/s, 1 objects/s recovering 2023-10-09T04:23:11.082 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:11.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:11.110 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:11.192 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:11.267 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131742 8.b 8:d953a743:::benchmark_data_smithi163_404719_object41158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.74536544099999, 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2377822549999999, 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.268 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131742 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516187+0000", 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516190+0000", 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2799999999999999e-06 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516199+0000", 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2289999999999992e-06 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516205+0000", 2023-10-09T04:23:11.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5300000000000004e-06 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516277+0000", 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2401e-05 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.516293+0000", 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5812e-05 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.647614+0000", 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1313209529999999 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.647795+0000", 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018145100000000001 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.647961+0000", 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165184 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659048+0000", 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011087575000000001 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659058+0000", 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0154e-05 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.666851+0000", 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0077926530000000001 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.750590+0000", 2023-10-09T04:23:11.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.083738669000000002 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.750667+0000", 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7825999999999997e-05 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.753969+0000", 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033019490000000002 2023-10-09T04:23:11.276 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131768 8.b 8:d94053f1:::benchmark_data_smithi163_404719_object41184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.68934643099999, 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1915944220000001, 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.277 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131768 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572206+0000", 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572209+0000", 2023-10-09T04:23:11.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2880000000000001e-06 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572218+0000", 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3899999999999999e-06 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572225+0000", 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2330000000000003e-06 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603410+0000", 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031185429000000001 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603422+0000", 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2248e-05 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654444+0000", 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0510221209999999 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654564+0000", 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012014 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:23:11.282 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.654689+0000", 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012443999999999999 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.666895+0000", 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012206318000000001 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679418+0000", 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012522881 2023-10-09T04:23:11.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679478+0000", 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0134999999999999e-05 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.760738+0000", 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.081259972999999999 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.760812+0000", 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3843000000000004e-05 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.763801+0000", 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029885020000000001 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.285 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131772 8.b 8:ddae5715:::benchmark_data_smithi163_404719_object41188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.68931630399999, 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1946515559999999, 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:23:11.286 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131772 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572236+0000", 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572239+0000", 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7470000000000001e-06 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572248+0000", 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0780000000000002e-06 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.572253+0000", 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3360000000000002e-06 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603454+0000", 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.031201550000000002 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.603462+0000", 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5059999999999998e-06 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659085+0000", 2023-10-09T04:23:11.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.055622584 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659249+0000", 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016422999999999999 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.659397+0000", 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014811800000000001 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679489+0000", 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020092334999999999 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679557+0000", 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8110999999999999e-05 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.684554+0000", 2023-10-09T04:23:11.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049966020000000002 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.763867+0000", 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.079313215000000006 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.763938+0000", 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0433999999999997e-05 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.766888+0000", 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029501950000000001 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:131788 8.b 8:d2de4d69:::benchmark_data_smithi163_404719_object41204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:23:11.294 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:16:23.619087+0000", 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 407.64246588700001, 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1507276369999999, 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 131788 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619087+0000", 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619087+0000", 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619085+0000", 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:23:11.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619093+0000", 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3429999999999996e-06 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619112+0000", 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9116000000000001e-05 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619122+0000", 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1449999999999997e-06 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619183+0000", 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1407000000000003e-05 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:23.619193+0000", 2023-10-09T04:23:11.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8390000000000002e-06 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.662664+0000", 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.043471458 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.662836+0000", 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017205899999999999 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.663011+0000", 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017457400000000001 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679498+0000", 2023-10-09T04:23:11.300 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:11.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016486912999999999 2023-10-09T04:23:11.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.679604+0000", 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000106127 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.689086+0000", 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0094817489999999994 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.766965+0000", 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.077878942000000007 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.767018+0000", 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3208000000000003e-05 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:16:24.769814+0000", 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027964800000000001 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:170514 8.1 8:8f7d6c46:::benchmark_data_smithi163_404719_object79930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 339.39303855200001, 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27164164200000002, 2023-10-09T04:23:11.304 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 170514 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868512+0000", 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868530+0000", 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8127000000000001e-05 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868545+0000", 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4941e-05 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868556+0000", 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1137e-05 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.871810+0000", 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032543070000000001 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.871939+0000", 2023-10-09T04:23:11.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012900100000000001 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.872059+0000", 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120219 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.878504+0000", 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064444419999999999 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.883814+0000", 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053105230000000002 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140056+0000", 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25624188199999998 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140112+0000", 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5835000000000003e-05 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140156+0000", 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3615999999999997e-05 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:63536 9.0 9:0aceaddd:::benchmark_data_smithi163_464650_object63535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 250.89294307399999, 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28875457799999998, 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 63536 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.312 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368608+0000", 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368630+0000", 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2226e-05 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368640+0000", 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8500000000000006e-06 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368647+0000", 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2179999999999996e-06 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534087+0000", 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16544025500000001 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534242+0000", 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000155162 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534420+0000", 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017757 2023-10-09T04:23:11.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.543991+0000", 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095711679999999997 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544001+0000", 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0030000000000001e-05 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.656468+0000", 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112466861 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.656522+0000", 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4855000000000002e-05 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657364+0000", 2023-10-09T04:23:11.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00084163200000000004 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:63537 9.0 9:03499599:::benchmark_data_smithi163_464650_object63536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 250.89112096599999, 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28755178100000001, 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.319 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 63537 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370430+0000", 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370480+0000", 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9536000000000001e-05 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370503+0000", 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3405e-05 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370518+0000", 2023-10-09T04:23:11.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4299e-05 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536564+0000", 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16604591899999999 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536746+0000", 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018275999999999999 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536962+0000", 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021562400000000001 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544049+0000", 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070868019999999997 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544171+0000", 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012230300000000001 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657426+0000", 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.113255047 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657481+0000", 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4827999999999998e-05 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657983+0000", 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00050239800000000004 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:108768 9.2 9:40bf3da0:::benchmark_data_smithi163_464650_object10042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 193.26492037200001, 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26903813999999998, 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 108768 2023-10-09T04:23:11.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996634+0000", 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3300000000000001e-06 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996640+0000", 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9089999999999997e-06 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996645+0000", 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7729999999999999e-06 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996907+0000", 2023-10-09T04:23:11.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026236199999999998 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.997011+0000", 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010401800000000001 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.997115+0000", 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103387 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.003248+0000", 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006133416 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.008657+0000", 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054094549999999996 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265580+0000", 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25692238499999998 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265624+0000", 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3671999999999998e-05 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265670+0000", 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6819e-05 2023-10-09T04:23:11.333 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181567 9.2 9:4eb3f07f:::benchmark_data_smithi163_464650_object82841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.364667+0000", 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.896885874999995, 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27748786399999997, 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.334 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181567 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364667+0000", 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364667+0000", 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364665+0000", 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364670+0000", 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8149999999999996e-06 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364679+0000", 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.166e-06 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364686+0000", 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7560000000000002e-06 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364774+0000", 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8299000000000003e-05 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364908+0000", 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000133788 2023-10-09T04:23:11.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.365032+0000", 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012386499999999999 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.368771+0000", 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037393410000000002 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371873+0000", 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003101295 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642059+0000", 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27018651900000001 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642104+0000", 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4570000000000002e-05 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642155+0000", 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0759e-05 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.341 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181570 9.2 9:4981e7d2:::benchmark_data_smithi163_464650_object82844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.371071+0000", 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.890481824999995, 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27127820699999999, 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181570 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371071+0000", 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371071+0000", 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371070+0000", 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371073+0000", 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8999999999999999e-06 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371082+0000", 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1450000000000008e-06 2023-10-09T04:23:11.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371087+0000", 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3970000000000002e-06 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.372364+0000", 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012767259999999999 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.372466+0000", 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102143 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.372568+0000", 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101891 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.375746+0000", 2023-10-09T04:23:11.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031787429999999999 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642225+0000", 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26647827499999999 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642259+0000", 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3791999999999998e-05 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642302+0000", 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3707e-05 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642349+0000", 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6697999999999999e-05 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.348 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181571 9.4 9:246f946c:::benchmark_data_smithi163_464650_object82845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.371096+0000", 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.890456232000005, 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.267938607, 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.349 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181571 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371096+0000", 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371096+0000", 2023-10-09T04:23:11.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371096+0000", 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371098+0000", 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0159999999999998e-06 2023-10-09T04:23:11.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371104+0000", 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5199999999999997e-06 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371108+0000", 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5229999999999997e-06 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371146+0000", 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7506999999999999e-05 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371272+0000", 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000126305 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371406+0000", 2023-10-09T04:23:11.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134022 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.377104+0000", 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056977080000000001 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.377124+0000", 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0469999999999999e-05 2023-10-09T04:23:11.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638926+0000", 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26180188199999999 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638974+0000", 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8143000000000003e-05 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.639035+0000", 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0849e-05 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181574 9.4 9:2aa97e88:::benchmark_data_smithi163_464650_object82848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.373839+0000", 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.887713943999998, 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26533669700000001, 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.356 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181574 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.373839+0000", 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.357 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.373839+0000", 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.373837+0000", 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.373843+0000", 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4500000000000003e-06 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.373855+0000", 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1962e-05 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.373863+0000", 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6090000000000001e-06 2023-10-09T04:23:11.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.374157+0000", 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029312799999999998 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.374269+0000", 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112779 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.360 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.374381+0000", 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011139800000000001 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.377217+0000", 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002836683 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.361 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.380146+0000", 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029287279999999998 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.639095+0000", 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25894927499999998 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.639140+0000", 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4212999999999998e-05 2023-10-09T04:23:11.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.639175+0000", 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5689000000000003e-05 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181585 9.2 9:40695247:::benchmark_data_smithi163_464650_object82859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.378714+0000", 2023-10-09T04:23:11.363 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.882838602999996, 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27045982299999999, 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181585 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.364 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378714+0000", 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378714+0000", 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378713+0000", 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378719+0000", 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8810000000000001e-06 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378731+0000", 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2203e-05 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378739+0000", 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8660000000000006e-06 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.381455+0000", 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027158270000000001 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.381579+0000", 2023-10-09T04:23:11.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012458799999999999 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.639098+0000", 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25751881399999998 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.645662+0000", 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065638379999999998 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.368 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.648839+0000", 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031765949999999999 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649077+0000", 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023882100000000001 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.369 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649128+0000", 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0918000000000003e-05 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649174+0000", 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5463000000000002e-05 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.370 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:181584 9.f e368/310 9:ff130522:::benchmark_data_smithi163_464650_object82858:head v 368'11395, mlcod=368'11392)", 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.382550+0000", 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.879002127000007, 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26560066500000001, 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382550+0000", 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382550+0000", 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382549+0000", 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382555+0000", 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.677e-06 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638736+0000", 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25618118400000001 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638744+0000", 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7209999999999994e-06 2023-10-09T04:23:11.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638785+0000", 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0581999999999999e-05 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638809+0000", 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4122e-05 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.648117+0000", 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0093081410000000007 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.648151+0000", 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4325000000000001e-05 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.375 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:181580 9.d e368/354 9:b6667363:::benchmark_data_smithi163_464650_object82854:head v 368'11326, mlcod=368'11323)", 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.382578+0000", 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 95.878974459000005, 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.266486371, 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382578+0000", 2023-10-09T04:23:11.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382578+0000", 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382578+0000", 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.382583+0000", 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1399999999999999e-06 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638766+0000", 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25618320700000002 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638773+0000", 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4470000000000003e-06 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638802+0000", 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9048999999999999e-05 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.638818+0000", 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6701000000000002e-05 2023-10-09T04:23:11.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649033+0000", 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010214702000000001 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649064+0000", 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1473000000000002e-05 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.380 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18929 10.2 10:4b019e5b:::benchmark_data_smithi163_523108_object18928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.977843+0000", 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 45.283709991000002, 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.073312767, 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:11.381 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18929 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977843+0000", 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977843+0000", 2023-10-09T04:23:11.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977840+0000", 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977867+0000", 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6684000000000001e-05 2023-10-09T04:23:11.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977892+0000", 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4726e-05 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977903+0000", 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1025e-05 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.384 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977980+0000", 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7068000000000001e-05 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978175+0000", 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000195582 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978346+0000", 2023-10-09T04:23:11.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017094699999999999 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.980370+0000", 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020234089999999999 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.982534+0000", 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021639900000000002 2023-10-09T04:23:11.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.001908+0000", 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019374031999999999 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.001922+0000", 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3845e-05 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.988348+0000", 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98642666099999998 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.988365+0000", 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6818000000000001e-05 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.051057+0000", 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.062691794 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.051155+0000", 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8371999999999999e-05 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.389 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57314 10.2 10:4147658d:::benchmark_data_smithi163_523108_object57313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.325214+0000", 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.9363382749999998, 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1597814309999999, 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57314 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.390 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325214+0000", 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325214+0000", 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325213+0000", 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325262+0000", 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9441999999999999e-05 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.392 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325276+0000", 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4296e-05 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325287+0000", 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0152999999999999e-05 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.393 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326696+0000", 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014094929999999999 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326849+0000", 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000152876 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.395 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326988+0000", 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000139398 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.336681+0000", 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096929470000000004 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.341192+0000", 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045103360000000002 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.341202+0000", 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0454e-05 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329833+0000", 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98863050699999999 2023-10-09T04:23:11.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329846+0000", 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3923e-05 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470799+0000", 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1409526269999999 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.471670+0000", 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00087087700000000004 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.484877+0000", 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013206542999999999 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.484996+0000", 2023-10-09T04:23:11.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011919200000000001 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57316 10.2 10:4066bc76:::benchmark_data_smithi163_523108_object57315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.326433+0000", 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.9351199750000001, 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1601299439999999, 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.400 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57316 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326433+0000", 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326433+0000", 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326432+0000", 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326466+0000", 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4625999999999997e-05 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326483+0000", 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7314000000000001e-05 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326493+0000", 2023-10-09T04:23:11.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2620000000000004e-06 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.329254+0000", 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027612349999999999 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.329406+0000", 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000152328 2023-10-09T04:23:11.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.329576+0000", 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016951800000000001 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.336689+0000", 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071129790000000002 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.405 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342259+0000", 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055705290000000003 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342272+0000", 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2525e-05 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329893+0000", 2023-10-09T04:23:11.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98762081999999995 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329904+0000", 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.114e-05 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.472026+0000", 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1421222550000001 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.472797+0000", 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000771256 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486491+0000", 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013694045 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486563+0000", 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1188999999999999e-05 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57320 10.2 10:473b6798:::benchmark_data_smithi163_523108_object57319:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.326501+0000", 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.9350518350000003, 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1599652659999999, 2023-10-09T04:23:11.409 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57320 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326501+0000", 2023-10-09T04:23:11.410 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326501+0000", 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326500+0000", 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326524+0000", 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3416000000000001e-05 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326531+0000", 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3289999999999996e-06 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326537+0000", 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7470000000000002e-06 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.331699+0000", 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005161845 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.331842+0000", 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014325699999999999 2023-10-09T04:23:11.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.331981+0000", 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013920499999999999 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.336694+0000", 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047133059999999996 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.415 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342967+0000", 2023-10-09T04:23:11.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062731410000000003 2023-10-09T04:23:11.417 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342980+0000", 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2309e-05 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329939+0000", 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98695938800000005 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329950+0000", 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1323e-05 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.472855+0000", 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1429041090000001 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:23:11.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.473758+0000", 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090376599999999997 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486384+0000", 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012625716 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486466+0000", 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1926000000000001e-05 2023-10-09T04:23:11.420 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57329 10.2 10:47781a5f:::benchmark_data_smithi163_523108_object57328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.358571+0000", 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 5.9029816620000002, 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1352695960000001, 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:11.421 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57329 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358571+0000", 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:11.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358571+0000", 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358569+0000", 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358592+0000", 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2637e-05 2023-10-09T04:23:11.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358609+0000", 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6674e-05 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358620+0000", 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0896e-05 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358699+0000", 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9841999999999999e-05 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358716+0000", 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6494000000000002e-05 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329977+0000", 2023-10-09T04:23:11.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97126138699999998 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329986+0000", 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8079999999999994e-06 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.477576+0000", 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1475897989999999 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.477764+0000", 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188104 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.477940+0000", 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000175589 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.483913+0000", 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0059738120000000002 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.485670+0000", 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001756514 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486360+0000", 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00069008199999999996 2023-10-09T04:23:11.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.493667+0000", 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0073070449999999999 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.493726+0000", 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8751e-05 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:11.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.493841+0000", 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000114654 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:11.430 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:23:11.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:11 smithi181 ceph-mon[196979]: failed to encode map e412 with expected crc 2023-10-09T04:23:11.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:11 smithi181 ceph-mon[196979]: failed to encode map e412 with expected crc 2023-10-09T04:23:11.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:11 smithi181 ceph-mon[196979]: pgmap v1095: 119 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 remapped+peering, 18 peering, 95 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 MiB/s wr, 307 op/s; 14854/173655 objects misplaced (8.554%); 67 KiB/s, 1 objects/s recovering 2023-10-09T04:23:11.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:11 smithi163 ceph-mon[213841]: failed to encode map e412 with expected crc 2023-10-09T04:23:11.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:11 smithi163 ceph-mon[213841]: failed to encode map e412 with expected crc 2023-10-09T04:23:11.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:11 smithi163 ceph-mon[213841]: pgmap v1095: 119 pgs: 2 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 remapped+peering, 18 peering, 95 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 MiB/s wr, 307 op/s; 14854/173655 objects misplaced (8.554%); 67 KiB/s, 1 objects/s recovering 2023-10-09T04:23:11.474 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:11.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:11.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:11.643 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:11.722 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 61237 61221 70.8273 57.25 0.00766781 0.0140977 2023-10-09T04:23:12.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:12 smithi163 ceph-mon[221067]: pgmap v1096: 119 pgs: 2 active+recovering+undersized+remapped, 107 active+clean, 1 peering, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 679 op/s; 2353/179700 objects degraded (1.309%); 30546/179700 objects misplaced (16.998%); 6.7 MiB/s, 108 objects/s recovering 2023-10-09T04:23:12.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:12 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 2353/179700 objects degraded (1.309%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:12.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:12 smithi163 ceph-mon[213841]: pgmap v1096: 119 pgs: 2 active+recovering+undersized+remapped, 107 active+clean, 1 peering, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 679 op/s; 2353/179700 objects degraded (1.309%); 30546/179700 objects misplaced (16.998%); 6.7 MiB/s, 108 objects/s recovering 2023-10-09T04:23:12.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:12 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 2353/179700 objects degraded (1.309%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:12.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:12.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:12 smithi181 ceph-mon[196979]: pgmap v1096: 119 pgs: 2 active+recovering+undersized+remapped, 107 active+clean, 1 peering, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 679 op/s; 2353/179700 objects degraded (1.309%); 30546/179700 objects misplaced (16.998%); 6.7 MiB/s, 108 objects/s recovering 2023-10-09T04:23:12.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:12 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 2353/179700 objects degraded (1.309%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:12.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:12.569 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:12.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:12.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:12.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:12.722 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 61859 61843 70.2464 38.875 0.0141341 0.0142295 2023-10-09T04:23:12.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:23:12.789 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:12.833 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:23:12.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:12.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:13.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:13.131 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:13.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:13.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:13.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:13.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:13.380 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 4, 3, 6, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:23:13.380 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:23:13.381 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [1, 2, 5, 7, 4, 3, 6, 0] 2023-10-09T04:23:13.381 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T04:23:13.410 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:13.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:13.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:13.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:13.615 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:13.662 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:13.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 62739 62723 69.9726 55 0.0261653 0.0142852 2023-10-09T04:23:13.749 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:13.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:13.780 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:13.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:13.919 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:13.946 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:23:13.951 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:13.986 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:23:14.036 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:23:14.080 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:23:14.127 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:23:14.174 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:23:14.204 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1578689050' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:14.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1578689050' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:14.213 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:14.223 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:23:14.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:14.259 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:23:14.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1578689050' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.287 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:23:14.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:14.411 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:14.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 63651 63635 69.745 57 0.0120312 0.0143317 2023-10-09T04:23:15.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:14 smithi181 ceph-mon[196979]: pgmap v1097: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 34 MiB/s wr, 536 op/s; 2353/180489 objects degraded (1.304%); 30809/180489 objects misplaced (17.070%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:23:15.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:14 smithi163 ceph-mon[213841]: pgmap v1097: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 34 MiB/s wr, 536 op/s; 2353/180489 objects degraded (1.304%); 30809/180489 objects misplaced (17.070%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:23:15.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:14 smithi163 ceph-mon[221067]: pgmap v1097: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 34 MiB/s wr, 536 op/s; 2353/180489 objects degraded (1.304%); 30809/180489 objects misplaced (17.070%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:23:15.440 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:15.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:15.549 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:15.645 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:15.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 64568 64552 69.5306 57.3125 0.0075377 0.0143775 2023-10-09T04:23:16.248 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:16.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:16.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:16.391 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:16.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 65339 65323 69.1689 48.1875 0.0187722 0.0144513 2023-10-09T04:23:16.782 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:16.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:16.800 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:16.891 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:16.958 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:17.128 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:17.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:17.216 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:16 smithi163 ceph-mon[213841]: pgmap v1098: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 2353/189351 objects degraded (1.243%); 31444/189351 objects misplaced (16.606%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:23:17.216 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:16 smithi163 ceph-mon[221067]: pgmap v1098: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 2353/189351 objects degraded (1.243%); 31444/189351 objects misplaced (16.606%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:23:17.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:17.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:16 smithi181 ceph-mon[196979]: pgmap v1098: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 2353/189351 objects degraded (1.243%); 31444/189351 objects misplaced (16.606%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:23:17.331 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:17.457 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:23:17.457 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:23:17.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:17.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:17.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:23:17.673 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:17.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:23:17.724914+0000 min lat: 0.00100671 max lat: 2.19023 avg lat: 0.0144941 2023-10-09T04:23:17.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:23:17.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 66248 66232 68.9629 56.8125 0.00518661 0.0144941 2023-10-09T04:23:17.766 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:18.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:17 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2353/189351 objects degraded (1.243%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:18.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:17 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2353/189351 objects degraded (1.243%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:18.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:17 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2353/189351 objects degraded (1.243%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:18.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 67038 67022 68.6418 49.375 0.0106109 0.0145633 2023-10-09T04:23:19.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:23:19.032 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:23:19.080 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:19.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:19.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:19 smithi163 ceph-mon[213841]: pgmap v1099: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 2353/192345 objects degraded (1.223%); 31924/192345 objects misplaced (16.597%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2881238335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:19.232 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:19 smithi163 ceph-mon[221067]: pgmap v1099: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 2353/192345 objects degraded (1.223%); 31924/192345 objects misplaced (16.597%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2881238335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:19.233 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:19.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:19 smithi181 ceph-mon[196979]: pgmap v1099: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 2353/192345 objects degraded (1.223%); 31924/192345 objects misplaced (16.597%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T04:23:19.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2881238335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:19.545 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:19.657 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:19.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:19.724 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 67709 67693 68.2112 41.9375 0.0155808 0.0146547 2023-10-09T04:23:19.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:19.812 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:19.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:19.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:20.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:20.114 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:20.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:20.272 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:20.275 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:20.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.302 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:20.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:20.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:20.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:20.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:20.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:20.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:20.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T04:23:20.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: failed to encode map e413 with expected crc 2023-10-09T04:23:20.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:20.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:20.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:20.312 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:20.404 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:20.489 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:20.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 68607 68591 68.0192 56.125 0.0198586 0.0146948 2023-10-09T04:23:20.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:20.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:20.852 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:20.892 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:23:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:23:20] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2023-10-09T04:23:20.892 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:20 smithi163 conmon[213817]: 2023-10-09T04:23:20.890+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e413 definitely_dead 0 2023-10-09T04:23:20.912 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:21.017 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:21.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:21.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:21.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[221067]: pgmap v1101: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 2.8 KiB/s rd, 47 MiB/s wr, 753 op/s; 2353/196197 objects degraded (1.199%); 32934/196197 objects misplaced (16.786%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:23:21.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2893010441' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:23:21.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:23:21.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[221067]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:23:21.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[213841]: pgmap v1101: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 2.8 KiB/s rd, 47 MiB/s wr, 753 op/s; 2353/196197 objects degraded (1.199%); 32934/196197 objects misplaced (16.786%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:23:21.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2893010441' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:23:21.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:23:21.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:21 smithi163 ceph-mon[213841]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:23:21.207 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:21.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:21 smithi181 ceph-mon[196979]: pgmap v1101: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 2.8 KiB/s rd, 47 MiB/s wr, 753 op/s; 2353/196197 objects degraded (1.199%); 32934/196197 objects misplaced (16.786%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:23:21.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2893010441' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:23:21.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:21 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:23:21.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:21 smithi181 ceph-mon[196979]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-09T04:23:21.364 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.4. 2023-10-09T04:23:21.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 69190 69174 67.5258 36.4375 0.00289569 0.0147207 2023-10-09T04:23:21.808 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:21.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:21.912 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:21.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:23:21.992 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:22.040 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:22.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:22.146 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:22.172 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:22.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 69190 69174 66.4873 0 - 0.0147207 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:22 smithi181 ceph-mon[196979]: pgmap v1103: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 2353/204717 objects degraded (1.149%); 33270/204717 objects misplaced (16.252%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:23:22.831 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[221067]: pgmap v1103: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 2353/204717 objects degraded (1.149%); 33270/204717 objects misplaced (16.252%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:23:22.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: osdmap e414: 8 total, 8 up, 7 in 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: failed to encode map e414 with expected crc 2023-10-09T04:23:22.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:22 smithi163 ceph-mon[213841]: pgmap v1103: 119 pgs: 2 active+recovering+undersized+remapped, 108 active+clean, 3 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 2353/204717 objects degraded (1.149%); 33270/204717 objects misplaced (16.252%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T04:23:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2353/204717 objects degraded (1.149%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:23 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2353/204717 objects degraded (1.149%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:23.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 69203 69187 65.4925 0.40625 0.0169569 0.014908 2023-10-09T04:23:23.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: osdmap e415: 8 total, 8 up, 7 in 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: failed to encode map e415 with expected crc 2023-10-09T04:23:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:23 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2353/204717 objects degraded (1.149%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:23.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:23.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:23.929 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:24.038 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:24.075 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:24.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:24.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:24.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:24.241 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: 10.2 continuing backfill to osd.0 from (0'0,413'4329] MIN to 413'4329 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.362 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: 10.6 continuing backfill to osd.0 from (0'0,413'4222] MIN to 413'4222 2023-10-09T04:23:24.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: 10.c continuing backfill to osd.0 from (0'0,413'4316] MIN to 413'4316 2023-10-09T04:23:24.363 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:24 smithi181 ceph-mon[196979]: pgmap v1106: 119 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 2 active+recovering+undersized+remapped, 95 active+clean, 2 active+recovery_wait+remapped, 3 active+undersized+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 8.5 KiB/s rd, 51 MiB/s wr, 823 op/s; 6685/205227 objects degraded (3.257%); 33403/205227 objects misplaced (16.276%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:23:24.484 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.484 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: 10.2 continuing backfill to osd.0 from (0'0,413'4329] MIN to 413'4329 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: 10.6 continuing backfill to osd.0 from (0'0,413'4222] MIN to 413'4222 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: 10.c continuing backfill to osd.0 from (0'0,413'4316] MIN to 413'4316 2023-10-09T04:23:24.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[221067]: pgmap v1106: 119 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 2 active+recovering+undersized+remapped, 95 active+clean, 2 active+recovery_wait+remapped, 3 active+undersized+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 8.5 KiB/s rd, 51 MiB/s wr, 823 op/s; 6685/205227 objects degraded (3.257%); 33403/205227 objects misplaced (16.276%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: osdmap e416: 8 total, 8 up, 7 in 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: 10.2 continuing backfill to osd.0 from (0'0,413'4329] MIN to 413'4329 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.486 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: 10.6 continuing backfill to osd.0 from (0'0,413'4222] MIN to 413'4222 2023-10-09T04:23:24.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: failed to encode map e416 with expected crc 2023-10-09T04:23:24.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: 10.c continuing backfill to osd.0 from (0'0,413'4316] MIN to 413'4316 2023-10-09T04:23:24.487 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:24 smithi163 ceph-mon[213841]: pgmap v1106: 119 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 2 active+recovering+undersized+remapped, 95 active+clean, 2 active+recovery_wait+remapped, 3 active+undersized+remapped, 5 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 8.5 KiB/s rd, 51 MiB/s wr, 823 op/s; 6685/205227 objects degraded (3.257%); 33403/205227 objects misplaced (16.276%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:23:24.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 69203 69187 64.5153 0 - 0.014908 2023-10-09T04:23:24.818 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:24.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:24.850 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:23:24.851 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:23:24.851 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:23:24.872 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:23:24.895 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:23:24.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:24.925 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:23:24.951 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:23:24.951 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:24.972 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:23:24.992 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:23:25.000 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:25.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:25.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:25.026 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:23:25.027 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.028 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:23:25.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:25.374 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:25.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:25.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:25.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:25.573 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:25.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:25.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:25 smithi181 ceph-mon[196979]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:25 smithi181 ceph-mon[196979]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:25 smithi181 ceph-mon[196979]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T04:23:25.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:25 smithi181 ceph-mon[196979]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.629 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:25 smithi181 ceph-mon[196979]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.630 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1835150209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:25.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:25.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 69203 69187 63.5668 0 - 0.014908 2023-10-09T04:23:25.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:25.760 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:25.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:25.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[213841]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[213841]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[213841]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[213841]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[213841]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1835150209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[221067]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[221067]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[221067]: osdmap e417: 8 total, 8 up, 7 in 2023-10-09T04:23:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[221067]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[221067]: failed to encode map e417 with expected crc 2023-10-09T04:23:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1835150209' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:25.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:25.949 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:26.057 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:26.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:26.163 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:26.192 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[221067]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[221067]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[221067]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T04:23:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[221067]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[221067]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[221067]: pgmap v1109: 119 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 11 peering, 1 unknown, 2 active+recovering+undersized+remapped, 93 active+clean, 1 active+recovery_wait+remapped, 2 active+undersized+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 155 op/s; 6684/206436 objects degraded (3.238%); 33491/206436 objects misplaced (16.223%); 3.5 MiB/s, 54 objects/s recovering 2023-10-09T04:23:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[213841]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[213841]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[213841]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T04:23:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[213841]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[213841]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:26 smithi163 ceph-mon[213841]: pgmap v1109: 119 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 11 peering, 1 unknown, 2 active+recovering+undersized+remapped, 93 active+clean, 1 active+recovery_wait+remapped, 2 active+undersized+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 155 op/s; 6684/206436 objects degraded (3.238%); 33491/206436 objects misplaced (16.223%); 3.5 MiB/s, 54 objects/s recovering 2023-10-09T04:23:26.726 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 69203 69187 62.6458 0 - 0.014908 2023-10-09T04:23:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:26 smithi181 ceph-mon[196979]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:26 smithi181 ceph-mon[196979]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:26 smithi181 ceph-mon[196979]: osdmap e418: 8 total, 8 up, 7 in 2023-10-09T04:23:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:26 smithi181 ceph-mon[196979]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:26 smithi181 ceph-mon[196979]: failed to encode map e418 with expected crc 2023-10-09T04:23:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:26 smithi181 ceph-mon[196979]: pgmap v1109: 119 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 11 peering, 1 unknown, 2 active+recovering+undersized+remapped, 93 active+clean, 1 active+recovery_wait+remapped, 2 active+undersized+remapped, 6 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 155 op/s; 6684/206436 objects degraded (3.238%); 33491/206436 objects misplaced (16.223%); 3.5 MiB/s, 54 objects/s recovering 2023-10-09T04:23:27.482 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:23:27.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:27.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:27.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 69203 69187 61.7511 0 - 0.014908 2023-10-09T04:23:27.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:27.756 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:27.804 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:27.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:27.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6684/206436 objects degraded (3.238%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:27.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6684/206436 objects degraded (3.238%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:27.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:27.954 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:28.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6684/206436 objects degraded (3.238%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:28.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:28.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:28.154 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:28.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:28.229 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:28.317 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:28.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:28.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:28.453 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:28.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:28.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:28.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:28.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:28.635 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:28.663 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:23:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:28.693 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:28.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:28.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 69232 69216 60.9072 0.3625 0.0105846 0.0151482 2023-10-09T04:23:28.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:28 smithi181 ceph-mon[196979]: pgmap v1110: 119 pgs: 1 activating+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 11 peering, 1 unknown, 2 active+recovering+undersized+remapped, 95 active+clean, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 140 op/s; 6684/206871 objects degraded (3.231%); 35293/206871 objects misplaced (17.060%); 5.0 MiB/s, 1 keys/s, 80 objects/s recovering 2023-10-09T04:23:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2507600104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:23:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:28 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:23:28.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:28.830 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:28 smithi163 ceph-mon[221067]: pgmap v1110: 119 pgs: 1 activating+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 11 peering, 1 unknown, 2 active+recovering+undersized+remapped, 95 active+clean, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 140 op/s; 6684/206871 objects degraded (3.231%); 35293/206871 objects misplaced (17.060%); 5.0 MiB/s, 1 keys/s, 80 objects/s recovering 2023-10-09T04:23:28.830 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2507600104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:23:28.830 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:28 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:23:28.830 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:28 smithi163 ceph-mon[213841]: pgmap v1110: 119 pgs: 1 activating+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 11 peering, 1 unknown, 2 active+recovering+undersized+remapped, 95 active+clean, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 140 op/s; 6684/206871 objects degraded (3.231%); 35293/206871 objects misplaced (17.060%); 5.0 MiB/s, 1 keys/s, 80 objects/s recovering 2023-10-09T04:23:28.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2507600104' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:23:28.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:28 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:23:28.842 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:29.009 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:23:29.009 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:23:29.010 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:23:29.010 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-09T04:23:29.010 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-09T04:23:29.705 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:23:29.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 69398 69382 60.2055 10.375 0.0135237 0.0161889 2023-10-09T04:23:30.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:29 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:30.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:23:30.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:29 smithi181 ceph-mon[196979]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T04:23:30.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:29 smithi181 ceph-mon[196979]: failed to encode map e419 with expected crc 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[221067]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[221067]: failed to encode map e419 with expected crc 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[213841]: osdmap e419: 8 total, 8 up, 7 in 2023-10-09T04:23:30.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:29 smithi163 ceph-mon[213841]: failed to encode map e419 with expected crc 2023-10-09T04:23:30.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 69489 69473 59.4589 5.6875 0.00361151 0.0161823 2023-10-09T04:23:30.741 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:30.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:30.844 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:30.870 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:30.914 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:30.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:30 smithi181 ceph-mon[196979]: failed to encode map e419 with expected crc 2023-10-09T04:23:30.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:30 smithi181 ceph-mon[196979]: pgmap v1112: 119 pgs: 1 activating+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 107 active+clean, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.8 KiB/s rd, 7.4 MiB/s wr, 123 op/s; 6287/207348 objects degraded (3.032%); 37607/207348 objects misplaced (18.137%); 8.5 MiB/s, 1 keys/s, 136 objects/s recovering 2023-10-09T04:23:30.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:30.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:30.948 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:30 smithi163 ceph-mon[221067]: failed to encode map e419 with expected crc 2023-10-09T04:23:30.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:30 smithi163 ceph-mon[221067]: pgmap v1112: 119 pgs: 1 activating+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 107 active+clean, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.8 KiB/s rd, 7.4 MiB/s wr, 123 op/s; 6287/207348 objects degraded (3.032%); 37607/207348 objects misplaced (18.137%); 8.5 MiB/s, 1 keys/s, 136 objects/s recovering 2023-10-09T04:23:30.949 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:30.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:30 smithi163 ceph-mon[213841]: failed to encode map e419 with expected crc 2023-10-09T04:23:30.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:30 smithi163 ceph-mon[213841]: pgmap v1112: 119 pgs: 1 activating+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 107 active+clean, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.8 KiB/s rd, 7.4 MiB/s wr, 123 op/s; 6287/207348 objects degraded (3.032%); 37607/207348 objects misplaced (18.137%); 8.5 MiB/s, 1 keys/s, 136 objects/s recovering 2023-10-09T04:23:30.949 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:30.949 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:23:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:23:30] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2023-10-09T04:23:31.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:31.052 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:31.081 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:31.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:31.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:31.125 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:31.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:31.248 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18926 10.5 10:a0371204:::benchmark_data_smithi163_523108_object18925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 65.291238063999998, 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0791226709999999, 2023-10-09T04:23:31.273 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18926 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:23:31.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977985+0000", 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978010+0000", 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5281000000000001e-05 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978022+0000", 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1691e-05 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978029+0000", 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6140000000000003e-06 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978125+0000", 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5649000000000005e-05 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978259+0000", 2023-10-09T04:23:31.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013442700000000001 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978399+0000", 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013955200000000001 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.979678+0000", 2023-10-09T04:23:31.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001279362 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.980540+0000", 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086218799999999995 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.007748+0000", 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.027207293 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.007761+0000", 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3307e-05 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979759+0000", 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97199803900000004 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979772+0000", 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2935999999999999e-05 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057024+0000", 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.07725206 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057109+0000", 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4544999999999998e-05 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.281 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18936 10.8 10:1447a75e:::benchmark_data_smithi163_523108_object18935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 65.270138289000002, 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0663146870000002, 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18936 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999084+0000", 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999108+0000", 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4119000000000001e-05 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999119+0000", 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0609e-05 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999129+0000", 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0173999999999999e-05 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.005396+0000", 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062664510000000001 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.005408+0000", 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2614e-05 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.978908+0000", 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97349999099999995 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.978922+0000", 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3569e-05 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058726+0000", 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.079804481 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.059470+0000", 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00074404699999999996 2023-10-09T04:23:31.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.059625+0000", 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015495099999999999 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060888+0000", 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012629589999999999 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.061724+0000", 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083528300000000003 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.064344+0000", 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026200139999999999 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.064384+0000", 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0055999999999997e-05 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.065400+0000", 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010168359999999999 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18951 10.5 10:a56ce1b4:::benchmark_data_smithi163_523108_object18950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:23:31.290 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 65.251032280000004, 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0484555979999999, 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18951 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.291 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018190+0000", 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018218+0000", 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7906e-05 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018235+0000", 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7246999999999999e-05 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018245+0000", 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.465e-06 2023-10-09T04:23:31.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018303+0000", 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8554999999999998e-05 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018315+0000", 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2088e-05 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979816+0000", 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96150034200000001 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979836+0000", 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0537e-05 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057168+0000", 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.077331303 2023-10-09T04:23:31.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057311+0000", 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014343400000000001 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057491+0000", 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000179817 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060562+0000", 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030713649999999999 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060602+0000", 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9830999999999999e-05 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.065968+0000", 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053659409999999999 2023-10-09T04:23:31.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.066023+0000", 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5339e-05 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.066647+0000", 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062404699999999997 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.298 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38490 10.5 10:a06fa733:::benchmark_data_smithi163_523108_object38489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.200497771999999, 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0132891260000001, 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38490 2023-10-09T04:23:31.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.300 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068746+0000", 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0443000000000001e-05 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068753+0000", 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9829999999999999e-06 2023-10-09T04:23:31.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068759+0000", 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5550000000000002e-06 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.077390+0000", 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086309060000000007 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.077400+0000", 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0161e-05 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075149+0000", 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99774961600000001 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075307+0000", 2023-10-09T04:23:31.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015781200000000001 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075470+0000", 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016307 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080366+0000", 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048961869999999998 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080409+0000", 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2290999999999999e-05 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081931+0000", 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015225309999999999 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081989+0000", 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7714999999999999e-05 2023-10-09T04:23:31.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.082015+0000", 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6398999999999999e-05 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38497 10.5 10:abb9441d:::benchmark_data_smithi163_523108_object38496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.078885+0000", 2023-10-09T04:23:31.306 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.190339180000002, 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.008207925, 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38497 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.078885+0000", 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.078885+0000", 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.078883+0000", 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.308 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.078907+0000", 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4312999999999999e-05 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.078925+0000", 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7600000000000001e-05 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.078933+0000", 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1000000000000004e-06 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.079041+0000", 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107837 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.079051+0000", 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0672999999999999e-05 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.077273+0000", 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99822179300000002 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.077427+0000", 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000153654 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.077572+0000", 2023-10-09T04:23:31.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014499599999999999 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081878+0000", 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043057219999999997 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081908+0000", 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0380000000000001e-05 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.086973+0000", 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005064631 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087049+0000", 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6790000000000002e-05 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087093+0000", 2023-10-09T04:23:31.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3358000000000001e-05 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38511 10.5 10:ae0c6440:::benchmark_data_smithi163_523108_object38510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.085594+0000", 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.183630520000001, 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.001577111, 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.314 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38511 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085594+0000", 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.315 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085594+0000", 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085592+0000", 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085616+0000", 2023-10-09T04:23:31.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3983999999999999e-05 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085629+0000", 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2568999999999999e-05 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085637+0000", 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1669999999999999e-06 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085699+0000", 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1995999999999994e-05 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.085710+0000", 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1086e-05 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.078849+0000", 2023-10-09T04:23:31.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99313923400000004 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.079018+0000", 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016883499999999999 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.079156+0000", 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000138121 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083695+0000", 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045389590000000004 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083732+0000", 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6735e-05 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087105+0000", 2023-10-09T04:23:31.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033734889999999999 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087141+0000", 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5957999999999997e-05 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087171+0000", 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9431e-05 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.321 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38513 10.5 10:ace42ba7:::benchmark_data_smithi163_523108_object38512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.087805+0000", 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.181419562000002, 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99943794900000005, 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.322 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38513 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087805+0000", 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087805+0000", 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087803+0000", 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087828+0000", 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4868999999999999e-05 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087843+0000", 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4747e-05 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087851+0000", 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9249999999999995e-06 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087910+0000", 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9104999999999997e-05 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.087925+0000", 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5112e-05 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080431+0000", 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99250614299999995 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080568+0000", 2023-10-09T04:23:31.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000136768 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080714+0000", 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014551899999999999 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083754+0000", 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030400689999999998 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083768+0000", 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4433000000000001e-05 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087179+0000", 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034107930000000001 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087215+0000", 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6137000000000003e-05 2023-10-09T04:23:31.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087242+0000", 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7506000000000001e-05 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38525 10.5 10:a41f4ea9:::benchmark_data_smithi163_523108_object38524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.099239+0000", 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.169985291000003, 2023-10-09T04:23:31.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98807896799999995, 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38525 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099239+0000", 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099239+0000", 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099237+0000", 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.331 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099267+0000", 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9666e-05 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099281+0000", 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.399e-05 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099287+0000", 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0889999999999996e-06 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099351+0000", 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4097999999999995e-05 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099363+0000", 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.19e-05 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.082037+0000", 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98267469600000001 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.082146+0000", 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010838800000000001 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.082281+0000", 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013544800000000001 2023-10-09T04:23:31.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083798+0000", 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001516611 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.085018+0000", 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012207489999999999 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087249+0000", 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022306090000000002 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087286+0000", 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6983000000000001e-05 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087318+0000", 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1655000000000001e-05 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.336 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38527 10.5 10:a29721c2:::benchmark_data_smithi163_523108_object38526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.099299+0000", 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.169924880000003, 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98819568899999999, 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.337 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38527 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099299+0000", 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099299+0000", 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099299+0000", 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099316+0000", 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7473999999999999e-05 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099321+0000", 2023-10-09T04:23:31.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4229999999999998e-06 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099343+0000", 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2291999999999999e-05 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099381+0000", 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7905999999999999e-05 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.099388+0000", 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.866e-06 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083810+0000", 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98442266300000003 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.083911+0000", 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000100716 2023-10-09T04:23:31.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.084007+0000", 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5359999999999995e-05 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.086931+0000", 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029244369999999998 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087353+0000", 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00042212400000000001 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087421+0000", 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8355000000000005e-05 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087461+0000", 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9206999999999999e-05 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087495+0000", 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4226000000000001e-05 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38535 10.5 10:a912bc89:::benchmark_data_smithi163_523108_object38534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.105038+0000", 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.164186368999999, 2023-10-09T04:23:31.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98259128299999998, 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38535 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105038+0000", 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105038+0000", 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105036+0000", 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105060+0000", 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4170999999999999e-05 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105075+0000", 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.482e-05 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105083+0000", 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0020000000000006e-06 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105143+0000", 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9376999999999997e-05 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.105155+0000", 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2958e-05 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.085042+0000", 2023-10-09T04:23:31.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97988604599999996 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.085168+0000", 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012600400000000001 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.085293+0000", 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012523100000000001 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087386+0000", 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002093513 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087393+0000", 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6000000000000003e-06 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087537+0000", 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014448 2023-10-09T04:23:31.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087589+0000", 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2126999999999997e-05 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087629+0000", 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9490999999999998e-05 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38550 10.5 10:a777dc28:::benchmark_data_smithi163_523108_object38549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.112439+0000", 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.156784735999999, 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98630311999999998, 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38550 2023-10-09T04:23:31.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112439+0000", 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112439+0000", 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112439+0000", 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112462+0000", 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3014e-05 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112470+0000", 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2749999999999995e-06 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112486+0000", 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6283e-05 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112553+0000", 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6236999999999996e-05 2023-10-09T04:23:31.355 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.112565+0000", 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2602999999999999e-05 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087678+0000", 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.975112706 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.356 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.087830+0000", 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015188499999999999 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.088009+0000", 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000179655 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.093226+0000", 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052167810000000002 2023-10-09T04:23:31.357 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.093268+0000", 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1616000000000003e-05 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098641+0000", 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053728939999999996 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.358 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098696+0000", 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5692999999999997e-05 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098742+0000", 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6057000000000003e-05 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.359 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38553 10.5 10:a650b8b5:::benchmark_data_smithi163_523108_object38552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.118497+0000", 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.150727039000003, 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.984167028, 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38553 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.360 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118497+0000", 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118497+0000", 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118496+0000", 2023-10-09T04:23:31.361 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118528+0000", 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2212999999999998e-05 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118540+0000", 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1639e-05 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.362 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118546+0000", 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0769999999999996e-06 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118612+0000", 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6536000000000001e-05 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118622+0000", 2023-10-09T04:23:31.363 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8230000000000006e-06 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.089755+0000", 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97113255899999995 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.089888+0000", 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000133864 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.364 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.090027+0000", 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000138956 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098604+0000", 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085770410000000005 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098783+0000", 2023-10-09T04:23:31.365 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017848000000000001 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102557+0000", 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037738479999999998 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102615+0000", 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7948999999999997e-05 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.366 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102664+0000", 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9317000000000002e-05 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38554 10.5 10:a39094a5:::benchmark_data_smithi163_523108_object38553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.118555+0000", 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.150668967000001, 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98430245199999999, 2023-10-09T04:23:31.367 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38554 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118555+0000", 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.368 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118555+0000", 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118555+0000", 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.369 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118572+0000", 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7350000000000002e-05 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118576+0000", 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.318e-06 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118582+0000", 2023-10-09T04:23:31.370 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8560000000000003e-06 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118643+0000", 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0946999999999998e-05 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.118649+0000", 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2750000000000001e-06 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.371 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.091554+0000", 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97290435900000005 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.091692+0000", 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000138007 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.091833+0000", 2023-10-09T04:23:31.372 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014096700000000001 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098816+0000", 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069834370000000003 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102674+0000", 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003858303 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.373 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102769+0000", 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4708000000000002e-05 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102818+0000", 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9280000000000003e-05 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.374 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102858+0000", 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9093e-05 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38569 10.5 10:ac91a68b:::benchmark_data_smithi163_523108_object38568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.129405+0000", 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.139819326000001, 2023-10-09T04:23:31.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97358195199999997, 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38569 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129405+0000", 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129405+0000", 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129404+0000", 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129425+0000", 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1279000000000001e-05 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129434+0000", 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6400000000000003e-06 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129439+0000", 2023-10-09T04:23:31.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8649999999999998e-06 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129485+0000", 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5209e-05 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.129493+0000", 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0099999999999995e-06 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.093291+0000", 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96379804800000002 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.093425+0000", 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013386800000000001 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.093566+0000", 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014097299999999999 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.098851+0000", 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0052856229999999997 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102680+0000", 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0038285250000000002 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102895+0000", 2023-10-09T04:23:31.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000215133 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102951+0000", 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6607999999999997e-05 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102987+0000", 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5222999999999998e-05 2023-10-09T04:23:31.382 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38587 10.5 10:a7fc607c:::benchmark_data_smithi163_523108_object38586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.147161+0000", 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.122063355999998, 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.9559453, 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.383 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38587 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147161+0000", 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.384 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147161+0000", 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147160+0000", 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147182+0000", 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1827999999999998e-05 2023-10-09T04:23:31.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147190+0000", 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0919999999999998e-06 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147196+0000", 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6080000000000003e-06 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147244+0000", 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.799e-05 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.147251+0000", 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1590000000000004e-06 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.094946+0000", 2023-10-09T04:23:31.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.94769497400000002 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.095069+0000", 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012312400000000001 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.095201+0000", 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000132149 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102718+0000", 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0075174509999999996 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102725+0000", 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5200000000000003e-06 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.103028+0000", 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00030260600000000002 2023-10-09T04:23:31.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.103069+0000", 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1766e-05 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.103106+0000", 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6621000000000001e-05 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.390 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38600 10.5 10:a7024472:::benchmark_data_smithi163_523108_object38599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.157426+0000", 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 46.111797783999997, 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.95105757499999999, 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38600 2023-10-09T04:23:31.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157426+0000", 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157426+0000", 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.392 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157425+0000", 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157449+0000", 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3652999999999999e-05 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157460+0000", 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0718e-05 2023-10-09T04:23:31.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157466+0000", 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.798e-06 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157520+0000", 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3971999999999998e-05 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.394 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.157530+0000", 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0039e-05 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.096722+0000", 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.93919199900000006 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.395 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.096872+0000", 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014940099999999999 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.097024+0000", 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015202400000000001 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.102730+0000", 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0057060050000000001 2023-10-09T04:23:31.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.106800+0000", 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0040701069999999999 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.108389+0000", 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001589055 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.108441+0000", 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1993000000000002e-05 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.108484+0000", 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2778999999999998e-05 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57312 10.8 10:143e0015:::benchmark_data_smithi163_523108_object57311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.930838786999999, 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.136295909, 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57312 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.399 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.400 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338384+0000", 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338414+0000", 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0681e-05 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338431+0000", 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6909999999999999e-05 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.401 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338443+0000", 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1537e-05 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342061+0000", 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036182689999999999 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342072+0000", 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1219999999999999e-05 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.464257+0000", 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.122184904 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465066+0000", 2023-10-09T04:23:31.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080893800000000004 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474628+0000", 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095617159999999996 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474681+0000", 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3440000000000003e-05 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.404 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57318 10.8 10:1fa756a5:::benchmark_data_smithi163_523108_object57317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.930765573999999, 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1361408630000001, 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.405 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57318 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338484+0000", 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6522000000000001e-05 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338493+0000", 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6600000000000001e-06 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338499+0000", 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.285e-06 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342095+0000", 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035956619999999999 2023-10-09T04:23:31.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342105+0000", 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5959999999999993e-06 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465152+0000", 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.123046999 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:23:31.409 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465958+0000", 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080678800000000001 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474519+0000", 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085601770000000004 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474599+0000", 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0765000000000003e-05 2023-10-09T04:23:31.410 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57319 10.a 10:5ee00c90:::benchmark_data_smithi163_523108_object57318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.930719405000001, 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1486770740000001, 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.411 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57319 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:23:31.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338504+0000", 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338532+0000", 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7617999999999999e-05 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338540+0000", 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0949999999999996e-06 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338545+0000", 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4469999999999997e-06 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.339505+0000", 2023-10-09T04:23:31.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00095972899999999999 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.339519+0000", 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3448000000000001e-05 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470780+0000", 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.131261257 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470931+0000", 2023-10-09T04:23:31.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015102200000000001 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.471097+0000", 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016623699999999999 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.478095+0000", 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069980989999999998 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.478878+0000", 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078291299999999999 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.481861+0000", 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029827709999999999 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486996+0000", 2023-10-09T04:23:31.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051355569999999998 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.487059+0000", 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2792999999999993e-05 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.487182+0000", 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012257699999999999 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.420 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57338 10.5 10:a63760e3:::benchmark_data_smithi163_523108_object57337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.784003961, 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0231315560000001, 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:23:31.421 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57338 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485218+0000", 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485241+0000", 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2555e-05 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485260+0000", 2023-10-09T04:23:31.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814e-05 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485267+0000", 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5599999999999996e-06 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485316+0000", 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8528e-05 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485332+0000", 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6419e-05 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.495704+0000", 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.010371613 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.495862+0000", 2023-10-09T04:23:31.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000157911 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.496027+0000", 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016513000000000001 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.505438+0000", 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0094115150000000005 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.507674+0000", 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022358320000000001 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.507710+0000", 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5670999999999998e-05 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508174+0000", 2023-10-09T04:23:31.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046372900000000002 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508224+0000", 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0794000000000002e-05 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508352+0000", 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127293 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.428 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.429 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.429 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:31.429 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:31.429 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:23:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:31.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:31.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:31.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:31.557 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:31.602 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:31.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:31.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:31.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:31.725 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 69489 69473 58.6556 0 - 0.0161823 2023-10-09T04:23:31.734 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:31.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:31.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:31.886 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:31.913 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:31.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:31.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:32.064 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:32.095 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:32.726 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 69527 69511 57.9054 1.1875 0.0439093 0.0170988 2023-10-09T04:23:32.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:23:32.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:32.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:32.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:33.012 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:33.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:32 smithi163 ceph-mon[213841]: pgmap v1113: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.4 KiB/s rd, 6.7 MiB/s wr, 111 op/s; 9592/207660 objects degraded (4.619%); 34084/207660 objects misplaced (16.413%); 8.9 MiB/s, 1 keys/s, 141 objects/s recovering 2023-10-09T04:23:33.062 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:32 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 9592/207660 objects degraded (4.619%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:33.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:32 smithi163 ceph-mon[221067]: pgmap v1113: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.4 KiB/s rd, 6.7 MiB/s wr, 111 op/s; 9592/207660 objects degraded (4.619%); 34084/207660 objects misplaced (16.413%); 8.9 MiB/s, 1 keys/s, 141 objects/s recovering 2023-10-09T04:23:33.062 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:32 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 9592/207660 objects degraded (4.619%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:33.074 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:33.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:32 smithi181 ceph-mon[196979]: pgmap v1113: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 108 active+clean, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.4 KiB/s rd, 6.7 MiB/s wr, 111 op/s; 9592/207660 objects degraded (4.619%); 34084/207660 objects misplaced (16.413%); 8.9 MiB/s, 1 keys/s, 141 objects/s recovering 2023-10-09T04:23:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:32 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 9592/207660 objects degraded (4.619%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:33.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:23:33.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:33.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:33.726 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 69839 69823 57.4002 19.5 0.109786 0.0173953 2023-10-09T04:23:33.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:33.857 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:33.921 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:33.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:34.026 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:34.071 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:34.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:34.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:34.726 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 70179 70163 56.9308 21.25 0.0081107 0.0175465 2023-10-09T04:23:34.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:34.819 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:34.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:34.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:34.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:35.036 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:35.074 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:35.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:34 smithi163 ceph-mon[213841]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2.5 KiB/s rd, 4.9 MiB/s wr, 79 op/s; 13885/208425 objects degraded (6.662%); 33622/208425 objects misplaced (16.131%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-10-09T04:23:35.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:35.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:34 smithi163 ceph-mon[221067]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2.5 KiB/s rd, 4.9 MiB/s wr, 79 op/s; 13885/208425 objects degraded (6.662%); 33622/208425 objects misplaced (16.131%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-10-09T04:23:35.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:35.095 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:35.125 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:35.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:35.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:35.261 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:35.264 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:34 smithi181 ceph-mon[196979]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2.5 KiB/s rd, 4.9 MiB/s wr, 79 op/s; 13885/208425 objects degraded (6.662%); 33622/208425 objects misplaced (16.131%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2023-10-09T04:23:35.264 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:35.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:35.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:35.412 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:35.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:35.482 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:35.565 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:23:35.609 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:35.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:35.631 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:23:35.670 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:23:35.716 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:23:35.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:35.726 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 70486 70470 56.447 19.1875 0.0142865 0.0176946 2023-10-09T04:23:35.749 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:23:35.801 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:35.813 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:23:35.852 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:23:35.896 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:23:35.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.923 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:23:35.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:23:36.024 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-09T04:23:36.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:35 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1727210185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:36.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:35 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2515075803' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:23:36.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1727210185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:36.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2515075803' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:23:36.143 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:36.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:36.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:35 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1727210185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:35 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2515075803' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T04:23:36.289 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:36.726 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 70671 70655 55.879 11.5625 0.0031711 0.0178135 2023-10-09T04:23:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: pgmap v1115: 119 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2.1 KiB/s rd, 5.7 MiB/s wr, 93 op/s; 10687/209193 objects degraded (5.109%); 36853/209193 objects misplaced (17.617%); 9.3 MiB/s, 0 keys/s, 150 objects/s recovering 2023-10-09T04:23:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2515075803' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:23:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T04:23:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:37 smithi181 ceph-mon[196979]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: pgmap v1115: 119 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2.1 KiB/s rd, 5.7 MiB/s wr, 93 op/s; 10687/209193 objects degraded (5.109%); 36853/209193 objects misplaced (17.617%); 9.3 MiB/s, 0 keys/s, 150 objects/s recovering 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2515075803' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[221067]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: pgmap v1115: 119 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2.1 KiB/s rd, 5.7 MiB/s wr, 93 op/s; 10687/209193 objects degraded (5.109%); 36853/209193 objects misplaced (17.617%); 9.3 MiB/s, 0 keys/s, 150 objects/s recovering 2023-10-09T04:23:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2515075803' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T04:23:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: osdmap e420: 8 total, 8 up, 7 in 2023-10-09T04:23:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:37 smithi163 ceph-mon[213841]: failed to encode map e420 with expected crc 2023-10-09T04:23:37.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:23:37.727631+0000 min lat: 0.00100671 max lat: 11.3806 avg lat: 0.0180325 2023-10-09T04:23:37.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:23:37.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 70954 70938 55.4011 17.6875 0.0508266 0.0180325 2023-10-09T04:23:38.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: 10.2 continuing backfill to osd.0 from (0'0,419'4395] MIN to 419'4395 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:38 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 10687/209193 objects degraded (5.109%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: 10.2 continuing backfill to osd.0 from (0'0,419'4395] MIN to 419'4395 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 10687/209193 objects degraded (5.109%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: osdmap e421: 8 total, 8 up, 7 in 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: 10.2 continuing backfill to osd.0 from (0'0,419'4395] MIN to 419'4395 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: failed to encode map e421 with expected crc 2023-10-09T04:23:38.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:38 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 10687/209193 objects degraded (5.109%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:38.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:38.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:38.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:38.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 71270 71254 54.961 19.75 0.010511 0.0181551 2023-10-09T04:23:38.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:38.799 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:38.848 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:38.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:38.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:38.982 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:39.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:39 smithi181 ceph-mon[196979]: pgmap v1118: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 9.9 MiB/s wr, 157 op/s; 6394/211134 objects degraded (3.028%); 40960/211134 objects misplaced (19.400%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:23:39.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:39.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:39 smithi163 ceph-mon[213841]: pgmap v1118: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 9.9 MiB/s wr, 157 op/s; 6394/211134 objects degraded (3.028%); 40960/211134 objects misplaced (19.400%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:23:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:39.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:39 smithi163 ceph-mon[221067]: pgmap v1118: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 9.9 MiB/s wr, 157 op/s; 6394/211134 objects degraded (3.028%); 40960/211134 objects misplaced (19.400%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:23:39.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:39.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:39.435 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:39.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:39.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:39.681 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:39.710 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:39.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 71551 71535 54.5051 17.5625 0.010149 0.0182874 2023-10-09T04:23:39.764 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:39.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:39.870 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:39.899 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:39.953 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:39.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:39.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:40.061 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:40.095 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:40.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:40.158 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:23:40.158 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:23:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:40.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:40.261 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:40.336 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:40.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:40.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:40.519 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:40.551 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:40.605 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:40.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:40.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:23:40.710 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:40.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 71898 71882 54.1097 21.6875 0.00388024 0.0184559 2023-10-09T04:23:40.774 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:40.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:40.919 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:23:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:23:40] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.43.0" 2023-10-09T04:23:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:40.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:41.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:41.035 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:23:41.051 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:41.103 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:41.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:41.210 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:41.238 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:41.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:41.293 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:41 smithi181 ceph-mon[196979]: pgmap v1119: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 14 MiB/s wr, 230 op/s; 6394/213198 objects degraded (2.999%); 40962/213198 objects misplaced (19.213%); 11 MiB/s, 174 objects/s recovering 2023-10-09T04:23:41.293 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2324950054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:41.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:41.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:41 smithi163 ceph-mon[221067]: pgmap v1119: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 14 MiB/s wr, 230 op/s; 6394/213198 objects degraded (2.999%); 40962/213198 objects misplaced (19.213%); 11 MiB/s, 174 objects/s recovering 2023-10-09T04:23:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2324950054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:41 smithi163 ceph-mon[213841]: pgmap v1119: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 14 MiB/s wr, 230 op/s; 6394/213198 objects degraded (2.999%); 40962/213198 objects misplaced (19.213%); 11 MiB/s, 174 objects/s recovering 2023-10-09T04:23:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2324950054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:41.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:41.455 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:41.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 72208 72192 53.6963 19.375 0.0205503 0.0186054 2023-10-09T04:23:41.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:23:42.064 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:42.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:42.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:42.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:23:42.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[221067]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2324950054' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[221067]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[221067]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[221067]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[213841]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2324950054' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[213841]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T04:23:42.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[213841]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.195 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:42 smithi163 ceph-mon[213841]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.196 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:42.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:42.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:42.261 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:42.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:42.380 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:42 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:23:42.380 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:42 smithi181 ceph-mon[196979]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.380 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2324950054' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:42.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:42 smithi181 ceph-mon[196979]: osdmap e422: 8 total, 8 up, 7 in 2023-10-09T04:23:42.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:42 smithi181 ceph-mon[196979]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.381 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:42 smithi181 ceph-mon[196979]: failed to encode map e422 with expected crc 2023-10-09T04:23:42.383 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:42.433 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:42.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:42.539 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:42.597 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:42.668 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:42.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:42.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 72477 72461 53.2625 16.8125 0.00606336 0.0187371 2023-10-09T04:23:42.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:42.788 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:42.811 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:42.858 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:42.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:42.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:42.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:42.992 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:43.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:43.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:43.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:43.183 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:43 smithi181 ceph-mon[196979]: pgmap v1121: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s; 6394/214527 objects degraded (2.981%); 41121/214527 objects misplaced (19.168%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T04:23:43.183 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:43 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/214527 objects degraded (2.981%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:43.185 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:43.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:43.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:43.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:43 smithi163 ceph-mon[213841]: pgmap v1121: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s; 6394/214527 objects degraded (2.981%); 41121/214527 objects misplaced (19.168%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T04:23:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:43 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/214527 objects degraded (2.981%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:43 smithi163 ceph-mon[221067]: pgmap v1121: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 19 MiB/s wr, 296 op/s; 6394/214527 objects degraded (2.981%); 41121/214527 objects misplaced (19.168%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T04:23:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:43 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/214527 objects degraded (2.981%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:43.351 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:43.383 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:43.430 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:43.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:43.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:43.569 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:43.619 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:43.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:43.630 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:23:43.630 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:23:43.630 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T04:23:43.630 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:23:43.630 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:23:43.631 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T04:23:43.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:43.727 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 72745 72729 52.838 16.75 0.00373428 0.0188904 2023-10-09T04:23:43.756 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:43.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:43.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:43.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:43.998 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:44.120 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:23:44.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:44.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:23:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:44.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:44.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:44.362 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:44.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 73053 73037 52.452 19.25 0.00947341 0.0190358 2023-10-09T04:23:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:45 smithi163 ceph-mon[213841]: pgmap v1122: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16 MiB/s wr, 258 op/s; 6394/216006 objects degraded (2.960%); 40559/216006 objects misplaced (18.777%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:23:45.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:45 smithi163 ceph-mon[221067]: pgmap v1122: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16 MiB/s wr, 258 op/s; 6394/216006 objects degraded (2.960%); 40559/216006 objects misplaced (18.777%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:23:45.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:45 smithi181 ceph-mon[196979]: pgmap v1122: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16 MiB/s wr, 258 op/s; 6394/216006 objects degraded (2.960%); 40559/216006 objects misplaced (18.777%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:23:45.637 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:45.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:45.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 73419 73403 52.1159 22.875 0.00368852 0.0191469 2023-10-09T04:23:45.742 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:45.774 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:45.789 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:23:45.807 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:45.824 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:23:45.830 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:45.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:45.860 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:23:45.890 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:23:45.932 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:23:45.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:45.965 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:23:45.967 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:46.023 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:23:46.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:46.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:46.034 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.037 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:23:46.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:46.190 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1134414684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:46.190 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1134414684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:46.224 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:23:46.287 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:23:46.287 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:23:46.287 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:23:46.287 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:23:46.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:46.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:46.509 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1134414684' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:46.591 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:46.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 73740 73724 51.7558 20.0625 0.0140018 0.0192937 2023-10-09T04:23:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:47 smithi163 ceph-mon[213841]: pgmap v1123: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 17 MiB/s wr, 273 op/s; 6394/217632 objects degraded (2.938%); 40811/217632 objects misplaced (18.752%); 11 MiB/s, 171 objects/s recovering 2023-10-09T04:23:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:47 smithi163 ceph-mon[221067]: pgmap v1123: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 17 MiB/s wr, 273 op/s; 6394/217632 objects degraded (2.938%); 40811/217632 objects misplaced (18.752%); 11 MiB/s, 171 objects/s recovering 2023-10-09T04:23:47.409 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:47.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:47.516 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:47.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:47 smithi181 ceph-mon[196979]: pgmap v1123: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 17 MiB/s wr, 273 op/s; 6394/217632 objects degraded (2.938%); 40811/217632 objects misplaced (18.752%); 11 MiB/s, 171 objects/s recovering 2023-10-09T04:23:47.558 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:47.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 90 12 74013 74001 51.3732 17.3125 0.0132666 0.019405 2023-10-09T04:23:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:48 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/217632 objects degraded (2.938%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:48 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/217632 objects degraded (2.938%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:48.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:23:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:48 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/217632 objects degraded (2.938%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:48.706 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:48.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:48.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:48.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.2636 2023-10-09T04:23:48.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 74013 2023-10-09T04:23:48.728 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T04:23:48.729 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T04:23:48.729 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 51.2478 2023-10-09T04:23:48.729 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 42.0835 2023-10-09T04:23:48.729 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 190.188 2023-10-09T04:23:48.729 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:23:48.729 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 819 2023-10-09T04:23:48.730 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 673.357 2023-10-09T04:23:48.730 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3043 2023-10-09T04:23:48.730 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 0 2023-10-09T04:23:48.730 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.0194719 2023-10-09T04:23:48.730 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.1378 2023-10-09T04:23:48.731 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 11.3806 2023-10-09T04:23:48.731 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00100671 2023-10-09T04:23:48.777 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:23:48.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:23:48.844 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:49.299 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:23:49.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:49.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:23:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:49 smithi163 ceph-mon[213841]: pgmap v1124: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 18 MiB/s wr, 295 op/s; 6394/219993 objects degraded (2.906%); 40613/219993 objects misplaced (18.461%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:23:49.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:49 smithi163 ceph-mon[221067]: pgmap v1124: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 18 MiB/s wr, 295 op/s; 6394/219993 objects degraded (2.906%); 40613/219993 objects misplaced (18.461%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:23:49.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:49.474 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:49 smithi181 ceph-mon[196979]: pgmap v1124: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 18 MiB/s wr, 295 op/s; 6394/219993 objects degraded (2.906%); 40613/219993 objects misplaced (18.461%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:23:49.533 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:49.635 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:49.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:49.744 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:49.780 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:49.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:49.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:49.951 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:49.997 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:50.178 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:50.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:50.286 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:50.315 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:50.330 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:50.330 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:50.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:50 smithi163 ceph-mon[213841]: pgmap v1125: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 288 op/s; 6394/221310 objects degraded (2.889%); 40394/221310 objects misplaced (18.252%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:23:50.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:50.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:50.331 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:50 smithi163 ceph-mon[221067]: pgmap v1125: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 288 op/s; 6394/221310 objects degraded (2.889%); 40394/221310 objects misplaced (18.252%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:23:50.370 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:50.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:50.474 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:50.505 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:50.558 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:50.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:50.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:50.713 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:23:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:50 smithi181 ceph-mon[196979]: pgmap v1125: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 288 op/s; 6394/221310 objects degraded (2.889%); 40394/221310 objects misplaced (18.252%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:23:50.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:50.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:50.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:50.935 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:50.995 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:50.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:50.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:51.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:23:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:23:50] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-10-09T04:23:51.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:23:51.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:51.130 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:51.186 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:51.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:51.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:51.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:51.318 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:51.380 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:51.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:51.404 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:23:51.444 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1287235422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:51.444 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:51.444 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1287235422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:51.444 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:51.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1287235422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:23:51.547 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:51.645 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:51.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:51.751 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:51.780 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:51.830 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:51.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:51.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:51.936 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:51.984 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:52.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:52.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:52.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:52.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:52.195 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:52.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:52.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:52.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:52.421 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:52 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:52 smithi181 ceph-mon[196979]: failed to encode map e423 with expected crc 2023-10-09T04:23:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:52 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:23:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:52 smithi181 ceph-mon[196979]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T04:23:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:52 smithi181 ceph-mon[196979]: failed to encode map e423 with expected crc 2023-10-09T04:23:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:52 smithi181 ceph-mon[196979]: pgmap v1127: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 287 op/s; 6394/221046 objects degraded (2.893%); 40356/221046 objects misplaced (18.257%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:23:52.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:52.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:52.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[221067]: failed to encode map e423 with expected crc 2023-10-09T04:23:52.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:23:52.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[221067]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T04:23:52.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[221067]: failed to encode map e423 with expected crc 2023-10-09T04:23:52.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[221067]: pgmap v1127: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 287 op/s; 6394/221046 objects degraded (2.893%); 40356/221046 objects misplaced (18.257%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:23:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:52.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:52.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:52.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[213841]: failed to encode map e423 with expected crc 2023-10-09T04:23:52.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:23:52.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[213841]: osdmap e423: 8 total, 8 up, 7 in 2023-10-09T04:23:52.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[213841]: failed to encode map e423 with expected crc 2023-10-09T04:23:52.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:52 smithi163 ceph-mon[213841]: pgmap v1127: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 287 op/s; 6394/221046 objects degraded (2.893%); 40356/221046 objects misplaced (18.257%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:23:52.650 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:52.700 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:52.732 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:72487 9.1 e328/320 9:83147396:::benchmark_data_smithi163_464650_object72486:head v 328'4340, mlcod=327'4338)", 2023-10-09T04:23:52.751 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:09.561632+0000", 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 283.185867766, 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.157487497, 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561632+0000", 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561632+0000", 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561632+0000", 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561665+0000", 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3730999999999999e-05 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561677+0000", 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2041999999999999e-05 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561685+0000", 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7209999999999994e-06 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561759+0000", 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4185999999999997e-05 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.561782+0000", 2023-10-09T04:23:52.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2688999999999999e-05 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719078+0000", 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.157295992 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719119+0000", 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1457e-05 2023-10-09T04:23:52.756 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:72491 9.f e328/310 9:f5ac6a05:::benchmark_data_smithi163_464650_object72490:head v 328'4546, mlcod=327'4544)", 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:09.566461+0000", 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 283.18103903100001, 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15776728600000001, 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.566461+0000", 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.566461+0000", 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.566460+0000", 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.566497+0000", 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6943999999999997e-05 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719033+0000", 2023-10-09T04:23:52.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.152535473 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719043+0000", 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0239e-05 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719109+0000", 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6026999999999997e-05 2023-10-09T04:23:52.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719147+0000", 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7617000000000003e-05 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.724187+0000", 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050405750000000003 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.724228+0000", 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0664000000000002e-05 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:72495 9.4 e328/310 9:2d93eae6:::benchmark_data_smithi163_464650_object72494:head v 328'4555, mlcod=327'4553)", 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:09.570039+0000", 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 283.17746018100001, 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.154477425, 2023-10-09T04:23:52.762 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.570039+0000", 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.570039+0000", 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.570039+0000", 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.570061+0000", 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2419999999999999e-05 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719016+0000", 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14895482600000001 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719033+0000", 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6750999999999999e-05 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719455+0000", 2023-10-09T04:23:52.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00042208399999999998 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.719476+0000", 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1614999999999999e-05 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.724469+0000", 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.004992944 2023-10-09T04:23:52.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:09.724517+0000", 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7435999999999998e-05 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:117618 9.3 e355/310 9:c9f8635d:::benchmark_data_smithi163_464650_object18892:head v 355'7264, mlcod=355'7262)", 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:13.201712+0000", 2023-10-09T04:23:52.767 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 219.54578788699999, 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13796134600000001, 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201712+0000", 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201712+0000", 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201711+0000", 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201717+0000", 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1279999999999996e-06 2023-10-09T04:23:52.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201727+0000", 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9240000000000002e-06 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201734+0000", 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5379999999999999e-06 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201778+0000", 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4103000000000001e-05 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201797+0000", 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9517e-05 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.339639+0000", 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13784185400000001 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.339673+0000", 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4001999999999997e-05 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(9.2 355/349 [PushOp(9:44662327:::benchmark_data_smithi163_464650_object75016:head, version: 329'4731, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:44662327:::benchmark_data_smithi163_464650_object75016:head@329'4731, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:13.201851+0000", 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 219.545648419, 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13438254899999999, 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201851+0000", 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201851+0000", 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201851+0000", 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201883+0000", 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1945e-05 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201899+0000", 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6673000000000001e-05 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201915+0000", 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5537999999999999e-05 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.335954+0000", 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13403963199999999 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.335969+0000", 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4820999999999999e-05 2023-10-09T04:23:52.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.336234+0000", 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026454899999999999 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "pg_backfill(progress 9.2 e 355/355 lb 9:4465e20e:::benchmark_data_smithi163_464650_object34109:head)", 2023-10-09T04:23:52.777 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:20:13.201931+0000", 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 219.54556827900001, 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.13438591799999999, 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201931+0000", 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201931+0000", 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201931+0000", 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201938+0000", 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9659999999999999e-06 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.201945+0000", 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.306e-06 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.335942+0000", 2023-10-09T04:23:52.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.133996436 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.336273+0000", 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00033147999999999999 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.336289+0000", 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6076e-05 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:20:13.336317+0000", 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7974e-05 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:154326 9.4 e362/310 9:2b21f9be:::benchmark_data_smithi163_464650_object55600:head v 362'9668, mlcod=362'9666)", 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:00.562066+0000", 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.185433598, 2023-10-09T04:23:52.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14992362100000001, 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562066+0000", 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562066+0000", 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562065+0000", 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562073+0000", 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6369999999999999e-06 2023-10-09T04:23:52.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562084+0000", 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1361e-05 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562099+0000", 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4782e-05 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562164+0000", 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5285999999999998e-05 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.562178+0000", 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4e-05 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.711944+0000", 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14976572899999999 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.711990+0000", 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5686000000000003e-05 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:154331 9.3 e362/310 9:c5746161:::benchmark_data_smithi163_464650_object55605:head v 362'9586, mlcod=362'9584)", 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:00.565184+0000", 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.182315598, 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15311612899999999, 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.565184+0000", 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.565184+0000", 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.565183+0000", 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.565189+0000", 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1240000000000004e-06 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.711937+0000", 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14674859700000001 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.711953+0000", 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5875999999999998e-05 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.712263+0000", 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00030981200000000002 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.712281+0000", 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8338999999999998e-05 2023-10-09T04:23:52.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.718262+0000", 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0059813410000000003 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.718300+0000", 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7696e-05 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.792 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:154337 9.2 e362/359 9:48c47876:::benchmark_data_smithi163_464650_object55611:head v 362'9686, mlcod=362'9684)", 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:00.571249+0000", 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 172.17625066900001, 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.14413437900000001, 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.571249+0000", 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.571249+0000", 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.571248+0000", 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.571257+0000", 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.0559999999999994e-06 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.712075+0000", 2023-10-09T04:23:52.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.140818317 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.712083+0000", 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1029999999999996e-06 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.712184+0000", 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101718 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.712209+0000", 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5018e-05 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.715341+0000", 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031312990000000001 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:00.715383+0000", 2023-10-09T04:23:52.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2772999999999999e-05 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:154811 9.4 e362/310 9:2f5d5e5c:::benchmark_data_smithi163_464650_object56085:head v 362'9692, mlcod=362'9690)", 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:01.189319+0000", 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 171.558180693, 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.123605728, 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.798 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189319+0000", 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189319+0000", 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189318+0000", 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189340+0000", 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1543e-05 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189352+0000", 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1717e-05 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189361+0000", 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1530000000000003e-06 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189401+0000", 2023-10-09T04:23:52.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0708999999999999e-05 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.189420+0000", 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8898000000000001e-05 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.312893+0000", 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12347314700000001 2023-10-09T04:23:52.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.312925+0000", 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1257999999999998e-05 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:154813 9.4 e362/310 9:2d91e47b:::benchmark_data_smithi163_464650_object56087:head v 362'9693, mlcod=362'9690)", 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:01.194140+0000", 2023-10-09T04:23:52.803 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 171.553359852, 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.11882628000000001, 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.194140+0000", 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.194140+0000", 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.194139+0000", 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.194146+0000", 2023-10-09T04:23:52.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.878e-06 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.298239+0000", 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.104092744 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.298252+0000", 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3553999999999999e-05 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.307415+0000", 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091630889999999993 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.307436+0000", 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0418e-05 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.312950+0000", 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055140349999999996 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:01.312966+0000", 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6209e-05 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "pg_backfill(progress 9.7 e 362/362 lb 9:ebb1bc94:::benchmark_data_smithi163_464650_object85967:head)", 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:03.333359+0000", 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 169.41414062800001, 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12963802699999999, 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333359+0000", 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333359+0000", 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333359+0000", 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333364+0000", 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2739999999999998e-06 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333368+0000", 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.895e-06 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333372+0000", 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.036e-06 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333594+0000", 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000222017 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.333605+0000", 2023-10-09T04:23:52.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1382e-05 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:03.462997+0000", 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.12939166699999999 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:165613 9.8 e364/348 9:1b0d2b01:::benchmark_data_smithi163_464650_object66887:head v 364'10355, mlcod=364'10353)", 2023-10-09T04:23:52.813 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:14.719938+0000", 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 158.02756200499999, 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15612555, 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719938+0000", 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719938+0000", 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719937+0000", 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719943+0000", 2023-10-09T04:23:52.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5370000000000003e-06 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719955+0000", 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1518999999999999e-05 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719962+0000", 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7950000000000002e-06 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720001+0000", 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9425000000000003e-05 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720022+0000", 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0568999999999999e-05 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.876030+0000", 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15600896 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.876063+0000", 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.2709000000000001e-05 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(9.8 364/348 [PushOp(9:17428940:::benchmark_data_smithi163_464650_object68818:head, version: 325'4299, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:17428940:::benchmark_data_smithi163_464650_object68818:head@325'4299, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:14.719977+0000", 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 158.02752305000001, 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15639677599999999, 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719977+0000", 2023-10-09T04:23:52.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719977+0000", 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.719976+0000", 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720004+0000", 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8113999999999999e-05 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720015+0000", 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0210999999999999e-05 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720024+0000", 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2779999999999999e-06 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.876110+0000", 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.156086207 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.822 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.876122+0000", 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1796e-05 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.876373+0000", 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000251494 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.823 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "pg_backfill(progress 9.8 e 364/364 lb 9:173d91ef:::benchmark_data_smithi163_464650_object97544:head)", 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:14.720031+0000", 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 158.02746880500001, 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.155995525, 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720031+0000", 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720031+0000", 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720031+0000", 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720037+0000", 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1840000000000004e-06 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720043+0000", 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720048+0000", 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0320000000000003e-06 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875958+0000", 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15590979499999999 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875975+0000", 2023-10-09T04:23:52.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7438000000000001e-05 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.876026+0000", 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0933999999999997e-05 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:165611 9.f e364/310 9:fe941a27:::benchmark_data_smithi163_464650_object66885:head v 364'10431, mlcod=364'10429)", 2023-10-09T04:23:52.828 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:14.720120+0000", 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 158.02737948800001, 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.17731719900000001, 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720120+0000", 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720120+0000", 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720120+0000", 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720124+0000", 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6949999999999999e-06 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720132+0000", 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3000000000000004e-06 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.720136+0000", 2023-10-09T04:23:52.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1559999999999997e-06 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875872+0000", 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15573611800000001 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875907+0000", 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4693999999999997e-05 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.897404+0000", 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.021497321999999999 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.897437+0000", 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3423000000000002e-05 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.833 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:165617 9.1 e364/320 9:829ddabf:::benchmark_data_smithi163_464650_object66891:head v 364'9960, mlcod=364'9957)", 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:14.721520+0000", 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 158.02597956700001, 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16268744500000001, 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.721520+0000", 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.721520+0000", 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.721519+0000", 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.721526+0000", 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8909999999999998e-06 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875874+0000", 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15434720800000001 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875884+0000", 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0788000000000001e-05 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875932+0000", 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7469999999999998e-05 2023-10-09T04:23:52.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875953+0000", 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1382999999999998e-05 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.884173+0000", 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082198480000000001 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.884208+0000", 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4508999999999998e-05 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.55504.0:165619 9.4 e364/310 9:2b3949d8:::benchmark_data_smithi163_464650_object66893:head v 364'10390, mlcod=364'10387)", 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:14.723499+0000", 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 158.02400085100001, 2023-10-09T04:23:52.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15798066499999999, 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.723499+0000", 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.840 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.723499+0000", 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.723498+0000", 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.723506+0000", 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9689999999999994e-06 2023-10-09T04:23:52.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875869+0000", 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15236291299999999 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875879+0000", 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0648e-05 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875922+0000", 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2515000000000002e-05 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.875946+0000", 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3964e-05 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:23:52.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.881444+0000", 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054977849999999998 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:14.881479+0000", 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5744000000000002e-05 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.844 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(9.8 364/348 [PushOp(9:1809fb84:::benchmark_data_smithi163_464650_object69052:head, version: 326'4324, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:1809fb84:::benchmark_data_smithi163_464650_object69052:head@326'4324, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:16.057215+0000", 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 156.69028505700001, 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18691282100000001, 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.057215+0000", 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.057215+0000", 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.057214+0000", 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.057234+0000", 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.967e-05 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.243761+0000", 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.18652756700000001 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.243783+0000", 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1719999999999999e-05 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.243851+0000", 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8054999999999998e-05 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.243861+0000", 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5489999999999995e-06 2023-10-09T04:23:52.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:16.244127+0000", 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026676599999999999 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(10.d 400/390 [PushOp(10:bdbc828a:::benchmark_data_smithi163_523108_object15376:head, version: 388'921, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:bdbc828a:::benchmark_data_smithi163_523108_object15376:head@388'921, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:51.611207+0000", 2023-10-09T04:23:52.849 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.136293025000001, 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.15263228400000001, 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611207+0000", 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:23:52.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611207+0000", 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611205+0000", 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611302+0000", 2023-10-09T04:23:52.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7255000000000003e-05 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611355+0000", 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2709e-05 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611383+0000", 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7766000000000001e-05 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611462+0000", 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8897e-05 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.611469+0000", 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8090000000000004e-06 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:51.763839+0000", 2023-10-09T04:23:52.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.152370385 2023-10-09T04:23:52.854 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.854 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.854 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.854 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:23:52.854 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:23:52.854 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:23:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:52.855 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:52.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:52.856 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:23:52.856 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:23:52.856 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:23:52.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:53.026 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:53.529 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:53.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:53.634 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:53.716 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:53 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/221046 objects degraded (2.893%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:53.721 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:53.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:53.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:53.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:53 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/221046 objects degraded (2.893%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:53.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:53 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/221046 objects degraded (2.893%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:53.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:53.906 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:53.954 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:53.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:53.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:54.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:54.112 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:54.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:54 smithi163 ceph-mon[213841]: pgmap v1128: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 282 op/s; 6394/219753 objects degraded (2.910%); 39215/219753 objects misplaced (17.845%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:23:54.529 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:23:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:54 smithi181 ceph-mon[196979]: pgmap v1128: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 282 op/s; 6394/219753 objects degraded (2.910%); 39215/219753 objects misplaced (17.845%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:23:54.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:54 smithi163 ceph-mon[221067]: pgmap v1128: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 13 MiB/s wr, 282 op/s; 6394/219753 objects degraded (2.910%); 39215/219753 objects misplaced (17.845%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:23:54.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:23:54.938 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:54.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:54.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:55.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:23:55.089 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:55.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:23:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:55.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:23:55.254 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:55.332 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:55.377 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:23:55.472 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:55.521 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:55.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:55.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:55.635 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:23:55.670 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:23:55.689 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:55.708 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:23:55.747 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:23:55.762 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:55.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:55.777 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:23:55.802 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:23:55.837 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:23:55.866 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:23:55.868 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.869 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:23:55.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:55.901 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1188099788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:55.904 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:55.967 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1188099788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:55.967 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1188099788' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:23:56.369 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:56.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:56.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:23:56.517 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:57.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:56 smithi181 ceph-mon[196979]: pgmap v1129: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 9.3 MiB/s wr, 290 op/s; 6394/217881 objects degraded (2.935%); 38886/217881 objects misplaced (17.847%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:23:57.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:56 smithi163 ceph-mon[221067]: pgmap v1129: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 9.3 MiB/s wr, 290 op/s; 6394/217881 objects degraded (2.935%); 38886/217881 objects misplaced (17.847%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:23:57.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:56 smithi163 ceph-mon[213841]: pgmap v1129: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 9.3 MiB/s wr, 290 op/s; 6394/217881 objects degraded (2.935%); 38886/217881 objects misplaced (17.847%); 11 MiB/s, 183 objects/s recovering 2023-10-09T04:23:57.696 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:23:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:57.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:23:57.732 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:57 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/217881 objects degraded (2.935%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:57.805 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:23:57.854 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:57.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:57 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/217881 objects degraded (2.935%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:57.976 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:57 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/217881 objects degraded (2.935%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:57.991 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:23:57.992 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:23:58.016 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:23:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:58.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:23:58.121 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:58.167 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:58.239 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:58.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:58.347 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:23:58.377 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:58.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:23:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:58.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:23:58.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:58.575 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:58.628 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:58.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:58.686 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:23:58.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:23:58.762 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:58.821 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:23:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:58.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:23:58.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:23:58.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:58 smithi181 ceph-mon[196979]: pgmap v1130: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 4.4 MiB/s wr, 292 op/s; 6394/215469 objects degraded (2.967%); 38085/215469 objects misplaced (17.675%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:23:58.952 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/780332562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:58.955 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:59.090 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:58 smithi163 ceph-mon[213841]: pgmap v1130: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 4.4 MiB/s wr, 292 op/s; 6394/215469 objects degraded (2.967%); 38085/215469 objects misplaced (17.675%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:23:59.090 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/780332562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:59.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:58 smithi163 ceph-mon[221067]: pgmap v1130: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 4.4 MiB/s wr, 292 op/s; 6394/215469 objects degraded (2.967%); 38085/215469 objects misplaced (17.675%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:23:59.091 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/780332562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:23:59.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:23:59.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:59.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:23:59.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:23:59.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:59.349 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:59.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:59.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:59.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:23:59.511 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:23:59.567 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:23:59.576 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:23:59.576 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:23:59.576 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-10-09T04:23:59.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:23:59.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:23:59.766 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:23:59.911 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:23:59.917 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:59.917 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[221067]: failed to encode map e424 with expected crc 2023-10-09T04:23:59.917 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[221067]: failed to encode map e424 with expected crc 2023-10-09T04:23:59.917 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/780332562' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[221067]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[213841]: failed to encode map e424 with expected crc 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[213841]: failed to encode map e424 with expected crc 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/780332562' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[213841]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T04:23:59.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:23:59 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:23:59.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:59 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:59 smithi181 ceph-mon[196979]: failed to encode map e424 with expected crc 2023-10-09T04:24:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:59 smithi181 ceph-mon[196979]: failed to encode map e424 with expected crc 2023-10-09T04:24:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/780332562' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:59 smithi181 ceph-mon[196979]: osdmap e424: 8 total, 8 up, 7 in 2023-10-09T04:24:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:23:59 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:00.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:00.102 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:00.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:00.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:00.247 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:00.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:00.343 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:00.404 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:00.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:00.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:00.702 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:00.762 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:00.771 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:00 smithi163 ceph-mon[213841]: pgmap v1132: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 52 KiB/s wr, 309 op/s; 6394/213132 objects degraded (3.000%); 37268/213132 objects misplaced (17.486%); 14 MiB/s, 225 objects/s recovering 2023-10-09T04:24:00.771 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:24:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:24:00] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-10-09T04:24:00.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:00 smithi163 ceph-mon[221067]: pgmap v1132: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 52 KiB/s wr, 309 op/s; 6394/213132 objects degraded (3.000%); 37268/213132 objects misplaced (17.486%); 14 MiB/s, 225 objects/s recovering 2023-10-09T04:24:01.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:00 smithi181 ceph-mon[196979]: pgmap v1132: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 52 KiB/s wr, 309 op/s; 6394/213132 objects degraded (3.000%); 37268/213132 objects misplaced (17.486%); 14 MiB/s, 225 objects/s recovering 2023-10-09T04:24:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:01.130 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:01.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:01.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:01.265 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:01.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:01.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:01.771 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:01.801 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:01.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:24:02.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:02.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:02.511 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:24:02.511 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:24:02.573 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:02.626 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:02.785 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:02.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:02.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:02.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:02.932 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:03.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:02 smithi163 ceph-mon[213841]: pgmap v1133: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 45 KiB/s wr, 311 op/s; 6394/211743 objects degraded (3.020%); 37084/211743 objects misplaced (17.514%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:24:03.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/211743 objects degraded (3.020%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:03.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:02 smithi163 ceph-mon[221067]: pgmap v1133: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 45 KiB/s wr, 311 op/s; 6394/211743 objects degraded (3.020%); 37084/211743 objects misplaced (17.514%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:24:03.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/211743 objects degraded (3.020%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:03.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:02 smithi181 ceph-mon[196979]: pgmap v1133: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 45 KiB/s wr, 311 op/s; 6394/211743 objects degraded (3.020%); 37084/211743 objects misplaced (17.514%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:24:03.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/211743 objects degraded (3.020%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:03.287 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:03.828 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:24:03.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:03.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:03.847 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:24:03.904 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:03.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:03.946 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:03.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:24:03.984 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:04.030 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:04.043 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:04.086 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:04.121 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:04.153 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:04.179 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.181 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:24:04.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1782250747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:04.211 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1782250747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:04.213 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:04.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:04.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1782250747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:04.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:04.373 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:04.646 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:05.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:05 smithi181 ceph-mon[196979]: from='client.36261 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:24:05.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:05 smithi181 ceph-mon[196979]: pgmap v1134: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 6394/210378 objects degraded (3.039%); 35815/210378 objects misplaced (17.024%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:24:05.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:05.214 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[213841]: from='client.36261 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:24:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[213841]: pgmap v1134: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 6394/210378 objects degraded (3.039%); 35815/210378 objects misplaced (17.024%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:24:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[221067]: from='client.36261 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:24:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[221067]: pgmap v1134: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 312 op/s; 6394/210378 objects degraded (3.039%); 35815/210378 objects misplaced (17.024%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:24:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:05.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:05.664 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:05.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:05.717 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1133,"stamp":"2023-10-09T04:24:01.930674+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4624595715,"num_objects":70581,"num_object_clones":0,"num_object_copies":211743,"num_objects_missing_on_primary":3833,"num_objects_missing":3833,"num_objects_degraded":6394,"num_objects_misplaced":37084,"num_objects_unfound":0,"num_objects_dirty":70581,"num_whiteouts":0,"num_read":90,"num_read_kb":168,"num_write":77576,"num_write_kb":4737302,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17346,"num_bytes_recovered":1133057711,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82518,"ondisk_log_size":82518,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17058960,"kb_used_data":12130456,"kb_used_omap":126,"kb_used_meta":4928257,"kb_avail":639157104,"statfs":{"total":671965249536,"available":654496874496,"internally_reserved":0,"allocated":12421586944,"data_stored":12410205134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":5046535803},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-203227136,"num_objects":-3101,"num_object_clones":0,"num_object_copies":-9303,"num_objects_missing_on_primary":-27,"num_objects_missing":-27,"num_objects_degraded":0,"num_objects_misplaced":-3272,"num_objects_unfound":0,"num_objects_dirty":-3101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3115,"num_write_kb":448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1934,"num_bytes_recovered":126746624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004465"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":548,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358218+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:23:51.358218+0000","last_peered":"2023-10-09T04:23:51.358218+0000","last_clean":"2023-10-09T04:23:51.358218+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:23:51.358218+0000","last_undegraded":"2023-10-09T04:23:51.358218+0000","last_fullsized":"2023-10-09T04:23:51.358218+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":207,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380219+0000","last_change":"2023-10-09T04:23:22.367540+0000","last_active":"2023-10-09T04:23:51.380219+0000","last_peered":"2023-10-09T04:23:51.380219+0000","last_clean":"2023-10-09T04:23:51.380219+0000","last_became_active":"2023-10-09T04:23:22.367185+0000","last_became_peered":"2023-10-09T04:23:22.367185+0000","last_unstale":"2023-10-09T04:23:51.380219+0000","last_undegraded":"2023-10-09T04:23:51.380219+0000","last_fullsized":"2023-10-09T04:23:51.380219+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:36.747472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":301,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.381794+0000","last_change":"2023-10-09T04:23:22.368166+0000","last_active":"2023-10-09T04:23:51.381794+0000","last_peered":"2023-10-09T04:23:51.381794+0000","last_clean":"2023-10-09T04:23:51.381794+0000","last_became_active":"2023-10-09T04:23:22.367876+0000","last_became_peered":"2023-10-09T04:23:22.367876+0000","last_unstale":"2023-10-09T04:23:51.381794+0000","last_undegraded":"2023-10-09T04:23:51.381794+0000","last_fullsized":"2023-10-09T04:23:51.381794+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:39.268330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":601,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359242+0000","last_change":"2023-10-09T04:23:22.379164+0000","last_active":"2023-10-09T04:23:51.359242+0000","last_peered":"2023-10-09T04:23:51.359242+0000","last_clean":"2023-10-09T04:23:51.359242+0000","last_became_active":"2023-10-09T04:23:22.378812+0000","last_became_peered":"2023-10-09T04:23:22.378812+0000","last_unstale":"2023-10-09T04:23:51.359242+0000","last_undegraded":"2023-10-09T04:23:51.359242+0000","last_fullsized":"2023-10-09T04:23:51.359242+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:58.326150+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":421,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.375962+0000","last_change":"2023-10-09T04:23:37.010935+0000","last_active":"2023-10-09T04:23:51.375962+0000","last_peered":"2023-10-09T04:23:51.375962+0000","last_clean":"2023-10-09T04:23:51.375962+0000","last_became_active":"2023-10-09T04:23:37.010628+0000","last_became_peered":"2023-10-09T04:23:37.010628+0000","last_unstale":"2023-10-09T04:23:51.375962+0000","last_undegraded":"2023-10-09T04:23:51.375962+0000","last_fullsized":"2023-10-09T04:23:51.375962+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:28.587185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372718+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:23:51.372718+0000","last_peered":"2023-10-09T04:23:51.372718+0000","last_clean":"2023-10-09T04:23:51.372718+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:23:51.372718+0000","last_undegraded":"2023-10-09T04:23:51.372718+0000","last_fullsized":"2023-10-09T04:23:51.372718+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":427,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358547+0000","last_change":"2023-10-09T04:23:25.382713+0000","last_active":"2023-10-09T04:23:51.358547+0000","last_peered":"2023-10-09T04:23:51.358547+0000","last_clean":"2023-10-09T04:23:51.358547+0000","last_became_active":"2023-10-09T04:23:25.382298+0000","last_became_peered":"2023-10-09T04:23:25.382298+0000","last_unstale":"2023-10-09T04:23:51.358547+0000","last_undegraded":"2023-10-09T04:23:51.358547+0000","last_fullsized":"2023-10-09T04:23:51.358547+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.371707+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:23:51.371707+0000","last_peered":"2023-10-09T04:23:51.371707+0000","last_clean":"2023-10-09T04:23:51.371707+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:23:51.371707+0000","last_undegraded":"2023-10-09T04:23:51.371707+0000","last_fullsized":"2023-10-09T04:23:51.371707+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359591+0000","last_change":"2023-10-09T04:22:13.634165+0000","last_active":"2023-10-09T04:23:51.359591+0000","last_peered":"2023-10-09T04:23:51.359591+0000","last_clean":"2023-10-09T04:23:51.359591+0000","last_became_active":"2023-10-09T04:22:13.632882+0000","last_became_peered":"2023-10-09T04:22:13.632882+0000","last_unstale":"2023-10-09T04:23:51.359591+0000","last_undegraded":"2023-10-09T04:23:51.359591+0000","last_fullsized":"2023-10-09T04:23:51.359591+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:28.354644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":823,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352583+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:23:51.352583+0000","last_peered":"2023-10-09T04:23:51.352583+0000","last_clean":"2023-10-09T04:23:51.352583+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:23:51.352583+0000","last_undegraded":"2023-10-09T04:23:51.352583+0000","last_fullsized":"2023-10-09T04:23:51.352583+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.374045+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:23:51.374045+0000","last_peered":"2023-10-09T04:23:51.374045+0000","last_clean":"2023-10-09T04:23:51.374045+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:23:51.374045+0000","last_undegraded":"2023-10-09T04:23:51.374045+0000","last_fullsized":"2023-10-09T04:23:51.374045+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":144,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.375846+0000","last_change":"2023-10-09T04:22:13.630073+0000","last_active":"2023-10-09T04:23:51.375846+0000","last_peered":"2023-10-09T04:23:51.375846+0000","last_clean":"2023-10-09T04:23:51.375846+0000","last_became_active":"2023-10-09T04:22:13.629719+0000","last_became_peered":"2023-10-09T04:22:13.629719+0000","last_unstale":"2023-10-09T04:23:51.375846+0000","last_undegraded":"2023-10-09T04:23:51.375846+0000","last_fullsized":"2023-10-09T04:23:51.375846+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:03.686524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":568,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380017+0000","last_change":"2023-10-09T04:23:25.384465+0000","last_active":"2023-10-09T04:23:51.380017+0000","last_peered":"2023-10-09T04:23:51.380017+0000","last_clean":"2023-10-09T04:23:51.380017+0000","last_became_active":"2023-10-09T04:23:25.384092+0000","last_became_peered":"2023-10-09T04:23:25.384092+0000","last_unstale":"2023-10-09T04:23:51.380017+0000","last_undegraded":"2023-10-09T04:23:51.380017+0000","last_fullsized":"2023-10-09T04:23:51.380017+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:07:06.280949+0000","scrub_duration":0.0013925070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":383,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359829+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:23:51.359829+0000","last_peered":"2023-10-09T04:23:51.359829+0000","last_clean":"2023-10-09T04:23:51.359829+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:23:51.359829+0000","last_undegraded":"2023-10-09T04:23:51.359829+0000","last_fullsized":"2023-10-09T04:23:51.359829+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.376025+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:23:51.376025+0000","last_peered":"2023-10-09T04:23:51.376025+0000","last_clean":"2023-10-09T04:23:51.376025+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:23:51.376025+0000","last_undegraded":"2023-10-09T04:23:51.376025+0000","last_fullsized":"2023-10-09T04:23:51.376025+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":628,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359515+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:23:51.359515+0000","last_peered":"2023-10-09T04:23:51.359515+0000","last_clean":"2023-10-09T04:23:51.359515+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:23:51.359515+0000","last_undegraded":"2023-10-09T04:23:51.359515+0000","last_fullsized":"2023-10-09T04:23:51.359515+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":482,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352918+0000","last_change":"2023-10-09T04:23:22.464995+0000","last_active":"2023-10-09T04:23:51.352918+0000","last_peered":"2023-10-09T04:23:51.352918+0000","last_clean":"2023-10-09T04:23:51.352918+0000","last_became_active":"2023-10-09T04:23:22.377288+0000","last_became_peered":"2023-10-09T04:23:22.377288+0000","last_unstale":"2023-10-09T04:23:51.352918+0000","last_undegraded":"2023-10-09T04:23:51.352918+0000","last_fullsized":"2023-10-09T04:23:51.352918+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:06.312840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.035722+0000","last_change":"2023-10-09T04:22:13.629778+0000","last_active":"2023-10-09T04:23:41.035722+0000","last_peered":"2023-10-09T04:23:41.035722+0000","last_clean":"2023-10-09T04:23:41.035722+0000","last_became_active":"2023-10-09T04:22:13.629473+0000","last_became_peered":"2023-10-09T04:22:13.629473+0000","last_unstale":"2023-10-09T04:23:41.035722+0000","last_undegraded":"2023-10-09T04:23:41.035722+0000","last_fullsized":"2023-10-09T04:23:41.035722+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:42.753957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":600,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.056023+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:23:41.056023+0000","last_peered":"2023-10-09T04:23:41.056023+0000","last_clean":"2023-10-09T04:23:41.056023+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:23:41.056023+0000","last_undegraded":"2023-10-09T04:23:41.056023+0000","last_fullsized":"2023-10-09T04:23:41.056023+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":111,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353597+0000","last_change":"2023-10-09T04:23:22.462142+0000","last_active":"2023-10-09T04:23:51.353597+0000","last_peered":"2023-10-09T04:23:51.353597+0000","last_clean":"2023-10-09T04:23:51.353597+0000","last_became_active":"2023-10-09T04:23:22.376278+0000","last_became_peered":"2023-10-09T04:23:22.376278+0000","last_unstale":"2023-10-09T04:23:51.353597+0000","last_undegraded":"2023-10-09T04:23:51.353597+0000","last_fullsized":"2023-10-09T04:23:51.353597+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:36.530321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":19,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358542+0000","last_change":"2023-10-09T04:23:22.376371+0000","last_active":"2023-10-09T04:23:51.358542+0000","last_peered":"2023-10-09T04:23:51.358542+0000","last_clean":"2023-10-09T04:23:51.358542+0000","last_became_active":"2023-10-09T04:23:22.376005+0000","last_became_peered":"2023-10-09T04:23:22.376005+0000","last_unstale":"2023-10-09T04:23:51.358542+0000","last_undegraded":"2023-10-09T04:23:51.358542+0000","last_fullsized":"2023-10-09T04:23:51.358542+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:49.927677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":442,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372010+0000","last_change":"2023-10-09T04:23:08.629976+0000","last_active":"2023-10-09T04:23:51.372010+0000","last_peered":"2023-10-09T04:23:51.372010+0000","last_clean":"2023-10-09T04:23:51.372010+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:23:51.372010+0000","last_undegraded":"2023-10-09T04:23:51.372010+0000","last_fullsized":"2023-10-09T04:23:51.372010+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":433,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353929+0000","last_change":"2023-10-09T04:23:22.455486+0000","last_active":"2023-10-09T04:23:51.353929+0000","last_peered":"2023-10-09T04:23:51.353929+0000","last_clean":"2023-10-09T04:23:51.353929+0000","last_became_active":"2023-10-09T04:23:22.373684+0000","last_became_peered":"2023-10-09T04:23:22.373684+0000","last_unstale":"2023-10-09T04:23:51.353929+0000","last_undegraded":"2023-10-09T04:23:51.353929+0000","last_fullsized":"2023-10-09T04:23:51.353929+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:38.278137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353412+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:23:51.353412+0000","last_peered":"2023-10-09T04:23:51.353412+0000","last_clean":"2023-10-09T04:23:51.353412+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:23:51.353412+0000","last_undegraded":"2023-10-09T04:23:51.353412+0000","last_fullsized":"2023-10-09T04:23:51.353412+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":588,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358155+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:23:51.358155+0000","last_peered":"2023-10-09T04:23:51.358155+0000","last_clean":"2023-10-09T04:23:51.358155+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:23:51.358155+0000","last_undegraded":"2023-10-09T04:23:51.358155+0000","last_fullsized":"2023-10-09T04:23:51.358155+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372531+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:23:51.372531+0000","last_peered":"2023-10-09T04:23:51.372531+0000","last_clean":"2023-10-09T04:23:51.372531+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:23:51.372531+0000","last_undegraded":"2023-10-09T04:23:51.372531+0000","last_fullsized":"2023-10-09T04:23:51.372531+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":402,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359617+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:23:51.359617+0000","last_peered":"2023-10-09T04:23:51.359617+0000","last_clean":"2023-10-09T04:23:51.359617+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:23:51.359617+0000","last_undegraded":"2023-10-09T04:23:51.359617+0000","last_fullsized":"2023-10-09T04:23:51.359617+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":550,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.376430+0000","last_change":"2023-10-09T04:23:25.386262+0000","last_active":"2023-10-09T04:23:51.376430+0000","last_peered":"2023-10-09T04:23:51.376430+0000","last_clean":"2023-10-09T04:23:51.376430+0000","last_became_active":"2023-10-09T04:23:25.385887+0000","last_became_peered":"2023-10-09T04:23:25.385887+0000","last_unstale":"2023-10-09T04:23:51.376430+0000","last_undegraded":"2023-10-09T04:23:51.376430+0000","last_fullsized":"2023-10-09T04:23:51.376430+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:48.918844+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.371728+0000","last_change":"2023-10-09T04:23:22.379011+0000","last_active":"2023-10-09T04:23:51.371728+0000","last_peered":"2023-10-09T04:23:51.371728+0000","last_clean":"2023-10-09T04:23:51.371728+0000","last_became_active":"2023-10-09T04:23:22.378391+0000","last_became_peered":"2023-10-09T04:23:22.378391+0000","last_unstale":"2023-10-09T04:23:51.371728+0000","last_undegraded":"2023-10-09T04:23:51.371728+0000","last_fullsized":"2023-10-09T04:23:51.371728+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:22.951460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372648+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:23:51.372648+0000","last_peered":"2023-10-09T04:23:51.372648+0000","last_clean":"2023-10-09T04:23:51.372648+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:23:51.372648+0000","last_undegraded":"2023-10-09T04:23:51.372648+0000","last_fullsized":"2023-10-09T04:23:51.372648+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":588,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358920+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:23:51.358920+0000","last_peered":"2023-10-09T04:23:51.358920+0000","last_clean":"2023-10-09T04:23:51.358920+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:23:51.358920+0000","last_undegraded":"2023-10-09T04:23:51.358920+0000","last_fullsized":"2023-10-09T04:23:51.358920+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":37,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.371844+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:23:51.371844+0000","last_peered":"2023-10-09T04:23:51.371844+0000","last_clean":"2023-10-09T04:23:51.371844+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:23:51.371844+0000","last_undegraded":"2023-10-09T04:23:51.371844+0000","last_fullsized":"2023-10-09T04:23:51.371844+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.378775+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:23:51.378775+0000","last_peered":"2023-10-09T04:23:51.378775+0000","last_clean":"2023-10-09T04:23:51.378775+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:23:51.378775+0000","last_undegraded":"2023-10-09T04:23:51.378775+0000","last_fullsized":"2023-10-09T04:23:51.378775+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"424'4728","reported_seq":7099,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:00.667742+0000","last_change":"2023-10-09T04:23:23.420174+0000","last_active":"2023-10-09T04:24:00.667742+0000","last_peered":"2023-10-09T04:24:00.667742+0000","last_clean":"2023-10-09T04:23:05.250360+0000","last_became_active":"2023-10-09T04:23:23.407465+0000","last_became_peered":"2023-10-09T04:23:23.407465+0000","last_unstale":"2023-10-09T04:24:00.667742+0000","last_undegraded":"2023-10-09T04:24:00.667742+0000","last_fullsized":"2023-10-09T04:24:00.667742+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4728,"log_dups_size":0,"ondisk_log_size":4728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:50.487410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804800,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4300,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4728,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1135,"num_bytes_recovered":74383360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,1,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":4300}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":846,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352670+0000","last_change":"2023-10-09T04:23:25.383275+0000","last_active":"2023-10-09T04:23:51.352670+0000","last_peered":"2023-10-09T04:23:51.352670+0000","last_clean":"2023-10-09T04:23:51.352670+0000","last_became_active":"2023-10-09T04:23:25.382932+0000","last_became_peered":"2023-10-09T04:23:25.382932+0000","last_unstale":"2023-10-09T04:23:51.352670+0000","last_undegraded":"2023-10-09T04:23:51.352670+0000","last_fullsized":"2023-10-09T04:23:51.352670+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.086212+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":454,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353653+0000","last_change":"2023-10-09T04:23:22.469398+0000","last_active":"2023-10-09T04:23:51.353653+0000","last_peered":"2023-10-09T04:23:51.353653+0000","last_clean":"2023-10-09T04:23:51.353653+0000","last_became_active":"2023-10-09T04:23:22.382490+0000","last_became_peered":"2023-10-09T04:23:22.382490+0000","last_unstale":"2023-10-09T04:23:51.353653+0000","last_undegraded":"2023-10-09T04:23:51.353653+0000","last_fullsized":"2023-10-09T04:23:51.353653+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:20.439036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":385,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358293+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:23:51.358293+0000","last_peered":"2023-10-09T04:23:51.358293+0000","last_clean":"2023-10-09T04:23:51.358293+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:23:51.358293+0000","last_undegraded":"2023-10-09T04:23:51.358293+0000","last_fullsized":"2023-10-09T04:23:51.358293+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"424'4838","reported_seq":4900,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.887990+0000","last_change":"2023-10-09T04:22:13.633298+0000","last_active":"2023-10-09T04:24:00.887990+0000","last_peered":"2023-10-09T04:24:00.887990+0000","last_clean":"2023-10-09T04:24:00.887990+0000","last_became_active":"2023-10-09T04:22:13.632986+0000","last_became_peered":"2023-10-09T04:22:13.632986+0000","last_unstale":"2023-10-09T04:24:00.887990+0000","last_undegraded":"2023-10-09T04:24:00.887990+0000","last_fullsized":"2023-10-09T04:24:00.887990+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4838,"log_dups_size":0,"ondisk_log_size":4838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:11.170006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286785536,"num_objects":4376,"num_object_clones":0,"num_object_copies":13128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4838,"num_write_kb":294848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":655,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.377566+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:23:51.377566+0000","last_peered":"2023-10-09T04:23:51.377566+0000","last_clean":"2023-10-09T04:23:51.377566+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:23:51.377566+0000","last_undegraded":"2023-10-09T04:23:51.377566+0000","last_fullsized":"2023-10-09T04:23:51.377566+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":430,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353341+0000","last_change":"2023-10-09T04:23:22.467275+0000","last_active":"2023-10-09T04:23:51.353341+0000","last_peered":"2023-10-09T04:23:51.353341+0000","last_clean":"2023-10-09T04:23:51.353341+0000","last_became_active":"2023-10-09T04:23:22.380682+0000","last_became_peered":"2023-10-09T04:23:22.380682+0000","last_unstale":"2023-10-09T04:23:51.353341+0000","last_undegraded":"2023-10-09T04:23:51.353341+0000","last_fullsized":"2023-10-09T04:23:51.353341+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:18.953922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":134,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.055831+0000","last_change":"2023-10-09T04:23:22.371153+0000","last_active":"2023-10-09T04:23:41.055831+0000","last_peered":"2023-10-09T04:23:41.055831+0000","last_clean":"2023-10-09T04:23:41.055831+0000","last_became_active":"2023-10-09T04:23:22.370388+0000","last_became_peered":"2023-10-09T04:23:22.370388+0000","last_unstale":"2023-10-09T04:23:41.055831+0000","last_undegraded":"2023-10-09T04:23:41.055831+0000","last_fullsized":"2023-10-09T04:23:41.055831+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:50.346844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"424'4832","reported_seq":7566,"reported_epoch":424,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:23:59.618713+0000","last_change":"2023-10-09T04:23:29.223689+0000","last_active":"2023-10-09T04:23:59.618713+0000","last_peered":"2023-10-09T04:23:59.618713+0000","last_clean":"2023-10-09T04:22:25.963778+0000","last_became_active":"2023-10-09T04:23:28.780792+0000","last_became_peered":"2023-10-09T04:23:28.780792+0000","last_unstale":"2023-10-09T04:23:59.618713+0000","last_undegraded":"2023-10-09T04:23:59.618713+0000","last_fullsized":"2023-10-09T04:23:28.693571+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4832,"log_dups_size":0,"ondisk_log_size":4832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:00.096345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288096256,"num_objects":4396,"num_object_clones":0,"num_object_copies":13188,"num_objects_missing_on_primary":1481,"num_objects_missing":1481,"num_objects_degraded":0,"num_objects_misplaced":2997,"num_objects_unfound":0,"num_objects_dirty":4396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4832,"num_write_kb":295296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1329,"num_bytes_recovered":87097344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"0,3","objects":1399},{"shards":"0,3,4","objects":1516},{"shards":"1,3,4","objects":1481}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":608,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359460+0000","last_change":"2023-10-09T04:23:25.386738+0000","last_active":"2023-10-09T04:23:51.359460+0000","last_peered":"2023-10-09T04:23:51.359460+0000","last_clean":"2023-10-09T04:23:51.359460+0000","last_became_active":"2023-10-09T04:23:25.386422+0000","last_became_peered":"2023-10-09T04:23:25.386422+0000","last_unstale":"2023-10-09T04:23:51.359460+0000","last_undegraded":"2023-10-09T04:23:51.359460+0000","last_fullsized":"2023-10-09T04:23:51.359460+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:35.084506+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":395,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352757+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:23:51.352757+0000","last_peered":"2023-10-09T04:23:51.352757+0000","last_clean":"2023-10-09T04:23:51.352757+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:23:51.352757+0000","last_undegraded":"2023-10-09T04:23:51.352757+0000","last_fullsized":"2023-10-09T04:23:51.352757+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":49,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.035603+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:23:41.035603+0000","last_peered":"2023-10-09T04:23:41.035603+0000","last_clean":"2023-10-09T04:23:41.035603+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:23:41.035603+0000","last_undegraded":"2023-10-09T04:23:41.035603+0000","last_fullsized":"2023-10-09T04:23:41.035603+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"424'4939","reported_seq":7436,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:23:59.531017+0000","last_change":"2023-10-09T04:23:22.509836+0000","last_active":"2023-10-09T04:23:59.531017+0000","last_peered":"2023-10-09T04:23:59.531017+0000","last_clean":"2023-10-09T04:23:05.287910+0000","last_became_active":"2023-10-09T04:23:07.492061+0000","last_became_peered":"2023-10-09T04:23:07.492061+0000","last_unstale":"2023-10-09T04:23:59.531017+0000","last_undegraded":"2023-10-09T04:23:59.531017+0000","last_fullsized":"2023-10-09T04:23:59.531017+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4939,"log_dups_size":0,"ondisk_log_size":4939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:05.613285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294846464,"num_objects":4499,"num_object_clones":0,"num_object_copies":13497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4499,"num_objects_unfound":0,"num_objects_dirty":4499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4939,"num_write_kb":302016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1207,"num_bytes_recovered":79101952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":4499}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":703,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353192+0000","last_change":"2023-10-09T04:23:25.380799+0000","last_active":"2023-10-09T04:23:51.353192+0000","last_peered":"2023-10-09T04:23:51.353192+0000","last_clean":"2023-10-09T04:23:51.353192+0000","last_became_active":"2023-10-09T04:23:25.380434+0000","last_became_peered":"2023-10-09T04:23:25.380434+0000","last_unstale":"2023-10-09T04:23:51.353192+0000","last_undegraded":"2023-10-09T04:23:51.353192+0000","last_fullsized":"2023-10-09T04:23:51.353192+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":423,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359222+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:23:51.359222+0000","last_peered":"2023-10-09T04:23:51.359222+0000","last_clean":"2023-10-09T04:23:51.359222+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:23:51.359222+0000","last_undegraded":"2023-10-09T04:23:51.359222+0000","last_fullsized":"2023-10-09T04:23:51.359222+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372993+0000","last_change":"2023-10-09T04:23:22.367231+0000","last_active":"2023-10-09T04:23:51.372993+0000","last_peered":"2023-10-09T04:23:51.372993+0000","last_clean":"2023-10-09T04:23:51.372993+0000","last_became_active":"2023-10-09T04:23:22.366606+0000","last_became_peered":"2023-10-09T04:23:22.366606+0000","last_unstale":"2023-10-09T04:23:51.372993+0000","last_undegraded":"2023-10-09T04:23:51.372993+0000","last_fullsized":"2023-10-09T04:23:51.372993+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:57.425211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"423'4837","reported_seq":7321,"reported_epoch":423,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:23:58.516606+0000","last_change":"2023-10-09T04:23:37.018351+0000","last_active":"2023-10-09T04:23:58.516606+0000","last_peered":"2023-10-09T04:23:58.516606+0000","last_clean":"2023-10-09T04:23:05.307068+0000","last_became_active":"2023-10-09T04:23:37.011211+0000","last_became_peered":"2023-10-09T04:23:37.011211+0000","last_unstale":"2023-10-09T04:23:58.516606+0000","last_undegraded":"2023-10-09T04:23:58.516606+0000","last_fullsized":"2023-10-09T04:23:58.516606+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4837,"log_dups_size":0,"ondisk_log_size":4837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:56.349299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289865728,"num_objects":4423,"num_object_clones":0,"num_object_copies":13269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4423,"num_objects_unfound":0,"num_objects_dirty":4423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4837,"num_write_kb":296320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1192,"num_bytes_recovered":78118912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":4423}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":847,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352614+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:23:51.352614+0000","last_peered":"2023-10-09T04:23:51.352614+0000","last_clean":"2023-10-09T04:23:51.352614+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:23:51.352614+0000","last_undegraded":"2023-10-09T04:23:51.352614+0000","last_fullsized":"2023-10-09T04:23:51.352614+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":44,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372859+0000","last_change":"2023-10-09T04:23:22.384794+0000","last_active":"2023-10-09T04:23:51.372859+0000","last_peered":"2023-10-09T04:23:51.372859+0000","last_clean":"2023-10-09T04:23:51.372859+0000","last_became_active":"2023-10-09T04:23:22.370199+0000","last_became_peered":"2023-10-09T04:23:22.370199+0000","last_unstale":"2023-10-09T04:23:51.372859+0000","last_undegraded":"2023-10-09T04:23:51.372859+0000","last_fullsized":"2023-10-09T04:23:51.372859+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:35.104932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":174,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.377387+0000","last_change":"2023-10-09T04:23:37.016405+0000","last_active":"2023-10-09T04:23:51.377387+0000","last_peered":"2023-10-09T04:23:51.377387+0000","last_clean":"2023-10-09T04:23:51.377387+0000","last_became_active":"2023-10-09T04:23:37.016108+0000","last_became_peered":"2023-10-09T04:23:37.016108+0000","last_unstale":"2023-10-09T04:23:51.377387+0000","last_undegraded":"2023-10-09T04:23:51.377387+0000","last_fullsized":"2023-10-09T04:23:51.377387+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:54.303768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"424'4799","reported_seq":4852,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T04:23:59.602620+0000","last_change":"2023-10-09T04:22:11.028798+0000","last_active":"2023-10-09T04:23:59.602620+0000","last_peered":"2023-10-09T04:23:59.602620+0000","last_clean":"2023-10-09T04:23:59.602620+0000","last_became_active":"2023-10-09T04:22:11.028513+0000","last_became_peered":"2023-10-09T04:22:11.028513+0000","last_unstale":"2023-10-09T04:23:59.602620+0000","last_undegraded":"2023-10-09T04:23:59.602620+0000","last_fullsized":"2023-10-09T04:23:59.602620+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4799,"log_dups_size":0,"ondisk_log_size":4799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:15:47.494441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286588928,"num_objects":4373,"num_object_clones":0,"num_object_copies":13119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4799,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":593,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358120+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:23:51.358120+0000","last_peered":"2023-10-09T04:23:51.358120+0000","last_clean":"2023-10-09T04:23:51.358120+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:23:51.358120+0000","last_undegraded":"2023-10-09T04:23:51.358120+0000","last_fullsized":"2023-10-09T04:23:51.358120+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.035240+0000","last_change":"2023-10-09T04:23:22.368751+0000","last_active":"2023-10-09T04:23:41.035240+0000","last_peered":"2023-10-09T04:23:41.035240+0000","last_clean":"2023-10-09T04:23:41.035240+0000","last_became_active":"2023-10-09T04:23:22.368404+0000","last_became_peered":"2023-10-09T04:23:22.368404+0000","last_unstale":"2023-10-09T04:23:41.035240+0000","last_undegraded":"2023-10-09T04:23:41.035240+0000","last_fullsized":"2023-10-09T04:23:41.035240+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:11.713600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379978+0000","last_change":"2023-10-09T04:23:22.367561+0000","last_active":"2023-10-09T04:23:51.379978+0000","last_peered":"2023-10-09T04:23:51.379978+0000","last_clean":"2023-10-09T04:23:51.379978+0000","last_became_active":"2023-10-09T04:23:22.367246+0000","last_became_peered":"2023-10-09T04:23:22.367246+0000","last_unstale":"2023-10-09T04:23:51.379978+0000","last_undegraded":"2023-10-09T04:23:51.379978+0000","last_fullsized":"2023-10-09T04:23:51.379978+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:52.238902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"423'4789","reported_seq":9984,"reported_epoch":423,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:23:58.637844+0000","last_change":"2023-10-09T04:23:23.644363+0000","last_active":"2023-10-09T04:23:58.637844+0000","last_peered":"2023-10-09T04:23:58.637844+0000","last_clean":"2023-10-09T04:23:21.324155+0000","last_became_active":"2023-10-09T04:23:23.587135+0000","last_became_peered":"2023-10-09T04:23:23.587135+0000","last_unstale":"2023-10-09T04:23:58.637844+0000","last_undegraded":"2023-10-09T04:23:58.637844+0000","last_fullsized":"2023-10-09T04:23:23.357656+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4789,"log_dups_size":0,"ondisk_log_size":4789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:39.963675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290127872,"num_objects":4427,"num_object_clones":0,"num_object_copies":13281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1768,"num_objects_unfound":0,"num_objects_dirty":4427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4789,"num_write_kb":294912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2564,"num_bytes_recovered":168034304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[1,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1768},{"shards":"1,6","objects":2659}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":528,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.373037+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:23:51.373037+0000","last_peered":"2023-10-09T04:23:51.373037+0000","last_clean":"2023-10-09T04:23:51.373037+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:23:51.373037+0000","last_undegraded":"2023-10-09T04:23:51.373037+0000","last_fullsized":"2023-10-09T04:23:51.373037+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":91,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.376914+0000","last_change":"2023-10-09T04:23:22.369499+0000","last_active":"2023-10-09T04:23:51.376914+0000","last_peered":"2023-10-09T04:23:51.376914+0000","last_clean":"2023-10-09T04:23:51.376914+0000","last_became_active":"2023-10-09T04:23:22.369029+0000","last_became_peered":"2023-10-09T04:23:22.369029+0000","last_unstale":"2023-10-09T04:23:51.376914+0000","last_undegraded":"2023-10-09T04:23:51.376914+0000","last_fullsized":"2023-10-09T04:23:51.376914+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:43.955123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":198,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.035212+0000","last_change":"2023-10-09T04:23:22.369235+0000","last_active":"2023-10-09T04:23:41.035212+0000","last_peered":"2023-10-09T04:23:41.035212+0000","last_clean":"2023-10-09T04:23:41.035212+0000","last_became_active":"2023-10-09T04:23:22.368922+0000","last_became_peered":"2023-10-09T04:23:22.368922+0000","last_unstale":"2023-10-09T04:23:41.035212+0000","last_undegraded":"2023-10-09T04:23:41.035212+0000","last_fullsized":"2023-10-09T04:23:41.035212+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:42.940762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"424'5100","reported_seq":7529,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:24:00.988603+0000","last_change":"2023-10-09T04:23:28.969635+0000","last_active":"2023-10-09T04:24:00.988603+0000","last_peered":"2023-10-09T04:24:00.988603+0000","last_clean":"2023-10-09T04:22:25.940554+0000","last_became_active":"2023-10-09T04:23:23.582458+0000","last_became_peered":"2023-10-09T04:23:23.582458+0000","last_unstale":"2023-10-09T04:24:00.988603+0000","last_undegraded":"2023-10-09T04:23:28.967543+0000","last_fullsized":"2023-10-09T04:23:23.363286+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:51.414559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298450944,"num_objects":4554,"num_object_clones":0,"num_object_copies":13662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3305,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5100,"num_write_kb":308928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1174,"num_bytes_recovered":76939264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"0,4,7","objects":1715},{"shards":"0,7","objects":1249},{"shards":"1,4,7","objects":1317}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":588,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358631+0000","last_change":"2023-10-09T04:22:16.643512+0000","last_active":"2023-10-09T04:23:51.358631+0000","last_peered":"2023-10-09T04:23:51.358631+0000","last_clean":"2023-10-09T04:23:51.358631+0000","last_became_active":"2023-10-09T04:22:16.643150+0000","last_became_peered":"2023-10-09T04:22:16.643150+0000","last_unstale":"2023-10-09T04:23:51.358631+0000","last_undegraded":"2023-10-09T04:23:51.358631+0000","last_fullsized":"2023-10-09T04:23:51.358631+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:51.421460+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.375629+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:23:51.375629+0000","last_peered":"2023-10-09T04:23:51.375629+0000","last_clean":"2023-10-09T04:23:51.375629+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:23:51.375629+0000","last_undegraded":"2023-10-09T04:23:51.375629+0000","last_fullsized":"2023-10-09T04:23:51.375629+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379878+0000","last_change":"2023-10-09T04:23:22.372732+0000","last_active":"2023-10-09T04:23:51.379878+0000","last_peered":"2023-10-09T04:23:51.379878+0000","last_clean":"2023-10-09T04:23:51.379878+0000","last_became_active":"2023-10-09T04:23:22.372423+0000","last_became_peered":"2023-10-09T04:23:22.372423+0000","last_unstale":"2023-10-09T04:23:51.379878+0000","last_undegraded":"2023-10-09T04:23:51.379878+0000","last_fullsized":"2023-10-09T04:23:51.379878+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:26:09.560915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"424'4803","reported_seq":7174,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:00.882594+0000","last_change":"2023-10-09T04:23:28.776984+0000","last_active":"2023-10-09T04:24:00.882594+0000","last_peered":"2023-10-09T04:24:00.882594+0000","last_clean":"2023-10-09T04:23:05.321351+0000","last_became_active":"2023-10-09T04:23:28.772370+0000","last_became_peered":"2023-10-09T04:23:28.772370+0000","last_unstale":"2023-10-09T04:24:00.882594+0000","last_undegraded":"2023-10-09T04:24:00.882594+0000","last_fullsized":"2023-10-09T04:24:00.882594+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:48.147071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285802496,"num_objects":4361,"num_object_clones":0,"num_object_copies":13083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4361,"num_objects_unfound":0,"num_objects_dirty":4361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4803,"num_write_kb":293248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1137,"num_bytes_recovered":74514432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,3,4],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"3,4,7","objects":4361}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":549,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.035400+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:23:41.035400+0000","last_peered":"2023-10-09T04:23:41.035400+0000","last_clean":"2023-10-09T04:23:41.035400+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:23:41.035400+0000","last_undegraded":"2023-10-09T04:23:41.035400+0000","last_fullsized":"2023-10-09T04:23:41.035400+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.424010+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:23:51.424010+0000","last_peered":"2023-10-09T04:23:51.424010+0000","last_clean":"2023-10-09T04:23:51.424010+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:23:51.424010+0000","last_undegraded":"2023-10-09T04:23:51.424010+0000","last_fullsized":"2023-10-09T04:23:51.424010+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":420,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359599+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:23:51.359599+0000","last_peered":"2023-10-09T04:23:51.359599+0000","last_clean":"2023-10-09T04:23:51.359599+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:23:51.359599+0000","last_undegraded":"2023-10-09T04:23:51.359599+0000","last_fullsized":"2023-10-09T04:23:51.359599+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":101,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380312+0000","last_change":"2023-10-09T04:23:22.376119+0000","last_active":"2023-10-09T04:23:51.380312+0000","last_peered":"2023-10-09T04:23:51.380312+0000","last_clean":"2023-10-09T04:23:51.380312+0000","last_became_active":"2023-10-09T04:23:22.375805+0000","last_became_peered":"2023-10-09T04:23:22.375805+0000","last_unstale":"2023-10-09T04:23:51.380312+0000","last_undegraded":"2023-10-09T04:23:51.380312+0000","last_fullsized":"2023-10-09T04:23:51.380312+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:41.416810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"424'4782","reported_seq":7700,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:23:59.544977+0000","last_change":"2023-10-09T04:23:22.384585+0000","last_active":"2023-10-09T04:23:59.544977+0000","last_peered":"2023-10-09T04:23:59.544977+0000","last_clean":"2023-10-09T04:23:05.256815+0000","last_became_active":"2023-10-09T04:23:07.467082+0000","last_became_peered":"2023-10-09T04:23:07.467082+0000","last_unstale":"2023-10-09T04:23:59.544977+0000","last_undegraded":"2023-10-09T04:23:59.544977+0000","last_fullsized":"2023-10-09T04:23:59.544977+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.977596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286130176,"num_objects":4366,"num_object_clones":0,"num_object_copies":13098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4177,"num_objects_unfound":0,"num_objects_dirty":4366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4782,"num_write_kb":292736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1412,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":4366}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":591,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358609+0000","last_change":"2023-10-09T04:23:08.631094+0000","last_active":"2023-10-09T04:23:51.358609+0000","last_peered":"2023-10-09T04:23:51.358609+0000","last_clean":"2023-10-09T04:23:51.358609+0000","last_became_active":"2023-10-09T04:23:08.630831+0000","last_became_peered":"2023-10-09T04:23:08.630831+0000","last_unstale":"2023-10-09T04:23:51.358609+0000","last_undegraded":"2023-10-09T04:23:51.358609+0000","last_fullsized":"2023-10-09T04:23:51.358609+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:17.921309+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":407,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358751+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:23:51.358751+0000","last_peered":"2023-10-09T04:23:51.358751+0000","last_clean":"2023-10-09T04:23:51.358751+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:23:51.358751+0000","last_undegraded":"2023-10-09T04:23:51.358751+0000","last_fullsized":"2023-10-09T04:23:51.358751+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":204,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372721+0000","last_change":"2023-10-09T04:23:22.370474+0000","last_active":"2023-10-09T04:23:51.372721+0000","last_peered":"2023-10-09T04:23:51.372721+0000","last_clean":"2023-10-09T04:23:51.372721+0000","last_became_active":"2023-10-09T04:23:22.370173+0000","last_became_peered":"2023-10-09T04:23:22.370173+0000","last_unstale":"2023-10-09T04:23:51.372721+0000","last_undegraded":"2023-10-09T04:23:51.372721+0000","last_fullsized":"2023-10-09T04:23:51.372721+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.871804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":366,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380088+0000","last_change":"2023-10-09T04:22:02.693705+0000","last_active":"2023-10-09T04:23:51.380088+0000","last_peered":"2023-10-09T04:23:51.380088+0000","last_clean":"2023-10-09T04:23:51.380088+0000","last_became_active":"2023-10-09T04:22:02.693400+0000","last_became_peered":"2023-10-09T04:22:02.693400+0000","last_unstale":"2023-10-09T04:23:51.380088+0000","last_undegraded":"2023-10-09T04:23:51.380088+0000","last_fullsized":"2023-10-09T04:23:51.380088+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.772664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"424'4956","reported_seq":5008,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-10-09T04:24:00.924029+0000","last_change":"2023-10-09T04:22:10.594373+0000","last_active":"2023-10-09T04:24:00.924029+0000","last_peered":"2023-10-09T04:24:00.924029+0000","last_clean":"2023-10-09T04:24:00.924029+0000","last_became_active":"2023-10-09T04:22:10.594072+0000","last_became_peered":"2023-10-09T04:22:10.594072+0000","last_unstale":"2023-10-09T04:24:00.924029+0000","last_undegraded":"2023-10-09T04:24:00.924029+0000","last_fullsized":"2023-10-09T04:24:00.924029+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4956,"log_dups_size":0,"ondisk_log_size":4956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:59.326741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292421632,"num_objects":4462,"num_object_clones":0,"num_object_copies":13386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4956,"num_write_kb":301376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":562,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372295+0000","last_change":"2023-10-09T04:23:25.379708+0000","last_active":"2023-10-09T04:23:51.372295+0000","last_peered":"2023-10-09T04:23:51.372295+0000","last_clean":"2023-10-09T04:23:51.372295+0000","last_became_active":"2023-10-09T04:23:25.379318+0000","last_became_peered":"2023-10-09T04:23:25.379318+0000","last_unstale":"2023-10-09T04:23:51.372295+0000","last_undegraded":"2023-10-09T04:23:51.372295+0000","last_fullsized":"2023-10-09T04:23:51.372295+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:36.564798+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":279,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.376755+0000","last_change":"2023-10-09T04:23:06.342643+0000","last_active":"2023-10-09T04:23:51.376755+0000","last_peered":"2023-10-09T04:23:51.376755+0000","last_clean":"2023-10-09T04:23:51.376755+0000","last_became_active":"2023-10-09T04:23:06.341890+0000","last_became_peered":"2023-10-09T04:23:06.341890+0000","last_unstale":"2023-10-09T04:23:51.376755+0000","last_undegraded":"2023-10-09T04:23:51.376755+0000","last_fullsized":"2023-10-09T04:23:51.376755+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:54.832808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":355,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379638+0000","last_change":"2023-10-09T04:22:13.627844+0000","last_active":"2023-10-09T04:23:51.379638+0000","last_peered":"2023-10-09T04:23:51.379638+0000","last_clean":"2023-10-09T04:23:51.379638+0000","last_became_active":"2023-10-09T04:22:13.627492+0000","last_became_peered":"2023-10-09T04:22:13.627492+0000","last_unstale":"2023-10-09T04:23:51.379638+0000","last_undegraded":"2023-10-09T04:23:51.379638+0000","last_fullsized":"2023-10-09T04:23:51.379638+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:00.686222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359664+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:23:51.359664+0000","last_peered":"2023-10-09T04:23:51.359664+0000","last_clean":"2023-10-09T04:23:51.359664+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:23:51.359664+0000","last_undegraded":"2023-10-09T04:23:51.359664+0000","last_fullsized":"2023-10-09T04:23:51.359664+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"423'4892","reported_seq":7518,"reported_epoch":423,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:23:57.518496+0000","last_change":"2023-10-09T04:23:23.417570+0000","last_active":"2023-10-09T04:23:57.518496+0000","last_peered":"2023-10-09T04:23:57.518496+0000","last_clean":"2023-10-09T04:23:05.323569+0000","last_became_active":"2023-10-09T04:23:07.488021+0000","last_became_peered":"2023-10-09T04:23:07.488021+0000","last_unstale":"2023-10-09T04:23:57.518496+0000","last_undegraded":"2023-10-09T04:23:57.518496+0000","last_fullsized":"2023-10-09T04:23:57.518496+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4892,"log_dups_size":0,"ondisk_log_size":4892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:27.368472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4290,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4892,"num_write_kb":300032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1366,"num_bytes_recovered":89522176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,4],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4484}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":407,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359162+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:23:51.359162+0000","last_peered":"2023-10-09T04:23:51.359162+0000","last_clean":"2023-10-09T04:23:51.359162+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:23:51.359162+0000","last_undegraded":"2023-10-09T04:23:51.359162+0000","last_fullsized":"2023-10-09T04:23:51.359162+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":878,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352034+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:23:51.352034+0000","last_peered":"2023-10-09T04:23:51.352034+0000","last_clean":"2023-10-09T04:23:51.352034+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:23:51.352034+0000","last_undegraded":"2023-10-09T04:23:51.352034+0000","last_fullsized":"2023-10-09T04:23:51.352034+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":360,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358814+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:23:51.358814+0000","last_peered":"2023-10-09T04:23:51.358814+0000","last_clean":"2023-10-09T04:23:51.358814+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:23:51.358814+0000","last_undegraded":"2023-10-09T04:23:51.358814+0000","last_fullsized":"2023-10-09T04:23:51.358814+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":161,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352991+0000","last_change":"2023-10-09T04:23:22.381833+0000","last_active":"2023-10-09T04:23:51.352991+0000","last_peered":"2023-10-09T04:23:51.352991+0000","last_clean":"2023-10-09T04:23:51.352991+0000","last_became_active":"2023-10-09T04:23:22.381088+0000","last_became_peered":"2023-10-09T04:23:22.381088+0000","last_unstale":"2023-10-09T04:23:51.352991+0000","last_undegraded":"2023-10-09T04:23:51.352991+0000","last_fullsized":"2023-10-09T04:23:51.352991+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:32.847976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"424'4838","reported_seq":7403,"reported_epoch":424,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:23:59.583625+0000","last_change":"2023-10-09T04:23:23.635949+0000","last_active":"2023-10-09T04:23:59.583625+0000","last_peered":"2023-10-09T04:23:59.583625+0000","last_clean":"2023-10-09T04:23:05.325743+0000","last_became_active":"2023-10-09T04:23:07.460500+0000","last_became_peered":"2023-10-09T04:23:07.460500+0000","last_unstale":"2023-10-09T04:23:59.583625+0000","last_undegraded":"2023-10-09T04:23:59.583625+0000","last_fullsized":"2023-10-09T04:23:59.583625+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4838,"log_dups_size":0,"ondisk_log_size":4838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:49.200134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287309824,"num_objects":4384,"num_object_clones":0,"num_object_copies":13152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4353,"num_objects_unfound":0,"num_objects_dirty":4384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4838,"num_write_kb":295104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1241,"num_bytes_recovered":81330176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":4384}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":540,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359067+0000","last_change":"2023-10-09T04:23:25.384702+0000","last_active":"2023-10-09T04:23:51.359067+0000","last_peered":"2023-10-09T04:23:51.359067+0000","last_clean":"2023-10-09T04:23:51.359067+0000","last_became_active":"2023-10-09T04:23:25.384324+0000","last_became_peered":"2023-10-09T04:23:25.384324+0000","last_unstale":"2023-10-09T04:23:51.359067+0000","last_undegraded":"2023-10-09T04:23:51.359067+0000","last_fullsized":"2023-10-09T04:23:51.359067+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:03.536732+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":424,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.351957+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:23:51.351957+0000","last_peered":"2023-10-09T04:23:51.351957+0000","last_clean":"2023-10-09T04:23:51.351957+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:23:51.351957+0000","last_undegraded":"2023-10-09T04:23:51.351957+0000","last_fullsized":"2023-10-09T04:23:51.351957+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372041+0000","last_change":"2023-10-09T04:23:22.378846+0000","last_active":"2023-10-09T04:23:51.372041+0000","last_peered":"2023-10-09T04:23:51.372041+0000","last_clean":"2023-10-09T04:23:51.372041+0000","last_became_active":"2023-10-09T04:23:22.377975+0000","last_became_peered":"2023-10-09T04:23:22.377975+0000","last_unstale":"2023-10-09T04:23:51.372041+0000","last_undegraded":"2023-10-09T04:23:51.372041+0000","last_fullsized":"2023-10-09T04:23:51.372041+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:47.753526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":27,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372578+0000","last_change":"2023-10-09T04:23:06.341197+0000","last_active":"2023-10-09T04:23:51.372578+0000","last_peered":"2023-10-09T04:23:51.372578+0000","last_clean":"2023-10-09T04:23:51.372578+0000","last_became_active":"2023-10-09T04:23:06.340773+0000","last_became_peered":"2023-10-09T04:23:06.340773+0000","last_unstale":"2023-10-09T04:23:51.372578+0000","last_undegraded":"2023-10-09T04:23:51.372578+0000","last_fullsized":"2023-10-09T04:23:51.372578+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:55.764215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"424'4738","reported_seq":7100,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:23:59.677253+0000","last_change":"2023-10-09T04:23:28.392348+0000","last_active":"2023-10-09T04:23:59.677253+0000","last_peered":"2023-10-09T04:23:59.677253+0000","last_clean":"2023-10-09T04:22:25.956502+0000","last_became_active":"2023-10-09T04:23:28.336672+0000","last_became_peered":"2023-10-09T04:23:28.336672+0000","last_unstale":"2023-10-09T04:23:59.677253+0000","last_undegraded":"2023-10-09T04:23:28.265355+0000","last_fullsized":"2023-10-09T04:23:23.362505+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4738,"log_dups_size":0,"ondisk_log_size":4738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:35.497725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281739288,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":2352,"num_objects_missing":2352,"num_objects_degraded":3089,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4738,"num_write_kb":289153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1132,"num_bytes_recovered":74186752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":737},{"shards":"0,6","objects":1211},{"shards":"3,4,6","objects":2352}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":541,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.041458+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:23:41.041458+0000","last_peered":"2023-10-09T04:23:41.041458+0000","last_clean":"2023-10-09T04:23:41.041458+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:23:41.041458+0000","last_undegraded":"2023-10-09T04:23:41.041458+0000","last_fullsized":"2023-10-09T04:23:41.041458+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":437,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.352835+0000","last_change":"2023-10-09T04:23:22.381998+0000","last_active":"2023-10-09T04:23:51.352835+0000","last_peered":"2023-10-09T04:23:51.352835+0000","last_clean":"2023-10-09T04:23:51.352835+0000","last_became_active":"2023-10-09T04:23:22.381540+0000","last_became_peered":"2023-10-09T04:23:22.381540+0000","last_unstale":"2023-10-09T04:23:51.352835+0000","last_undegraded":"2023-10-09T04:23:51.352835+0000","last_fullsized":"2023-10-09T04:23:51.352835+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:21.053870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":215,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380504+0000","last_change":"2023-10-09T04:22:02.694465+0000","last_active":"2023-10-09T04:23:51.380504+0000","last_peered":"2023-10-09T04:23:51.380504+0000","last_clean":"2023-10-09T04:23:51.380504+0000","last_became_active":"2023-10-09T04:22:02.694159+0000","last_became_peered":"2023-10-09T04:22:02.694159+0000","last_unstale":"2023-10-09T04:23:51.380504+0000","last_undegraded":"2023-10-09T04:23:51.380504+0000","last_fullsized":"2023-10-09T04:23:51.380504+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:29:44.036560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":143,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359073+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:23:51.359073+0000","last_peered":"2023-10-09T04:23:51.359073+0000","last_clean":"2023-10-09T04:23:51.359073+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:23:51.359073+0000","last_undegraded":"2023-10-09T04:23:51.359073+0000","last_fullsized":"2023-10-09T04:23:51.359073+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"423'4755","reported_seq":9578,"reported_epoch":423,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:23:57.537273+0000","last_change":"2023-10-09T04:23:32.723188+0000","last_active":"2023-10-09T04:23:57.537273+0000","last_peered":"2023-10-09T04:23:57.537273+0000","last_clean":"2023-10-09T04:23:21.349728+0000","last_became_active":"2023-10-09T04:23:32.569693+0000","last_became_peered":"2023-10-09T04:23:32.569693+0000","last_unstale":"2023-10-09T04:23:57.537273+0000","last_undegraded":"2023-10-09T04:23:57.537273+0000","last_fullsized":"2023-10-09T04:23:32.497946+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4755,"log_dups_size":0,"ondisk_log_size":4755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:17.334468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286720000,"num_objects":4375,"num_object_clones":0,"num_object_copies":13125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1916,"num_objects_unfound":0,"num_objects_dirty":4375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4755,"num_write_kb":292160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2375,"num_bytes_recovered":155648000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":1916},{"shards":"2,6","objects":2459}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":434,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379907+0000","last_change":"2023-10-09T04:23:25.381197+0000","last_active":"2023-10-09T04:23:51.379907+0000","last_peered":"2023-10-09T04:23:51.379907+0000","last_clean":"2023-10-09T04:23:51.379907+0000","last_became_active":"2023-10-09T04:23:25.380615+0000","last_became_peered":"2023-10-09T04:23:25.380615+0000","last_unstale":"2023-10-09T04:23:51.379907+0000","last_undegraded":"2023-10-09T04:23:51.379907+0000","last_fullsized":"2023-10-09T04:23:51.379907+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":384,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.353828+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:23:51.353828+0000","last_peered":"2023-10-09T04:23:51.353828+0000","last_clean":"2023-10-09T04:23:51.353828+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:23:51.353828+0000","last_undegraded":"2023-10-09T04:23:51.353828+0000","last_fullsized":"2023-10-09T04:23:51.353828+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":362,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.056147+0000","last_change":"2023-10-09T04:23:22.371790+0000","last_active":"2023-10-09T04:23:41.056147+0000","last_peered":"2023-10-09T04:23:41.056147+0000","last_clean":"2023-10-09T04:23:41.056147+0000","last_became_active":"2023-10-09T04:23:22.371464+0000","last_became_peered":"2023-10-09T04:23:22.371464+0000","last_unstale":"2023-10-09T04:23:41.056147+0000","last_undegraded":"2023-10-09T04:23:41.056147+0000","last_fullsized":"2023-10-09T04:23:41.056147+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:38.674475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":82,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:54.619389+0000","last_change":"2023-10-09T04:23:06.341990+0000","last_active":"2023-10-09T04:23:54.619389+0000","last_peered":"2023-10-09T04:23:54.619389+0000","last_clean":"2023-10-09T04:23:54.619389+0000","last_became_active":"2023-10-09T04:23:06.341695+0000","last_became_peered":"2023-10-09T04:23:06.341695+0000","last_unstale":"2023-10-09T04:23:54.619389+0000","last_undegraded":"2023-10-09T04:23:54.619389+0000","last_fullsized":"2023-10-09T04:23:54.619389+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:25.333591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.f","version":"423'4846","reported_seq":4895,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:57.502656+0000","last_change":"2023-10-09T04:22:13.630026+0000","last_active":"2023-10-09T04:23:57.502656+0000","last_peered":"2023-10-09T04:23:57.502656+0000","last_clean":"2023-10-09T04:23:57.502656+0000","last_became_active":"2023-10-09T04:22:13.629701+0000","last_became_peered":"2023-10-09T04:22:13.629701+0000","last_unstale":"2023-10-09T04:23:57.502656+0000","last_undegraded":"2023-10-09T04:23:57.502656+0000","last_fullsized":"2023-10-09T04:23:57.502656+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:47.148119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293339136,"num_objects":4476,"num_object_clones":0,"num_object_copies":13428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4846,"num_write_kb":298304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":597,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.377496+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:23:51.377496+0000","last_peered":"2023-10-09T04:23:51.377496+0000","last_clean":"2023-10-09T04:23:51.377496+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:23:51.377496+0000","last_undegraded":"2023-10-09T04:23:51.377496+0000","last_fullsized":"2023-10-09T04:23:51.377496+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":266,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379809+0000","last_change":"2023-10-09T04:22:13.630037+0000","last_active":"2023-10-09T04:23:51.379809+0000","last_peered":"2023-10-09T04:23:51.379809+0000","last_clean":"2023-10-09T04:23:51.379809+0000","last_became_active":"2023-10-09T04:22:13.629726+0000","last_became_peered":"2023-10-09T04:22:13.629726+0000","last_unstale":"2023-10-09T04:23:51.379809+0000","last_undegraded":"2023-10-09T04:23:51.379809+0000","last_fullsized":"2023-10-09T04:23:51.379809+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:32.281519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":44,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372800+0000","last_change":"2023-10-09T04:23:22.459821+0000","last_active":"2023-10-09T04:23:51.372800+0000","last_peered":"2023-10-09T04:23:51.372800+0000","last_clean":"2023-10-09T04:23:51.372800+0000","last_became_active":"2023-10-09T04:23:22.374691+0000","last_became_peered":"2023-10-09T04:23:22.374691+0000","last_unstale":"2023-10-09T04:23:51.372800+0000","last_undegraded":"2023-10-09T04:23:51.372800+0000","last_fullsized":"2023-10-09T04:23:51.372800+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:36.436501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":161,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.358530+0000","last_change":"2023-10-09T04:23:22.376671+0000","last_active":"2023-10-09T04:23:51.358530+0000","last_peered":"2023-10-09T04:23:51.358530+0000","last_clean":"2023-10-09T04:23:51.358530+0000","last_became_active":"2023-10-09T04:23:22.376262+0000","last_became_peered":"2023-10-09T04:23:22.376262+0000","last_unstale":"2023-10-09T04:23:51.358530+0000","last_undegraded":"2023-10-09T04:23:51.358530+0000","last_fullsized":"2023-10-09T04:23:51.358530+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:04.864080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":453,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.371961+0000","last_change":"2023-10-09T04:23:08.628990+0000","last_active":"2023-10-09T04:23:51.371961+0000","last_peered":"2023-10-09T04:23:51.371961+0000","last_clean":"2023-10-09T04:23:51.371961+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:23:51.371961+0000","last_undegraded":"2023-10-09T04:23:51.371961+0000","last_fullsized":"2023-10-09T04:23:51.371961+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.373141+0000","last_change":"2023-10-09T04:23:22.374059+0000","last_active":"2023-10-09T04:23:51.373141+0000","last_peered":"2023-10-09T04:23:51.373141+0000","last_clean":"2023-10-09T04:23:51.373141+0000","last_became_active":"2023-10-09T04:23:22.373732+0000","last_became_peered":"2023-10-09T04:23:22.373732+0000","last_unstale":"2023-10-09T04:23:51.373141+0000","last_undegraded":"2023-10-09T04:23:51.373141+0000","last_fullsized":"2023-10-09T04:23:51.373141+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:37.083009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":260,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380638+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:23:51.380638+0000","last_peered":"2023-10-09T04:23:51.380638+0000","last_clean":"2023-10-09T04:23:51.380638+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:23:51.380638+0000","last_undegraded":"2023-10-09T04:23:51.380638+0000","last_fullsized":"2023-10-09T04:23:51.380638+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":497,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.041376+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:23:41.041376+0000","last_peered":"2023-10-09T04:23:41.041376+0000","last_clean":"2023-10-09T04:23:41.041376+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:23:41.041376+0000","last_undegraded":"2023-10-09T04:23:41.041376+0000","last_fullsized":"2023-10-09T04:23:41.041376+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":427,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359248+0000","last_change":"2023-10-09T04:23:22.457283+0000","last_active":"2023-10-09T04:23:51.359248+0000","last_peered":"2023-10-09T04:23:51.359248+0000","last_clean":"2023-10-09T04:23:51.359248+0000","last_became_active":"2023-10-09T04:23:22.376038+0000","last_became_peered":"2023-10-09T04:23:22.376038+0000","last_unstale":"2023-10-09T04:23:51.359248+0000","last_undegraded":"2023-10-09T04:23:51.359248+0000","last_fullsized":"2023-10-09T04:23:51.359248+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:17:27.509695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380321+0000","last_change":"2023-10-09T04:23:37.123396+0000","last_active":"2023-10-09T04:23:51.380321+0000","last_peered":"2023-10-09T04:23:51.380321+0000","last_clean":"2023-10-09T04:23:51.380321+0000","last_became_active":"2023-10-09T04:23:37.123063+0000","last_became_peered":"2023-10-09T04:23:37.123063+0000","last_unstale":"2023-10-09T04:23:51.380321+0000","last_undegraded":"2023-10-09T04:23:51.380321+0000","last_fullsized":"2023-10-09T04:23:51.380321+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:49.975104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":438,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.359352+0000","last_change":"2023-10-09T04:23:22.463818+0000","last_active":"2023-10-09T04:23:51.359352+0000","last_peered":"2023-10-09T04:23:51.359352+0000","last_clean":"2023-10-09T04:23:51.359352+0000","last_became_active":"2023-10-09T04:23:22.376113+0000","last_became_peered":"2023-10-09T04:23:22.376113+0000","last_unstale":"2023-10-09T04:23:51.359352+0000","last_undegraded":"2023-10-09T04:23:51.359352+0000","last_fullsized":"2023-10-09T04:23:51.359352+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:12.436065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":204,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.372216+0000","last_change":"2023-10-09T04:23:22.375144+0000","last_active":"2023-10-09T04:23:51.372216+0000","last_peered":"2023-10-09T04:23:51.372216+0000","last_clean":"2023-10-09T04:23:51.372216+0000","last_became_active":"2023-10-09T04:23:22.374905+0000","last_became_peered":"2023-10-09T04:23:22.374905+0000","last_unstale":"2023-10-09T04:23:51.372216+0000","last_undegraded":"2023-10-09T04:23:51.372216+0000","last_fullsized":"2023-10-09T04:23:51.372216+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:32.802152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":771,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379638+0000","last_change":"2023-10-09T04:22:16.629747+0000","last_active":"2023-10-09T04:23:51.379638+0000","last_peered":"2023-10-09T04:23:51.379638+0000","last_clean":"2023-10-09T04:23:51.379638+0000","last_became_active":"2023-10-09T04:22:16.629376+0000","last_became_peered":"2023-10-09T04:22:16.629376+0000","last_unstale":"2023-10-09T04:23:51.379638+0000","last_undegraded":"2023-10-09T04:23:51.379638+0000","last_fullsized":"2023-10-09T04:23:51.379638+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:21:55.959478+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:23.285011+0000","scrub_duration":0.0016555039999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":279,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.380225+0000","last_change":"2023-10-09T04:22:13.627837+0000","last_active":"2023-10-09T04:23:51.380225+0000","last_peered":"2023-10-09T04:23:51.380225+0000","last_clean":"2023-10-09T04:23:51.380225+0000","last_became_active":"2023-10-09T04:22:13.627540+0000","last_became_peered":"2023-10-09T04:22:13.627540+0000","last_unstale":"2023-10-09T04:23:51.380225+0000","last_undegraded":"2023-10-09T04:23:51.380225+0000","last_fullsized":"2023-10-09T04:23:51.380225+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:24.491041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":278,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.376830+0000","last_change":"2023-10-09T04:23:06.342832+0000","last_active":"2023-10-09T04:23:51.376830+0000","last_peered":"2023-10-09T04:23:51.376830+0000","last_clean":"2023-10-09T04:23:51.376830+0000","last_became_active":"2023-10-09T04:23:06.342306+0000","last_became_peered":"2023-10-09T04:23:06.342306+0000","last_unstale":"2023-10-09T04:23:51.376830+0000","last_undegraded":"2023-10-09T04:23:51.376830+0000","last_fullsized":"2023-10-09T04:23:51.376830+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:19.300451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":538,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.379670+0000","last_change":"2023-10-09T04:23:25.385143+0000","last_active":"2023-10-09T04:23:51.379670+0000","last_peered":"2023-10-09T04:23:51.379670+0000","last_clean":"2023-10-09T04:23:51.379670+0000","last_became_active":"2023-10-09T04:23:25.384812+0000","last_became_peered":"2023-10-09T04:23:25.384812+0000","last_unstale":"2023-10-09T04:23:51.379670+0000","last_undegraded":"2023-10-09T04:23:51.379670+0000","last_fullsized":"2023-10-09T04:23:51.379670+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:02.335901+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":33,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-10-09T04:23:51.374129+0000","last_change":"2023-10-09T04:23:06.385526+0000","last_active":"2023-10-09T04:23:51.374129+0000","last_peered":"2023-10-09T04:23:51.374129+0000","last_clean":"2023-10-09T04:23:51.374129+0000","last_became_active":"2023-10-09T04:23:06.345850+0000","last_became_peered":"2023-10-09T04:23:06.345850+0000","last_unstale":"2023-10-09T04:23:51.374129+0000","last_undegraded":"2023-10-09T04:23:51.374129+0000","last_fullsized":"2023-10-09T04:23:51.374129+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":160,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-10-09T04:23:41.034858+0000","last_change":"2023-10-09T04:23:22.374292+0000","last_active":"2023-10-09T04:23:41.034858+0000","last_peered":"2023-10-09T04:23:41.034858+0000","last_clean":"2023-10-09T04:23:41.034858+0000","last_became_active":"2023-10-09T04:23:22.373904+0000","last_became_peered":"2023-10-09T04:23:22.373904+0000","last_unstale":"2023-10-09T04:23:41.034858+0000","last_undegraded":"2023-10-09T04:23:41.034858+0000","last_fullsized":"2023-10-09T04:23:41.034858+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:20.848759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4623892504,"num_objects":70556,"num_object_clones":0,"num_object_copies":211668,"num_objects_missing_on_primary":3833,"num_objects_missing":3833,"num_objects_degraded":6394,"num_objects_misplaced":37084,"num_objects_unfound":0,"num_objects_dirty":70556,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":77472,"num_write_kb":4736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17264,"num_bytes_recovered":1131413504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14926032896,"data_stored":14926020680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77472,"ondisk_log_size":77472,"up":48,"acting":43,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":11468,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":697707,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":43,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1632739,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201658,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2515040,"kb_used_data":1606936,"kb_used_omap":0,"kb_used_meta":908096,"kb_avail":91230112,"statfs":{"total":95995035648,"available":93419634688,"internally_reserved":0,"allocated":1645502464,"data_stored":1644245999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929890304},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":3.4470000000000001,"15min":3.4260000000000002},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":4.601,"5min":148.65899999999999,"15min":148.65899999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":1.4630000000000001,"5min":3.6720000000000002,"15min":3.4449999999999998},"min":{"1min":0.26400000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.2919999999999998,"5min":148.642,"15min":148.642},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:23:12 2023","interfaces":[{"interface":"back","average":{"1min":14.718999999999999,"5min":9.7829999999999995,"15min":6.7460000000000004},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":261.935,"5min":261.935,"15min":261.935},"last":0.33600000000000002},{"interface":"front","average":{"1min":14.441000000000001,"5min":9.4329999999999998,"15min":6.4889999999999999},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":262.07600000000002,"5min":262.07600000000002,"15min":262.07600000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":3.706,"15min":2.5459999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.3470000000000004,"5min":253.41200000000001,"15min":253.41200000000001},"last":1.7290000000000001},{"interface":"front","average":{"1min":1.516,"5min":4.2130000000000001,"15min":2.9630000000000001},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":8.5280000000000005,"5min":253.39400000000001,"15min":253.39400000000001},"last":1.7410000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:23:12 2023","interfaces":[{"interface":"back","average":{"1min":13.992000000000001,"5min":4.0359999999999996,"15min":2.9119999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":255.97300000000001,"5min":255.97300000000001,"15min":255.97300000000001},"last":7.4859999999999998},{"interface":"front","average":{"1min":14.292999999999999,"5min":4.4109999999999996,"15min":3.1720000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":256.06099999999998,"5min":256.06099999999998,"15min":256.06099999999998},"last":7.5}]},{"osd":4,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.89100000000000001,"15min":0.82999999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23400000000000001},"max":{"1min":3.4630000000000001,"5min":4.8529999999999998,"15min":16.195},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.3260000000000001,"15min":1.179},"min":{"1min":0.27900000000000003,"5min":0.251,"15min":0.251},"max":{"1min":7.2549999999999999,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.439}]},{"osd":5,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.6339999999999999,"15min":1.5449999999999999},"min":{"1min":0.29699999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.1440000000000001,"5min":13.122999999999999,"15min":120.066},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.234,"15min":1.379},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":15.795,"5min":15.795,"15min":120.09699999999999},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.51,"15min":1.159},"min":{"1min":0.311,"5min":0.311,"15min":0.307},"max":{"1min":5.0599999999999996,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.505},{"interface":"front","average":{"1min":1.359,"5min":1.0289999999999999,"15min":0.91200000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":4.2430000000000003,"5min":5.968,"15min":8.9009999999999998},"last":0.57299999999999995}]}]},{"osd":6,"up_from":47,"seq":201863463293,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307976,"kb_used_data":1994304,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":91437176,"statfs":{"total":95995035648,"available":93631668224,"internally_reserved":0,"allocated":2042167296,"data_stored":2040925830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":4.6429999999999998,"15min":4.0469999999999997},"min":{"1min":0.215,"5min":0.19800000000000001,"15min":0.193},"max":{"1min":2.3250000000000002,"5min":133.78999999999999,"15min":133.78999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.2989999999999999,"5min":4.6360000000000001,"15min":4.2050000000000001},"min":{"1min":0.26200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":10.039,"5min":133.69300000000001,"15min":133.69300000000001},"last":0.27700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:23:10 2023","interfaces":[{"interface":"back","average":{"1min":16.623999999999999,"5min":8.4819999999999993,"15min":5.3129999999999997},"min":{"1min":0.25900000000000001,"5min":0.24299999999999999,"15min":0.22800000000000001},"max":{"1min":315.73599999999999,"5min":315.73599999999999,"15min":315.73599999999999},"last":1.659},{"interface":"front","average":{"1min":17.366,"5min":8.7409999999999997,"15min":5.4560000000000004},"min":{"1min":0.26400000000000001,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":316.91500000000002,"5min":316.91500000000002,"15min":316.91500000000002},"last":0.36499999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:23:42 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.22,"15min":2.681},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23899999999999999},"max":{"1min":4.431,"5min":6.266,"15min":139.06899999999999},"last":2.6850000000000001},{"interface":"front","average":{"1min":1.292,"5min":1.2609999999999999,"15min":2.722},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.28599999999999998},"max":{"1min":4.4619999999999997,"5min":6.3040000000000003,"15min":139.173},"last":2.7090000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:23:10 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":2.1099999999999999,"15min":2.774},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0629999999999997,"5min":43.475999999999999,"15min":57.509999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.964,"15min":2.6379999999999999},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.3969999999999998,"5min":43.369,"15min":57.453000000000003},"last":0.499}]},{"osd":4,"last update":"Mon Oct 9 04:23:10 2023","interfaces":[{"interface":"back","average":{"1min":9.0069999999999997,"5min":2.5059999999999998,"15min":1.3420000000000001},"min":{"1min":0.28000000000000003,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":161.27500000000001,"5min":161.27500000000001,"15min":161.27500000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":9.1199999999999992,"5min":2.7770000000000001,"15min":1.4750000000000001},"min":{"1min":0.34000000000000002,"5min":0.27100000000000002,"15min":0.22600000000000001},"max":{"1min":160.32599999999999,"5min":160.32599999999999,"15min":160.32599999999999},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 04:23:10 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.95099999999999996,"15min":1.024},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1709999999999998,"5min":7.032,"15min":22.879000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.069,"15min":1.083},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.7690000000000001,"5min":6.4109999999999996,"15min":23.006},"last":0.40300000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.5950000000000002,"15min":1.5740000000000001},"min":{"1min":0.36799999999999999,"5min":0.32600000000000001,"15min":0.313},"max":{"1min":4.4939999999999998,"5min":161.39400000000001,"15min":161.39400000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.6259999999999999,"15min":1.6160000000000001},"min":{"1min":0.375,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":2.25,"5min":161.41200000000001,"15min":161.41200000000001},"last":0.34499999999999997}]}]},{"osd":1,"up_from":89,"seq":382252089603,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2176336,"kb_used_data":1506056,"kb_used_omap":9,"kb_used_meta":670262,"kb_avail":91568816,"statfs":{"total":95995035648,"available":93766467584,"internally_reserved":0,"allocated":1542201344,"data_stored":1540320227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":686349289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:18 2023","interfaces":[{"interface":"back","average":{"1min":21.552,"5min":5.923,"15min":5.2009999999999996},"min":{"1min":0.29499999999999998,"5min":0.17899999999999999,"15min":0.17199999999999999},"max":{"1min":259.26799999999997,"5min":259.26799999999997,"15min":259.26799999999997},"last":0.42499999999999999},{"interface":"front","average":{"1min":20.960999999999999,"5min":5.9640000000000004,"15min":5.9610000000000003},"min":{"1min":0.27300000000000002,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":258.46899999999999,"5min":258.46899999999999,"15min":258.46899999999999},"last":1.1699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:23:47 2023","interfaces":[{"interface":"back","average":{"1min":14.804,"5min":5.133,"15min":3.3380000000000001},"min":{"1min":0.23699999999999999,"5min":0.20699999999999999,"15min":0.191},"max":{"1min":259.315,"5min":259.315,"15min":259.315},"last":0.45700000000000002},{"interface":"front","average":{"1min":15.205,"5min":5.6239999999999997,"15min":3.7759999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":259.733,"5min":259.733,"15min":259.733},"last":1.224}]},{"osd":3,"last update":"Mon Oct 9 04:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.109,"15min":3.5190000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.6900000000000004,"5min":19.640000000000001,"15min":138.87299999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.94599999999999995,"5min":2.2749999999999999,"15min":4.0979999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.5870000000000002,"5min":12.353999999999999,"15min":138.49799999999999},"last":1.3089999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:23:18 2023","interfaces":[{"interface":"back","average":{"1min":15.848000000000001,"5min":4.9610000000000003,"15min":3.1160000000000001},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":257.83300000000003,"5min":257.83300000000003,"15min":257.83300000000003},"last":1.2569999999999999},{"interface":"front","average":{"1min":16.253,"5min":4.601,"15min":2.8300000000000001},"min":{"1min":0.39300000000000002,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":257.863,"5min":257.863,"15min":257.863},"last":0.50800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:23:18 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":2.169,"15min":2.3809999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":3.984,"5min":25.696999999999999,"15min":38.848999999999997},"last":1.2829999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":2.206,"15min":2.423},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":4.0380000000000003,"5min":21.893999999999998,"15min":39.593000000000004},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 04:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.734,"15min":2.0659999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.30299999999999999},"max":{"1min":3.9350000000000001,"5min":8.3529999999999998,"15min":25.931000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.5149999999999999,"15min":1.696},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29799999999999999},"max":{"1min":7.1399999999999997,"5min":19.440000000000001,"15min":19.440000000000001},"last":0.67000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:23:18 2023","interfaces":[{"interface":"back","average":{"1min":16.654,"5min":6.9260000000000002,"15min":3.9390000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":259.08499999999998,"5min":259.08499999999998,"15min":259.08499999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":16.988,"5min":6.5019999999999998,"15min":3.5760000000000001},"min":{"1min":0.39700000000000002,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":259.13299999999998,"5min":259.13299999999998,"15min":259.13299999999998},"last":0.63800000000000001}]}]},{"osd":0,"up_from":84,"seq":360777253129,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1584704,"kb_used_data":966776,"kb_used_omap":28,"kb_used_meta":617891,"kb_avail":92160448,"statfs":{"total":95995035648,"available":94372298752,"internally_reserved":0,"allocated":989978624,"data_stored":988000499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":632721232},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:23:14 2023","interfaces":[{"interface":"back","average":{"1min":6.5609999999999999,"5min":5.6289999999999996,"15min":4.923},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":96.305000000000007,"5min":209.215,"15min":209.215},"last":0.377},{"interface":"front","average":{"1min":6.0890000000000004,"5min":4.8849999999999998,"15min":4.968},"min":{"1min":0.26700000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":95.861000000000004,"5min":95.861000000000004,"15min":120.934},"last":1.0780000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":3.8170000000000002,"15min":3.0819999999999999},"min":{"1min":0.22500000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.01,"5min":88.088999999999999,"15min":88.088999999999999},"last":5.6130000000000004},{"interface":"front","average":{"1min":2.3399999999999999,"5min":3.9540000000000002,"15min":3.2839999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":17.655000000000001,"5min":88.004999999999995,"15min":88.004999999999995},"last":0.72599999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:23:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":3.2429999999999999,"15min":4.1260000000000003},"min":{"1min":0.217,"5min":0.192,"15min":0.192},"max":{"1min":6.2949999999999999,"5min":87.891999999999996,"15min":156.488},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.749,"5min":3.1059999999999999,"15min":4.1150000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.2850000000000001,"5min":88.147999999999996,"15min":156.50700000000001},"last":2.843}]},{"osd":4,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":3.1619999999999999,"15min":2.4489999999999998},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":7.468,"5min":88.058000000000007,"15min":88.058000000000007},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.6339999999999999,"15min":2.3399999999999999},"min":{"1min":0.29899999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.6079999999999997,"5min":49.607999999999997,"15min":49.607999999999997},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.5329999999999999,"15min":3.2280000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":5.6050000000000004,"5min":49.526000000000003,"15min":156.96100000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.2029999999999998,"15min":2.7320000000000002},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":1.99,"5min":49.356999999999999,"15min":156.97300000000001},"last":0.48199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.742,"15min":2.4670000000000001},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.30599999999999999},"max":{"1min":2.984,"5min":7.8700000000000001,"15min":137.36099999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.6110000000000002,"15min":2.9300000000000002},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.3140000000000001,"5min":88.180000000000007,"15min":136.88999999999999},"last":1.1100000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.8859999999999999,"15min":2.4020000000000001},"min":{"1min":0.40899999999999997,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":4.1790000000000003,"5min":11.298999999999999,"15min":121.324},"last":0.438},{"interface":"front","average":{"1min":1.883,"5min":2.9710000000000001,"15min":2.6120000000000001},"min":{"1min":0.435,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.8010000000000002,"5min":88.120000000000005,"15min":88.120000000000005},"last":1.1579999999999999}]}]},{"osd":2,"up_from":94,"seq":403726926077,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2660588,"kb_used_data":1988324,"kb_used_omap":81,"kb_used_meta":672110,"kb_avail":91084564,"statfs":{"total":95995035648,"available":93270593536,"internally_reserved":0,"allocated":2036043776,"data_stored":2034146847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":688240858},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.718,"15min":3.7839999999999998},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17399999999999999},"max":{"1min":5.8499999999999996,"5min":21.792000000000002,"15min":81.385999999999996},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.9930000000000001,"5min":2.0329999999999999,"15min":3.7469999999999999},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.20000000000000001},"max":{"1min":8.0800000000000001,"5min":21.603000000000002,"15min":79.453999999999994},"last":0.62}]},{"osd":1,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":5.718,"15min":4.4560000000000004},"min":{"1min":0.28100000000000003,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":6.0529999999999999,"5min":158.13399999999999,"15min":158.13399999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.0590000000000002,"5min":5.6210000000000004,"15min":4.6139999999999999},"min":{"1min":0.314,"5min":0.253,"15min":0.20799999999999999},"max":{"1min":5.8840000000000003,"5min":161.12,"15min":161.12},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:23:01 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":2.3250000000000002,"15min":2.677},"min":{"1min":0.26800000000000002,"5min":0.251,"15min":0.20599999999999999},"max":{"1min":3.6720000000000002,"5min":51.524999999999999,"15min":58.997},"last":1.855},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.7930000000000001,"15min":2.9609999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23499999999999999},"max":{"1min":6.0789999999999997,"5min":51.505000000000003,"15min":66.135000000000005},"last":0.35199999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.9119999999999999,"15min":2.1349999999999998},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":4.7889999999999997,"5min":24.814,"15min":58.798000000000002},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.2689999999999999,"15min":1.9119999999999999},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":3.629,"5min":24.021000000000001,"15min":56.616},"last":0.33100000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":3.1299999999999999,"15min":2.7429999999999999},"min":{"1min":0.42699999999999999,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":7.984,"5min":82.030000000000001,"15min":82.030000000000001},"last":0.496},{"interface":"front","average":{"1min":1.4770000000000001,"5min":3.3319999999999999,"15min":2.734},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":4.484,"5min":82.016000000000005,"15min":82.016000000000005},"last":0.441}]},{"osd":6,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.2210000000000001,"15min":2.4009999999999998},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.308},"max":{"1min":5.6269999999999998,"5min":7.3230000000000004,"15min":114.14},"last":1.6599999999999999},{"interface":"front","average":{"1min":2.476,"5min":1.9970000000000001,"15min":2.6360000000000001},"min":{"1min":0.497,"5min":0.32100000000000001,"15min":0.26800000000000002},"max":{"1min":8.0280000000000005,"5min":15.456,"15min":114.123},"last":1.738}]},{"osd":7,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.3929999999999998,"15min":2.8679999999999999},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.33700000000000002},"max":{"1min":2.6930000000000001,"5min":101.675,"15min":117.32599999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.3599999999999999,"5min":3.177,"15min":3.145},"min":{"1min":0.54200000000000004,"5min":0.34499999999999997,"15min":0.34399999999999997},"max":{"1min":10.839,"5min":101.72499999999999,"15min":117.28700000000001},"last":0.53800000000000003}]}]},{"osd":3,"up_from":99,"seq":425201762551,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3063460,"kb_used_data":2205660,"kb_used_omap":7,"kb_used_meta":857784,"kb_avail":90681692,"statfs":{"total":95995035648,"available":92858052608,"internally_reserved":0,"allocated":2258595840,"data_stored":2256704670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":878371432},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":2.3340000000000001,"15min":4.3559999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.16900000000000001},"max":{"1min":8.3279999999999994,"5min":39.718000000000004,"15min":125.765},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.6519999999999999,"5min":3.1579999999999999,"15min":4.9089999999999998},"min":{"1min":0.20899999999999999,"5min":0.185,"15min":0.185},"max":{"1min":7.5549999999999997,"5min":38.786000000000001,"15min":125.46899999999999},"last":0.25700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":9.2970000000000006,"5min":4.9009999999999998,"15min":5.3410000000000002},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":170.196,"5min":170.196,"15min":186.21299999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":8.891,"5min":4.952,"15min":5.3209999999999997},"min":{"1min":0.24099999999999999,"5min":0.221,"15min":0.193},"max":{"1min":166.953,"5min":166.953,"15min":191.78},"last":6.0419999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.665,"15min":2.9889999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.187},"max":{"1min":6.9160000000000004,"5min":43.152999999999999,"15min":127.599},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.6160000000000001,"15min":4.117},"min":{"1min":0.22900000000000001,"5min":0.223,"15min":0.214},"max":{"1min":5.4859999999999998,"5min":43.134,"15min":129.98599999999999},"last":0.39100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":11.085000000000001,"5min":3.7149999999999999,"15min":2.8130000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":211.84700000000001,"5min":211.84700000000001,"15min":211.84700000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":11.095000000000001,"5min":3.6840000000000002,"15min":2.8759999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26900000000000002},"max":{"1min":211.17500000000001,"5min":211.17500000000001,"15min":211.17500000000001},"last":0.51100000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.873,"15min":2.2240000000000002},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.6150000000000002,"5min":11.772,"15min":17.901},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.321,"15min":1.782},"min":{"1min":0.34100000000000003,"5min":0.29699999999999999,"15min":0.248},"max":{"1min":3.4830000000000001,"5min":8.4670000000000005,"15min":18.068000000000001},"last":0.52900000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.113,"15min":1.6719999999999999},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.26500000000000001},"max":{"1min":5.8689999999999998,"5min":5.8689999999999998,"15min":14.976000000000001},"last":0.495},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.7150000000000001,"15min":2.085},"min":{"1min":0.40400000000000003,"5min":0.379,"15min":0.29999999999999999},"max":{"1min":2.2450000000000001,"5min":10.071,"15min":15.622},"last":0.371}]},{"osd":7,"last update":"Mon Oct 9 04:23:13 2023","interfaces":[{"interface":"back","average":{"1min":10.938000000000001,"5min":4.2229999999999999,"15min":2.9009999999999998},"min":{"1min":0.37,"5min":0.35399999999999998,"15min":0.34399999999999997},"max":{"1min":212.62700000000001,"5min":212.62700000000001,"15min":212.62700000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":11.317,"5min":4.1799999999999997,"15min":3.0110000000000001},"min":{"1min":0.39200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":212.04599999999999,"5min":212.04599999999999,"15min":212.04599999999999},"last":1.131}]}]},{"osd":4,"up_from":0,"seq":133143986567,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724931,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2750856,"kb_used_data":1862400,"kb_used_omap":0,"kb_used_meta":888448,"kb_avail":90994296,"statfs":{"total":95995035648,"available":93178159104,"internally_reserved":0,"allocated":1907097600,"data_stored":1905861062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909770752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.2480000000000002,"15min":4.9980000000000002},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.193},"max":{"1min":24.076000000000001,"5min":63.914999999999999,"15min":151.23599999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.4860000000000002,"5min":2.585,"15min":5.383},"min":{"1min":0.27500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":22.629999999999999,"5min":61.281999999999996,"15min":151.566},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:23:04 2023","interfaces":[{"interface":"back","average":{"1min":29.010000000000002,"5min":8.2840000000000007,"15min":5.6619999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":296.23099999999999,"5min":296.23099999999999,"15min":296.23099999999999},"last":7.7969999999999997},{"interface":"front","average":{"1min":28.632000000000001,"5min":8.3859999999999992,"15min":5.8899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":295.065,"5min":295.065,"15min":295.065},"last":0.80800000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":12.301,"5min":5.3419999999999996,"15min":3.827},"min":{"1min":0.36799999999999999,"5min":0.28199999999999997,"15min":0.24399999999999999},"max":{"1min":168.779,"5min":168.779,"15min":168.779},"last":1.764},{"interface":"front","average":{"1min":11.641,"5min":5.0430000000000001,"15min":3.597},"min":{"1min":0.33000000000000002,"5min":0.27000000000000002,"15min":0.25600000000000001},"max":{"1min":168.03100000000001,"5min":168.03100000000001,"15min":168.03100000000001},"last":2.8199999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:23:04 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.2000000000000002,"15min":2.0750000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.7349999999999999,"5min":41,"15min":41},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":2.206,"15min":2.1989999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.23499999999999999},"max":{"1min":4.7510000000000003,"5min":40.988,"15min":40.988},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.1000000000000001,"15min":0.98099999999999998},"min":{"1min":0.29399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.3740000000000001,"5min":9.2579999999999991,"15min":10.162000000000001},"last":0.374},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.125,"15min":0.94299999999999995},"min":{"1min":0.31,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":2.0339999999999998,"5min":7.5890000000000004,"15min":11.297000000000001},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.054,"15min":1.385},"min":{"1min":0.30599999999999999,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":8.5350000000000001,"5min":9.4090000000000007,"15min":123.239},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.999,"15min":1.4550000000000001},"min":{"1min":0.34799999999999998,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":3.0579999999999998,"5min":6.1689999999999996,"15min":123.014},"last":0.47399999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":1.236,"15min":1.01},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":2.8300000000000001,"5min":9.3800000000000008,"15min":12.096},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.252,"15min":1.0800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.307},"max":{"1min":3.302,"5min":9.3019999999999996,"15min":9.9760000000000009},"last":0.44400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":981602304,"data_stored":981598232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1532428288,"data_stored":1532428288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2023620608,"data_stored":2023620608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2251689984,"data_stored":2251685912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2564882432,"data_stored":2564882432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1902837760,"data_stored":1902837760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2029522944,"data_stored":2029518872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1639448576,"data_stored":1639448576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:24:05.725 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:24:05.725 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-10-09T04:24:05.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:05.856 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:06.237 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:06.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:06.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:06.443 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:06.904 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:07.063 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:07.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:07.174 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:07.246 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:07.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:07 smithi181 ceph-mon[196979]: pgmap v1135: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 6394/208674 objects degraded (3.064%); 35534/208674 objects misplaced (17.028%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:24:07.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:07 smithi163 ceph-mon[221067]: pgmap v1135: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 6394/208674 objects degraded (3.064%); 35534/208674 objects misplaced (17.028%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:24:07.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:07 smithi163 ceph-mon[213841]: pgmap v1135: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 6394/208674 objects degraded (3.064%); 35534/208674 objects misplaced (17.028%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:24:07.332 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:07.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:07.440 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:07.480 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:07.643 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:07.647 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:24:07.647 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:24:07.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:24:07.647 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:24:07.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18925 10.d 10:b9680846:::benchmark_data_smithi163_523108_object18924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.977791+0000", 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.665655246, 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.077021335, 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.648 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18925 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977791+0000", 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977791+0000", 2023-10-09T04:24:07.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977790+0000", 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977829+0000", 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9242000000000001e-05 2023-10-09T04:24:07.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977851+0000", 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2330000000000001e-05 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977865+0000", 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4075e-05 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.651 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977923+0000", 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7549000000000001e-05 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.978102+0000", 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000178617 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.978216+0000", 2023-10-09T04:24:07.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114561 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.981345+0000", 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003128629 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998415+0000", 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017070535000000001 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998424+0000", 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7660000000000004e-06 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.006913+0000", 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0084885939999999 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.006922+0000", 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4809999999999995e-06 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.054717+0000", 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.047794412 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.054813+0000", 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5995000000000005e-05 2023-10-09T04:24:07.655 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18927 10.1 10:8a1d16d1:::benchmark_data_smithi163_523108_object18926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.977890+0000", 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.66555606, 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1134202800000002, 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:07.656 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18927 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977890+0000", 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977890+0000", 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977890+0000", 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977919+0000", 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9342000000000001e-05 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977927+0000", 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4499999999999998e-06 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977932+0000", 2023-10-09T04:24:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8879999999999996e-06 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.977984+0000", 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2302000000000002e-05 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.978130+0000", 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000145537 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.978268+0000", 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0001381 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.981291+0000", 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030233859999999999 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.982020+0000", 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00072912699999999999 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.989548+0000", 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0075281080000000004 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.989560+0000", 2023-10-09T04:24:07.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1456e-05 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.997795+0000", 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0082357260000001 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.997807+0000", 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.183e-05 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.091201+0000", 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0933938160000001 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.091311+0000", 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109633 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.664 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18938 10.1 10:8a3ba1ef:::benchmark_data_smithi163_523108_object18937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998601+0000", 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.644845961, 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.100801304, 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.665 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18938 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998601+0000", 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998601+0000", 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998599+0000", 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998637+0000", 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8504999999999998e-05 2023-10-09T04:24:07.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998649+0000", 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1931e-05 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998656+0000", 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7630000000000003e-06 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.668 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998695+0000", 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8798999999999999e-05 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998700+0000", 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3319999999999999e-06 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.997834+0000", 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99913390199999996 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.997842+0000", 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.627e-06 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.092832+0000", 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.09499039 2023-10-09T04:24:07.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.092953+0000", 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121316 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.093087+0000", 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013340100000000001 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.096062+0000", 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029754159999999998 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.096129+0000", 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6821000000000003e-05 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.098227+0000", 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020979950000000001 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.098264+0000", 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7236000000000001e-05 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.099402+0000", 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001137632 2023-10-09T04:24:07.673 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18939 10.d 10:b25a92e7:::benchmark_data_smithi163_523108_object18938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998669+0000", 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.644777069, 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.066478912, 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.674 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18939 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998669+0000", 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998669+0000", 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998669+0000", 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998705+0000", 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6369999999999999e-05 2023-10-09T04:24:07.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998713+0000", 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8229999999999995e-06 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998718+0000", 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1710000000000001e-06 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.677 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999212+0000", 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00049371699999999996 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999221+0000", 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8340000000000005e-06 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.006951+0000", 2023-10-09T04:24:07.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0077303179999999 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.006959+0000", 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4939999999999997e-06 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.055602+0000", 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0486431890000001 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.055765+0000", 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000163549 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.055957+0000", 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000191416 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.057875+0000", 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001917801 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.059542+0000", 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0016669219999999999 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.064218+0000", 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046768979999999996 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.064287+0000", 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8332999999999998e-05 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.065148+0000", 2023-10-09T04:24:07.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086156500000000001 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18935 10.b 10.f449325b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998843+0000", 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.644603499, 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0116172450000001, 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:07.683 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18935 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998843+0000", 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998843+0000", 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998842+0000", 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999056+0000", 2023-10-09T04:24:07.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021340500000000001 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999064+0000", 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1480000000000006e-06 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999068+0000", 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0940000000000001e-06 2023-10-09T04:24:07.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999709+0000", 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064072700000000001 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999715+0000", 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.392e-06 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.010450+0000", 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.010734977 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.010460+0000", 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0071999999999999e-05 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18943 10.d 10:b491eeed:::benchmark_data_smithi163_523108_object18942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:26.008902+0000", 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.634544568, 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0581308389999999, 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18943 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.689 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.008902+0000", 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.008902+0000", 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.008901+0000", 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.008957+0000", 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6178000000000003e-05 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.008973+0000", 2023-10-09T04:24:07.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6016999999999999e-05 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.008980+0000", 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1520000000000003e-06 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009035+0000", 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5099999999999998e-05 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009046+0000", 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0998999999999999e-05 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.007112+0000", 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99806552500000001 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.007121+0000", 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5969999999999998e-06 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.056862+0000", 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.049740398 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.056982+0000", 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120048 2023-10-09T04:24:07.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.057145+0000", 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016394399999999999 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.058144+0000", 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00099804400000000001 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.695 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.059548+0000", 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014048050000000001 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.066155+0000", 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066065200000000003 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.066216+0000", 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1136000000000006e-05 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.067033+0000", 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000816799 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18945 10.b 10.49234edb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:26.009053+0000", 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.634393934, 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0016895050000001, 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18945 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.698 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009053+0000", 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009053+0000", 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009052+0000", 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009085+0000", 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2678999999999997e-05 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009094+0000", 2023-10-09T04:24:07.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3209999999999994e-06 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009099+0000", 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8199999999999996e-06 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009140+0000", 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1130000000000001e-05 2023-10-09T04:24:07.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.009147+0000", 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3259999999999998e-06 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.010737+0000", 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0015900200000001 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.010742+0000", 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6399999999999996e-06 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18948 10.b 10.743ca7cb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:26.014622+0000", 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.628824127, 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99613612100000004, 2023-10-09T04:24:07.703 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18948 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014622+0000", 2023-10-09T04:24:07.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014622+0000", 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014622+0000", 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.705 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014662+0000", 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0426999999999998e-05 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014671+0000", 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3389999999999993e-06 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014674+0000", 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4529999999999998e-06 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014713+0000", 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9365e-05 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014720+0000", 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.241e-06 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.010755+0000", 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99603533700000002 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.010759+0000", 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5899999999999999e-06 2023-10-09T04:24:07.708 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18949 10.4 10:2ee1aa1f:::benchmark_data_smithi163_523108_object18948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:26.014682+0000", 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 101.62876425100001, 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.065725729, 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.709 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18949 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014682+0000", 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014682+0000", 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014682+0000", 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014708+0000", 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6625999999999999e-05 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014714+0000", 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2079999999999997e-06 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014717+0000", 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7140000000000002e-06 2023-10-09T04:24:07.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014749+0000", 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1485000000000001e-05 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014754+0000", 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1939999999999997e-06 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.006467+0000", 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99171312499999997 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:27.006479+0000", 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2318e-05 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.714 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.074854+0000", 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0683750030000001 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.075232+0000", 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037772599999999999 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:24:07.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.075329+0000", 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7109999999999997e-05 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.077901+0000", 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025717650000000002 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.078874+0000", 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00097333900000000004 2023-10-09T04:24:07.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.079918+0000", 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001043476 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.079955+0000", 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7756000000000002e-05 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.080408+0000", 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00045254099999999999 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38639 10.b 10:d6cd64ec:::benchmark_data_smithi163_523108_object38638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.088633+0000", 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.554813316999997, 2023-10-09T04:24:07.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99945987000000003, 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38639 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088633+0000", 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088633+0000", 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088633+0000", 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088716+0000", 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3393000000000006e-05 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088726+0000", 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0138e-05 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088732+0000", 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5450000000000003e-06 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088792+0000", 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0294000000000002e-05 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.088800+0000", 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4929999999999998e-06 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.079747+0000", 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99094658999999996 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.079872+0000", 2023-10-09T04:24:07.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012476699999999999 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.080021+0000", 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014910699999999999 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085409+0000", 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053880660000000004 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085511+0000", 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010228100000000001 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088037+0000", 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0025258910000000002 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088069+0000", 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1633e-05 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088093+0000", 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4298e-05 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.726 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38691 10.b 10:d57a1ca2:::benchmark_data_smithi163_523108_object38690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.133686+0000", 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.509761002000005, 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95696628900000003, 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38691 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.133686+0000", 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.133686+0000", 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.133685+0000", 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.133760+0000", 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5574999999999994e-05 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.133771+0000", 2023-10-09T04:24:07.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0747999999999999e-05 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.133776+0000", 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0810000000000001e-06 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.134014+0000", 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000237858 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.730 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.134027+0000", 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3093e-05 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.081337+0000", 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94730953900000003 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.081432+0000", 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5006999999999993e-05 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.081626+0000", 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000194148 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085535+0000", 2023-10-09T04:24:07.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0039089459999999999 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085542+0000", 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6510000000000004e-06 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090557+0000", 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005015594 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090612+0000", 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4780000000000001e-05 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090652+0000", 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9876999999999998e-05 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.734 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38710 10.b 10:ddb795a1:::benchmark_data_smithi163_523108_object38709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.154344+0000", 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.489102281000001, 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93650758700000003, 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.735 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38710 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154344+0000", 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154344+0000", 2023-10-09T04:24:07.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154343+0000", 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154369+0000", 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.5890000000000001e-05 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154380+0000", 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1471000000000001e-05 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154386+0000", 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1870000000000002e-06 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154489+0000", 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102629 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.154498+0000", 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7749999999999998e-06 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.082960+0000", 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92846175900000005 2023-10-09T04:24:07.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.083054+0000", 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4802999999999993e-05 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.083195+0000", 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000140166 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085547+0000", 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002352579 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085569+0000", 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1605999999999999e-05 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090767+0000", 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051986059999999997 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090817+0000", 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9625e-05 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090852+0000", 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4799999999999999e-05 2023-10-09T04:24:07.742 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38719 10.b 10:def489e4:::benchmark_data_smithi163_523108_object38718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.160788+0000", 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.482658638000004, 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93020705000000004, 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.743 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38719 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160788+0000", 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160788+0000", 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160787+0000", 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160854+0000", 2023-10-09T04:24:07.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7224000000000006e-05 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160865+0000", 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0739e-05 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160871+0000", 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0780000000000001e-06 2023-10-09T04:24:07.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160937+0000", 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6161999999999994e-05 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160947+0000", 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9529999999999994e-06 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.084149+0000", 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.923201944 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.084257+0000", 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108374 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.084359+0000", 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000101815 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.087979+0000", 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036202059999999999 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.087987+0000", 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6350000000000006e-06 2023-10-09T04:24:07.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090905+0000", 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002918328 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090955+0000", 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0405999999999998e-05 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090995+0000", 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9499000000000002e-05 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38720 10.b 10:d67399cd:::benchmark_data_smithi163_523108_object38719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.160885+0000", 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.482561450999995, 2023-10-09T04:24:07.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93023093400000001, 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38720 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160885+0000", 2023-10-09T04:24:07.752 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160885+0000", 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160884+0000", 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160919+0000", 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4193999999999998e-05 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160930+0000", 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1214e-05 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160935+0000", 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3539999999999999e-06 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160987+0000", 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1375999999999997e-05 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.160995+0000", 2023-10-09T04:24:07.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9529999999999999e-06 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085612+0000", 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.92461774100000005 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085730+0000", 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011757299999999999 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.085881+0000", 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015107400000000001 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088011+0000", 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021296869999999999 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088104+0000", 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2872000000000001e-05 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091041+0000", 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029378450000000001 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091079+0000", 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7960999999999998e-05 2023-10-09T04:24:07.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091116+0000", 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6692e-05 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38786 10.b 10:d5aa623e:::benchmark_data_smithi163_523108_object38785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.759 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.211392+0000", 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.432054328000007, 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87981499600000002, 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38786 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.760 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211392+0000", 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211392+0000", 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211391+0000", 2023-10-09T04:24:07.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211457+0000", 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5916999999999999e-05 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211468+0000", 2023-10-09T04:24:07.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1643e-05 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211475+0000", 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2360000000000001e-06 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211541+0000", 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6081999999999995e-05 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.211549+0000", 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1899999999999995e-06 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.086965+0000", 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87541615399999995 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.087030+0000", 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5463999999999995e-05 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.087117+0000", 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6761999999999994e-05 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090501+0000", 2023-10-09T04:24:07.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033837009999999998 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090718+0000", 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021749200000000001 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091148+0000", 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042974099999999998 2023-10-09T04:24:07.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091175+0000", 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7322e-05 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091207+0000", 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1714000000000002e-05 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.767 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38808 10.b 10:de736122:::benchmark_data_smithi163_523108_object38807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.231454+0000", 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.411992165000001, 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86284239100000004, 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.768 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38808 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231454+0000", 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231454+0000", 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231453+0000", 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231515+0000", 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1619999999999996e-05 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.770 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231534+0000", 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9069999999999999e-05 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231539+0000", 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8280000000000002e-06 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231600+0000", 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1443000000000001e-05 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.231610+0000", 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.696e-06 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088122+0000", 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85651197700000004 2023-10-09T04:24:07.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088216+0000", 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3838999999999994e-05 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.088374+0000", 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015870599999999999 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.090731+0000", 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023561049999999998 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091234+0000", 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050329499999999996 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094213+0000", 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0029788700000000002 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094260+0000", 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7375e-05 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094297+0000", 2023-10-09T04:24:07.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6680000000000001e-05 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38839 10.b 10:d0eaec96:::benchmark_data_smithi163_523108_object38838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.252747+0000", 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.390699544, 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84174139100000001, 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.776 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38839 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252747+0000", 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252747+0000", 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252746+0000", 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252781+0000", 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4295000000000003e-05 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252788+0000", 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2590000000000003e-06 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252792+0000", 2023-10-09T04:24:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3619999999999999e-06 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252842+0000", 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9382000000000003e-05 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.252850+0000", 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2689999999999999e-06 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.089327+0000", 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.83647727599999999 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.089439+0000", 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111767 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.089636+0000", 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00019667900000000001 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094158+0000", 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045220429999999999 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094169+0000", 2023-10-09T04:24:07.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0718e-05 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094398+0000", 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022906599999999999 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094448+0000", 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0479000000000002e-05 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094488+0000", 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0302000000000001e-05 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38854 10.b 10:de3ff64a:::benchmark_data_smithi163_523108_object38853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.267988+0000", 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.375458825999999, 2023-10-09T04:24:07.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82661561900000002, 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38854 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.267988+0000", 2023-10-09T04:24:07.785 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.267988+0000", 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.267986+0000", 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.268066+0000", 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0037000000000006e-05 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.268081+0000", 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5475e-05 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.268088+0000", 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9890000000000003e-06 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.268141+0000", 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2590000000000003e-05 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.268149+0000", 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4020000000000002e-06 2023-10-09T04:24:07.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091256+0000", 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82310700000000003 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091379+0000", 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122917 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.789 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.091530+0000", 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015039799999999999 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094176+0000", 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0026463540000000001 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094353+0000", 2023-10-09T04:24:07.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017710499999999999 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094529+0000", 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017561399999999999 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094569+0000", 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0524e-05 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094603+0000", 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4128999999999998e-05 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38858 10.b 10:d86630ae:::benchmark_data_smithi163_523108_object38857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.270747+0000", 2023-10-09T04:24:07.792 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.372699416000003, 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82403050300000003, 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38858 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270747+0000", 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270747+0000", 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270746+0000", 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:07.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270805+0000", 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8578000000000001e-05 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270816+0000", 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1018e-05 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270820+0000", 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5190000000000003e-06 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270875+0000", 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4481000000000002e-05 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.270881+0000", 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9869999999999996e-06 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.092691+0000", 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82181025900000004 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.092816+0000", 2023-10-09T04:24:07.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012457200000000001 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.092941+0000", 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012526200000000001 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094365+0000", 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014235280000000001 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094650+0000", 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000285476 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094700+0000", 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9891000000000001e-05 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.799 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094744+0000", 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4085999999999998e-05 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094778+0000", 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3599000000000001e-05 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:38879 10.b 10:d918c4e2:::benchmark_data_smithi163_523108_object38878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:46.289636+0000", 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 81.353810636999995, 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.816648861, 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 38879 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:07.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289636+0000", 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289636+0000", 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289635+0000", 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289669+0000", 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3487e-05 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:07.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289676+0000", 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8460000000000002e-06 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289680+0000", 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1269999999999996e-06 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289741+0000", 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1618000000000005e-05 2023-10-09T04:24:07.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:46.289750+0000", 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2800000000000003e-06 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094809+0000", 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80505932199999997 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.094928+0000", 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118987 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.095084+0000", 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015575699999999999 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.100745+0000", 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056616499999999998 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.106123+0000", 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053770909999999996 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.106190+0000", 2023-10-09T04:24:07.807 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7525000000000001e-05 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.106243+0000", 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2913e-05 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:47.106285+0000", 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1758999999999998e-05 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.808 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.809 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.809 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:07.809 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:07.809 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:24:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:07.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:07.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:07.954 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:08.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:08.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:08.070 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:24:08.126 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:08.159 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:08.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1252260871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:08 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/208674 objects degraded (3.064%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:08 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1252260871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/208674 objects degraded (3.064%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:08.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1252260871' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:08.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:08.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/208674 objects degraded (3.064%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:08.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:08.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:08 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:08.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:24:08.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:08.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:08.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:08.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:24:08.961 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:09.021 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:09.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:09.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[221067]: pgmap v1136: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 6394/206304 objects degraded (3.099%); 34384/206304 objects misplaced (16.667%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:24:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[221067]: failed to encode map e425 with expected crc 2023-10-09T04:24:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[221067]: failed to encode map e425 with expected crc 2023-10-09T04:24:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[221067]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[213841]: pgmap v1136: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 6394/206304 objects degraded (3.099%); 34384/206304 objects misplaced (16.667%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[213841]: failed to encode map e425 with expected crc 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[213841]: failed to encode map e425 with expected crc 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:24:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:09 smithi163 ceph-mon[213841]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T04:24:09.157 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:09.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:09 smithi181 ceph-mon[196979]: pgmap v1136: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 6394/206304 objects degraded (3.099%); 34384/206304 objects misplaced (16.667%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:24:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:09 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:09 smithi181 ceph-mon[196979]: failed to encode map e425 with expected crc 2023-10-09T04:24:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:09 smithi181 ceph-mon[196979]: failed to encode map e425 with expected crc 2023-10-09T04:24:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:09 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:24:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:09 smithi181 ceph-mon[196979]: osdmap e425: 8 total, 8 up, 7 in 2023-10-09T04:24:09.376 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:09.416 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:09.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:09.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:09.615 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:09.850 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:09.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:09.900 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:09.962 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:10.033 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:10.263 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:10.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:10.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:10.406 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:10.453 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:10.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:10.563 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:10.629 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:10.751 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:10.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:24:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:24:10] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.43.0" 2023-10-09T04:24:10.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:10.890 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:10.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:10.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:10.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:11.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:11.063 INFO:teuthology.orchestra.run.smithi163.stderr:set pool 10 pg_num to 26 2023-10-09T04:24:11.095 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:11.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: pgmap v1138: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 287 op/s; 6394/204492 objects degraded (3.127%); 33373/204492 objects misplaced (16.320%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:24:11.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: failed to encode map e426 with expected crc 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: failed to encode map e426 with expected crc 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2781593694' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:11.109 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: pgmap v1138: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 287 op/s; 6394/204492 objects degraded (3.127%); 33373/204492 objects misplaced (16.320%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: failed to encode map e426 with expected crc 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: failed to encode map e426 with expected crc 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2781593694' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: pgmap v1138: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 287 op/s; 6394/204492 objects degraded (3.127%); 33373/204492 objects misplaced (16.320%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: failed to encode map e426 with expected crc 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: osdmap e426: 8 total, 8 up, 7 in 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: failed to encode map e426 with expected crc 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2781593694' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:24:11.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:11.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:11.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:11.369 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:24:11.369 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:24:11.369 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:24:11.370 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:24:11.508 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:24:11.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:11.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:11.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:12.056 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:12.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:12.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:12.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:12.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[213841]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[213841]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2781593694' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[213841]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[213841]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[221067]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[221067]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2781593694' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[221067]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T04:24:12.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[221067]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.280 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:12 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:24:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:12 smithi181 ceph-mon[196979]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:12 smithi181 ceph-mon[196979]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2781593694' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:24:12.317 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:12 smithi181 ceph-mon[196979]: osdmap e427: 8 total, 8 up, 7 in 2023-10-09T04:24:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:12 smithi181 ceph-mon[196979]: failed to encode map e427 with expected crc 2023-10-09T04:24:12.318 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:12 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:24:12.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:24:12.369 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:12.518 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:12.613 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:12.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:12.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:12.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:12.760 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:12.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:12.801 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:12.853 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:12.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:12.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:12.989 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:13.011 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:13.031 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:13.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:13.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:13.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:13.050 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:13.073 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:13.092 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:13.122 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:13.139 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:13.142 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:13.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.165 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:24:13.167 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:13.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: pgmap v1141: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 6394/202560 objects degraded (3.157%); 33177/202560 objects misplaced (16.379%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: failed to encode map e428 with expected crc 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: failed to encode map e428 with expected crc 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6394/202560 objects degraded (3.157%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: failed to encode map e429 with expected crc 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: 10.8 continuing backfill to osd.0 from (0'0,427'5073] MIN to 427'5073 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: 10.5 continuing backfill to osd.0 from (0'0,427'5179] MIN to 427'5179 2023-10-09T04:24:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2644300070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:13.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: pgmap v1141: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 6394/202560 objects degraded (3.157%); 33177/202560 objects misplaced (16.379%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:24:13.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: failed to encode map e428 with expected crc 2023-10-09T04:24:13.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:24:13.296 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: failed to encode map e428 with expected crc 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6394/202560 objects degraded (3.157%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: failed to encode map e429 with expected crc 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: 10.8 continuing backfill to osd.0 from (0'0,427'5073] MIN to 427'5073 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: 10.5 continuing backfill to osd.0 from (0'0,427'5179] MIN to 427'5179 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2644300070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: pgmap v1141: 119 pgs: 3 active+recovering+undersized+remapped, 107 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 6394/202560 objects degraded (3.157%); 33177/202560 objects misplaced (16.379%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: failed to encode map e428 with expected crc 2023-10-09T04:24:13.297 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: osdmap e428: 8 total, 8 up, 7 in 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: failed to encode map e428 with expected crc 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6394/202560 objects degraded (3.157%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: osdmap e429: 8 total, 8 up, 7 in 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: failed to encode map e429 with expected crc 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: 10.8 continuing backfill to osd.0 from (0'0,427'5073] MIN to 427'5073 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: 10.5 continuing backfill to osd.0 from (0'0,427'5179] MIN to 427'5179 2023-10-09T04:24:13.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2644300070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:13.317 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:13.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:13.422 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:13.449 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:13.493 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:13.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:13.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:13.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:13.630 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:13.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:13.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:13.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:13.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:13.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:13.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:13.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:14.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.069 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: 10.6 continuing backfill to osd.0 from (0'0,427'4942] MIN to 427'4942 2023-10-09T04:24:14.070 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: 10.2 continuing backfill to osd.0 from (0'0,427'5104] MIN to 427'5104 2023-10-09T04:24:14.070 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T04:24:14.070 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[221067]: failed to encode map e430 with expected crc 2023-10-09T04:24:14.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:14.098 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: 10.6 continuing backfill to osd.0 from (0'0,427'4942] MIN to 427'4942 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: 10.2 continuing backfill to osd.0 from (0'0,427'5104] MIN to 427'5104 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T04:24:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:14 smithi163 ceph-mon[213841]: failed to encode map e430 with expected crc 2023-10-09T04:24:14.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:14.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:14.451 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:14.483 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:14.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: failed to encode map e429 with expected crc 2023-10-09T04:24:14.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: 10.6 continuing backfill to osd.0 from (0'0,427'4942] MIN to 427'4942 2023-10-09T04:24:14.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: 10.2 continuing backfill to osd.0 from (0'0,427'5104] MIN to 427'5104 2023-10-09T04:24:14.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: osdmap e430: 8 total, 8 up, 7 in 2023-10-09T04:24:14.508 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:14 smithi181 ceph-mon[196979]: failed to encode map e430 with expected crc 2023-10-09T04:24:14.538 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:14.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:14.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:14.681 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:14.792 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:14.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:14.895 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:14.927 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:14.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:14.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:14.998 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:24:14.998 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: pgmap v1145: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 9 unknown, 2 active+recovering+undersized+remapped, 108 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 6394/201717 objects degraded (3.170%); 30538/201717 objects misplaced (15.139%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: 10.18 continuing backfill to osd.0 from (0'0,427'5073] MIN to 427'5073 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: 10.15 continuing backfill to osd.0 from (0'0,427'5179] MIN to 427'5179 2023-10-09T04:24:15.071 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: pgmap v1145: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 9 unknown, 2 active+recovering+undersized+remapped, 108 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 6394/201717 objects degraded (3.170%); 30538/201717 objects misplaced (15.139%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: 10.18 continuing backfill to osd.0 from (0'0,427'5073] MIN to 427'5073 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: 10.15 continuing backfill to osd.0 from (0'0,427'5179] MIN to 427'5179 2023-10-09T04:24:15.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:15 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:15.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:15.106 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:15.151 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:15.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:15.256 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:15.288 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:15.341 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:15.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:15.445 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:15.474 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: failed to encode map e430 with expected crc 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: pgmap v1145: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 9 unknown, 2 active+recovering+undersized+remapped, 108 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 6394/201717 objects degraded (3.170%); 30538/201717 objects misplaced (15.139%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: osdmap e431: 8 total, 8 up, 7 in 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: 10.18 continuing backfill to osd.0 from (0'0,427'5073] MIN to 427'5073 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: 10.15 continuing backfill to osd.0 from (0'0,427'5179] MIN to 427'5179 2023-10-09T04:24:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:15 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:15.522 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:15.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:15.626 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:15.654 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:15.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:15.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:15.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:15.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:15.831 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:15.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:15.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:15.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:16.017 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:16.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:16.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:16.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.078 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: 10.12 continuing backfill to osd.0 from (0'0,427'5104] MIN to 427'5104 2023-10-09T04:24:16.079 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: 10.16 continuing backfill to osd.0 from (0'0,427'4942] MIN to 427'4942 2023-10-09T04:24:16.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3075051728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:16.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:16.088 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:24:16.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:16.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:16.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:16.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:16.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: 10.12 continuing backfill to osd.0 from (0'0,427'5104] MIN to 427'5104 2023-10-09T04:24:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: 10.16 continuing backfill to osd.0 from (0'0,427'4942] MIN to 427'4942 2023-10-09T04:24:16.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3075051728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:16.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:16 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:16.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:16.394 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:16.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:24:16.480 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:16.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: 10.12 continuing backfill to osd.0 from (0'0,427'5104] MIN to 427'5104 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: failed to encode map e431 with expected crc 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: 10.16 continuing backfill to osd.0 from (0'0,427'4942] MIN to 427'4942 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3075051728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:16 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:16.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:16.619 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:16.701 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:16.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:16.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:16.791 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:24:16.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:16.879 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:16.963 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:16.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:16.990 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:24:16.990 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:24:16.990 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:24:16.991 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T04:24:16.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T04:24:17.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:17.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:17 smithi181 ceph-mon[196979]: pgmap v1147: 129 pgs: 1 active+undersized+degraded+remapped, 3 remapped+peering, 1 active+remapped+backfilling, 5 unknown, 2 active+recovering+undersized+remapped, 109 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 6257/200886 objects degraded (3.115%); 26259/200886 objects misplaced (13.072%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:24:17.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:17 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:24:17.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:17 smithi181 ceph-mon[196979]: failed to encode map e432 with expected crc 2023-10-09T04:24:17.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:17 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:17.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:17 smithi181 ceph-mon[196979]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T04:24:17.086 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:17 smithi181 ceph-mon[196979]: failed to encode map e432 with expected crc 2023-10-09T04:24:17.101 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:17.209 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:17.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:17.253 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:17.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[213841]: pgmap v1147: 129 pgs: 1 active+undersized+degraded+remapped, 3 remapped+peering, 1 active+remapped+backfilling, 5 unknown, 2 active+recovering+undersized+remapped, 109 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 6257/200886 objects degraded (3.115%); 26259/200886 objects misplaced (13.072%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:24:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:24:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[213841]: failed to encode map e432 with expected crc 2023-10-09T04:24:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[213841]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T04:24:17.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[213841]: failed to encode map e432 with expected crc 2023-10-09T04:24:17.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[221067]: pgmap v1147: 129 pgs: 1 active+undersized+degraded+remapped, 3 remapped+peering, 1 active+remapped+backfilling, 5 unknown, 2 active+recovering+undersized+remapped, 109 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 6257/200886 objects degraded (3.115%); 26259/200886 objects misplaced (13.072%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:24:17.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:24:17.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[221067]: failed to encode map e432 with expected crc 2023-10-09T04:24:17.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:17.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[221067]: osdmap e432: 8 total, 8 up, 7 in 2023-10-09T04:24:17.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:17 smithi163 ceph-mon[221067]: failed to encode map e432 with expected crc 2023-10-09T04:24:17.337 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:17.366 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:17.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:17.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:17.571 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:17.602 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:17.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:17.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:17.911 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:17.941 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:17.993 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:17.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:18.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:18.130 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:18.185 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:18.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:18.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:18.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:18.335 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:18.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:18 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 6257/200886 objects degraded (3.115%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:18.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:18 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 6257/200886 objects degraded (3.115%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:18 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 6257/200886 objects degraded (3.115%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:18.569 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:18.608 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:18.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:18.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:18.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:18.759 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:19.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:19.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:19.054 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:24:19.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:19 smithi163 ceph-mon[213841]: pgmap v1149: 129 pgs: 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 3 remapped+peering, 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 6423/199746 objects degraded (3.216%); 24322/199746 objects misplaced (12.176%); 19 MiB/s, 300 objects/s recovering 2023-10-09T04:24:19.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:19 smithi163 ceph-mon[221067]: pgmap v1149: 129 pgs: 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 3 remapped+peering, 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 6423/199746 objects degraded (3.216%); 24322/199746 objects misplaced (12.176%); 19 MiB/s, 300 objects/s recovering 2023-10-09T04:24:19.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:19.185 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:19.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:19 smithi181 ceph-mon[196979]: pgmap v1149: 129 pgs: 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 3 remapped+peering, 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 6423/199746 objects degraded (3.216%); 24322/199746 objects misplaced (12.176%); 19 MiB/s, 300 objects/s recovering 2023-10-09T04:24:19.660 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:19.660 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:19.661 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:19.661 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:19.661 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:19.661 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:19.683 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:19.717 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.721 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:24:19.900 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:19.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:19.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:20.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131725 8.3 8:c2b90b78:::benchmark_data_smithi163_404719_object41141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.58570612699998, 2023-10-09T04:24:20.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1240774949999999, 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131725 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:24:20.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474536+0000", 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474538+0000", 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4329999999999998e-06 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474549+0000", 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0927000000000001e-05 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474553+0000", 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8800000000000001e-06 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474594+0000", 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1332999999999999e-05 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.474602+0000", 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:24:20.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468816+0000", 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99421445799999997 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468827+0000", 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1093e-05 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.586018+0000", 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.117190543 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.588171+0000", 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021532040000000001 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.588321+0000", 2023-10-09T04:24:20.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015008999999999999 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598261+0000", 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099394879999999998 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598299+0000", 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8522000000000001e-05 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598361+0000", 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1239000000000001e-05 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598457+0000", 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6027999999999996e-05 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598531+0000", 2023-10-09T04:24:20.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4678999999999994e-05 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598613+0000", 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2201999999999997e-05 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131729 8.3 8:c6b15366:::benchmark_data_smithi163_404719_object41145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.073 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.575621103, 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1141904230000002, 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131729 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484621+0000", 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484620+0000", 2023-10-09T04:24:20.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484626+0000", 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0870000000000003e-06 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484639+0000", 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3502e-05 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484647+0000", 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484692+0000", 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5667999999999997e-05 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484702+0000", 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5850000000000006e-06 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468853+0000", 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98415053600000002 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468860+0000", 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1189999999999999e-06 2023-10-09T04:24:20.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591501+0000", 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1226411199999999 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591590+0000", 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9573000000000005e-05 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.591667+0000", 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7033000000000006e-05 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598305+0000", 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066379239999999999 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598404+0000", 2023-10-09T04:24:20.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9100999999999999e-05 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598650+0000", 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00024551400000000001 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598726+0000", 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6175999999999997e-05 2023-10-09T04:24:20.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598754+0000", 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7898999999999998e-05 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598811+0000", 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7370999999999997e-05 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.082 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131731 8.3 8:c392b46b:::benchmark_data_smithi163_404719_object41147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.57554950600002, 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1142496469999998, 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.083 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131731 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484693+0000", 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484692+0000", 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484695+0000", 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7630000000000001e-06 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484701+0000", 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1909999999999997e-06 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484705+0000", 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9700000000000001e-06 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484752+0000", 2023-10-09T04:24:20.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6964e-05 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.484761+0000", 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5590000000000006e-06 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468882+0000", 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.98412151699999995 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468889+0000", 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8260000000000004e-06 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594688+0000", 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1257991060000001 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594798+0000", 2023-10-09T04:24:20.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010983 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.594900+0000", 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102451 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598656+0000", 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0037554839999999999 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598683+0000", 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6998000000000001e-05 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598703+0000", 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9925999999999999e-05 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598843+0000", 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014050299999999999 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598872+0000", 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8965000000000001e-05 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598942+0000", 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0003e-05 2023-10-09T04:24:20.091 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131738 8.3 8:cc1c5c5f:::benchmark_data_smithi163_404719_object41154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.56492510499999, 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1194174960000001, 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131738 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:24:20.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495317+0000", 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495319+0000", 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9379999999999999e-06 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495324+0000", 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1200000000000001e-06 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495326+0000", 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4820000000000001e-06 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495378+0000", 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1295000000000003e-05 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.495384+0000", 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4320000000000004e-06 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468935+0000", 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97355068700000003 2023-10-09T04:24:20.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468944+0000", 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.816e-06 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.598966+0000", 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.130022938 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.599049+0000", 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2161000000000002e-05 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.599124+0000", 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4906000000000004e-05 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614430+0000", 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015306491 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614476+0000", 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6394999999999999e-05 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614536+0000", 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9799000000000001e-05 2023-10-09T04:24:20.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614566+0000", 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0008000000000001e-05 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614623+0000", 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6487999999999999e-05 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614735+0000", 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111797 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131739 8.3 8:ccd4c823:::benchmark_data_smithi163_404719_object41155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.54422318799999, 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.098935633, 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.101 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131739 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516019+0000", 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516018+0000", 2023-10-09T04:24:20.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516024+0000", 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.421e-06 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516038+0000", 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3558e-05 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516044+0000", 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4400000000000002e-06 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516099+0000", 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4361999999999999e-05 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.516110+0000", 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1661000000000001e-05 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468972+0000", 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95286181000000003 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.468979+0000", 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6800000000000004e-06 2023-10-09T04:24:20.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602170+0000", 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.133191676 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602304+0000", 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013313999999999999 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.602429+0000", 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012518499999999999 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614498+0000", 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012068972000000001 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614780+0000", 2023-10-09T04:24:20.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000281978 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614813+0000", 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3337000000000003e-05 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614843+0000", 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0256999999999998e-05 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614872+0000", 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.898e-05 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614955+0000", 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2304000000000003e-05 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131750 8.3 8:caad7466:::benchmark_data_smithi163_404719_object41166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.52741784, 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0823976559999999, 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131750 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.111 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532824+0000", 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532826+0000", 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.779e-06 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532829+0000", 2023-10-09T04:24:20.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4290000000000001e-06 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532832+0000", 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3879999999999998e-06 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532884+0000", 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2642999999999997e-05 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.532892+0000", 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6290000000000001e-06 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469000+0000", 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93610853400000005 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469007+0000", 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3670000000000002e-06 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.605999+0000", 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.136991761 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.606144+0000", 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014551200000000001 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.606301+0000", 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000156458 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614505+0000", 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082041619999999992 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614978+0000", 2023-10-09T04:24:20.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00047337600000000002 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615009+0000", 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0454999999999999e-05 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615040+0000", 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1758000000000003e-05 2023-10-09T04:24:20.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615086+0000", 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5494999999999997e-05 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615222+0000", 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013613699999999999 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.119 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131757 8.3 8:cde86114:::benchmark_data_smithi163_404719_object41173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.50913495200001, 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0643897739999999, 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.120 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131757 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551107+0000", 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551105+0000", 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551114+0000", 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2609999999999993e-06 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551130+0000", 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5934e-05 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551137+0000", 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1019999999999999e-06 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551191+0000", 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4138000000000002e-05 2023-10-09T04:24:20.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.551202+0000", 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.164e-05 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469028+0000", 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.91782591899999999 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469035+0000", 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2580000000000001e-06 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610215+0000", 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1411807170000001 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610340+0000", 2023-10-09T04:24:20.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012489900000000001 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.610434+0000", 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4154999999999996e-05 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.614745+0000", 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043102599999999998 2023-10-09T04:24:20.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615262+0000", 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00051697899999999998 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615294+0000", 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2675000000000001e-05 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615328+0000", 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3495999999999999e-05 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615374+0000", 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6514000000000002e-05 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615497+0000", 2023-10-09T04:24:20.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122583 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131773 8.3 8:cc396e7f:::benchmark_data_smithi163_404719_object41189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 476.47581163199999, 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0561476569999999, 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.129 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131773 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584430+0000", 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584429+0000", 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584435+0000", 2023-10-09T04:24:20.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4090000000000002e-06 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584448+0000", 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.361e-05 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584456+0000", 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6389999999999992e-06 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584515+0000", 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8638999999999999e-05 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:23.584525+0000", 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0314e-05 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469134+0000", 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88460910100000001 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.469142+0000", 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0639999999999994e-06 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615534+0000", 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1463917850000001 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615640+0000", 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010655800000000001 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.615751+0000", 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011056 2023-10-09T04:24:20.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627128+0000", 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011377376 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631239+0000", 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0041103110000000002 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631277+0000", 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8482000000000002e-05 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640379+0000", 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0091021850000000005 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640436+0000", 2023-10-09T04:24:20.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6545000000000002e-05 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640578+0000", 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142261 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131846 8.3 8:c021821e:::benchmark_data_smithi163_404719_object41262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.138 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 475.27187799500001, 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.852533604, 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131846 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788364+0000", 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788363+0000", 2023-10-09T04:24:20.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788368+0000", 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5560000000000001e-06 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788375+0000", 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1559999999999998e-06 2023-10-09T04:24:20.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788379+0000", 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0430000000000001e-06 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788435+0000", 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5859000000000002e-05 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.788441+0000", 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.156e-06 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619638+0000", 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.831196571 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619758+0000", 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120253 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.619848+0000", 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9886e-05 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631195+0000", 2023-10-09T04:24:20.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011347183 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631309+0000", 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113992 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640695+0000", 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093864970000000006 2023-10-09T04:24:20.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640734+0000", 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8285999999999999e-05 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640782+0000", 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8615999999999997e-05 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640898+0000", 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011548900000000001 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131856 8.3 8:c67caa71:::benchmark_data_smithi163_404719_object41272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 475.24072003700002, 2023-10-09T04:24:20.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82161764000000004, 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131856 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:24:20.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819522+0000", 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819524+0000", 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1560000000000002e-06 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819529+0000", 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2159999999999997e-06 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819532+0000", 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.79e-06 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819581+0000", 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8748999999999997e-05 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.819589+0000", 2023-10-09T04:24:20.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8879999999999997e-06 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623021+0000", 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.80343243799999997 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623126+0000", 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105138 2023-10-09T04:24:20.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.623256+0000", 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129383 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631246+0000", 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079899900000000006 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631345+0000", 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8989999999999999e-05 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640971+0000", 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0096260070000000007 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641005+0000", 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4100000000000002e-05 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641043+0000", 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8825000000000002e-05 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641140+0000", 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6241000000000002e-05 2023-10-09T04:24:20.155 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131865 8.3 8:c397b806:::benchmark_data_smithi163_404719_object41281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 475.21881431399999, 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79990704499999998, 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.156 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131865 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841428+0000", 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841427+0000", 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841432+0000", 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7679999999999997e-06 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841444+0000", 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1588e-05 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841449+0000", 2023-10-09T04:24:20.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0769999999999998e-06 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841505+0000", 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6144000000000003e-05 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.841514+0000", 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6100000000000006e-06 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627156+0000", 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78564268000000004 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627290+0000", 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013397600000000001 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.627401+0000", 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011101 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640330+0000", 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012928057 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640641+0000", 2023-10-09T04:24:20.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031165099999999997 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641173+0000", 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00053144399999999995 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641202+0000", 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9847000000000001e-05 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641241+0000", 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8713e-05 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641335+0000", 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3648999999999997e-05 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.164 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131881 8.3 8:cd41d762:::benchmark_data_smithi163_404719_object41297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 475.18213012799998, 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.79379251699999998, 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.165 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131881 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878112+0000", 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878111+0000", 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878118+0000", 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.956e-06 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878131+0000", 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3141e-05 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878138+0000", 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8109999999999997e-06 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878215+0000", 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7330000000000007e-05 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.878228+0000", 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3209e-05 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631369+0000", 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75314052899999995 2023-10-09T04:24:20.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631495+0000", 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125823 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.631615+0000", 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120013 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640655+0000", 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0090400859999999993 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640937+0000", 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028247100000000001 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649876+0000", 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089392050000000004 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671738+0000", 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021861968999999998 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671788+0000", 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9493999999999997e-05 2023-10-09T04:24:20.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671905+0000", 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011675699999999999 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131884 8.3 8:c4f20fc8:::benchmark_data_smithi163_404719_object41300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:24:20.173 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 475.17692640600001, 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78877238199999999, 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131884 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883316+0000", 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883314+0000", 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:24:20.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883321+0000", 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0639999999999996e-06 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883334+0000", 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2595000000000001e-05 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883340+0000", 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3409999999999999e-06 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883406+0000", 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6045999999999998e-05 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:24.883416+0000", 2023-10-09T04:24:20.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0387000000000001e-05 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635386+0000", 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75196957900000005 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635512+0000", 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012591300000000001 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.635633+0000", 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121377 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.640702+0000", 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050688790000000001 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641370+0000", 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00066799299999999997 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649915+0000", 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085445550000000006 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671963+0000", 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022048742 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.671998+0000", 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4585000000000002e-05 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672088+0000", 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0163999999999999e-05 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131932 8.3 8:c8cbc02f:::benchmark_data_smithi163_404719_object41348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 475.05210967099998, 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.66412386599999995, 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131932 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.182 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008132+0000", 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008131+0000", 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008140+0000", 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.901e-06 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008151+0000", 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1178999999999999e-05 2023-10-09T04:24:20.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008158+0000", 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3949999999999998e-06 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008225+0000", 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7541999999999998e-05 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.008237+0000", 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1732999999999999e-05 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641390+0000", 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.633153454 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641535+0000", 2023-10-09T04:24:20.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014420899999999999 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.641667+0000", 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205199999999999 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649884+0000", 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082173850000000007 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649944+0000", 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0229000000000002e-05 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.649976+0000", 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1479000000000001e-05 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672141+0000", 2023-10-09T04:24:20.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022165625000000001 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672185+0000", 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3373e-05 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672256+0000", 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1606000000000005e-05 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.189 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.35475.0:131959 8.3 8:c13e43ce:::benchmark_data_smithi163_404719_object41375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 474.98339862, 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.59562893100000003, 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.35475", 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:24:20.190 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 131959 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076843+0000", 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076848+0000", 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076858+0000", 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0603e-05 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076863+0000", 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4159999999999997e-06 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076923+0000", 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9760999999999999e-05 2023-10-09T04:24:20.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.076933+0000", 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0849999999999999e-05 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645730+0000", 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.56879615900000002 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645860+0000", 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013080899999999999 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2,6", 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.645984+0000", 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012326399999999999 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658944+0000", 2023-10-09T04:24:20.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012960704999999999 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658951+0000", 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0469999999999996e-06 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.658992+0000", 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1028000000000001e-05 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672327+0000", 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013334278999999999 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672372+0000", 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4783000000000001e-05 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:16:25.672472+0000", 2023-10-09T04:24:20.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010089 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:110800 9.7 9:edeb27ba:::benchmark_data_smithi163_464650_object12074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:00.967321+0000", 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 259.09292106300001, 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.133645886, 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:20.198 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 110800 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967321+0000", 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967321+0000", 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967320+0000", 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967327+0000", 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7200000000000006e-06 2023-10-09T04:24:20.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967341+0000", 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4019000000000001e-05 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967348+0000", 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1210000000000001e-06 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967524+0000", 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017569300000000001 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967643+0000", 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119367 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967759+0000", 2023-10-09T04:24:20.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115884 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.971245+0000", 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034856560000000002 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.972618+0000", 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013727710000000001 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.986601+0000", 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013982827 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.986611+0000", 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0759e-05 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:02.030984+0000", 2023-10-09T04:24:20.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0443723739999999 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:02.030995+0000", 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1212e-05 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:03.100902+0000", 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.069907159 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:03.100967+0000", 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4851000000000006e-05 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18942 10.6 10:65a48f0d:::benchmark_data_smithi163_523108_object18941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998930+0000", 2023-10-09T04:24:20.206 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 114.06131231400001, 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1092201259999999, 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18942 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998930+0000", 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998930+0000", 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998928+0000", 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998967+0000", 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8402000000000003e-05 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998981+0000", 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4372999999999999e-05 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998989+0000", 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6279999999999996e-06 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999285+0000", 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029607 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999296+0000", 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1276e-05 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992447+0000", 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99315083699999995 2023-10-09T04:24:20.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992457+0000", 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7880000000000001e-06 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.099919+0000", 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.107462452 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.100039+0000", 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120226 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.100200+0000", 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016061299999999999 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.103011+0000", 2023-10-09T04:24:20.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002811502 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.107047+0000", 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040354299999999996 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.107095+0000", 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8204000000000001e-05 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.107151+0000", 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5983999999999998e-05 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.108150+0000", 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0009989720000000001 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18934 10.a 10:5bb0a4f3:::benchmark_data_smithi163_523108_object18933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 114.061245825, 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.089428222, 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18934 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999025+0000", 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9019000000000001e-05 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999033+0000", 2023-10-09T04:24:20.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4149999999999999e-06 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999038+0000", 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8799999999999999e-06 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999631+0000", 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00059334500000000001 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999639+0000", 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2660000000000004e-06 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992278+0000", 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99263969799999996 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992293+0000", 2023-10-09T04:24:20.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4462e-05 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.083758+0000", 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.091465635 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.084269+0000", 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0005109 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.084451+0000", 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000181521 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.085405+0000", 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00095371799999999999 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.085693+0000", 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028888200000000002 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.087633+0000", 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019396140000000001 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.087675+0000", 2023-10-09T04:24:20.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2209000000000002e-05 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.088425+0000", 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074923399999999999 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18946 10.6 10:6fc4e677:::benchmark_data_smithi163_523108_object18945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:24:20.225 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:26.014450+0000", 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 114.04579165600001, 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1049869910000001, 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18946 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014450+0000", 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014450+0000", 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014449+0000", 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014536+0000", 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7131000000000003e-05 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014549+0000", 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2559e-05 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014554+0000", 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7010000000000001e-06 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014605+0000", 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0887999999999998e-05 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014615+0000", 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7159999999999998e-06 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.994586+0000", 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.979970966 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.994596+0000", 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0206e-05 2023-10-09T04:24:20.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.101902+0000", 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.107305389 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.102016+0000", 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011417900000000001 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:24:20.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.102161+0000", 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014525900000000001 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.105251+0000", 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030898129999999998 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.110748+0000", 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005497325 2023-10-09T04:24:20.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.118450+0000", 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077021490000000001 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.118525+0000", 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4603000000000004e-05 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.119437+0000", 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00091260099999999997 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:57337 10.6 10:69db66ce:::benchmark_data_smithi163_523108_object57336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:06.473218+0000", 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 73.587024099000004, 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0186636200000001, 2023-10-09T04:24:20.234 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 57337 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473218+0000", 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473218+0000", 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473216+0000", 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473271+0000", 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4496000000000001e-05 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473282+0000", 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0693000000000001e-05 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473286+0000", 2023-10-09T04:24:20.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3499999999999999e-06 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473336+0000", 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0385e-05 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:06.473343+0000", 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2890000000000003e-06 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.472387+0000", 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99904401899999995 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.472509+0000", 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000122218 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.472666+0000", 2023-10-09T04:24:20.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015699600000000001 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.474828+0000", 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021626800000000002 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.476333+0000", 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0015045099999999999 2023-10-09T04:24:20.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.486661+0000", 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010327512 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.491713+0000", 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050524899999999998 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.491770+0000", 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7119e-05 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:07.491882+0000", 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011151500000000001 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:20.242 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:20.243 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:24:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:20.244 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:20.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:20.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:20.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:20.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:20.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/114056717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:20.318 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[213841]: pgmap v1150: 129 pgs: 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 6423/198303 objects degraded (3.239%); 30203/198303 objects misplaced (15.231%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:24:20.345 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:20.386 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/114056717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:20 smithi181 ceph-mon[196979]: pgmap v1150: 129 pgs: 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 6423/198303 objects degraded (3.239%); 30203/198303 objects misplaced (15.231%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:24:20.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:20.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:20.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/114056717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:20.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:20 smithi163 ceph-mon[221067]: pgmap v1150: 129 pgs: 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 6423/198303 objects degraded (3.239%); 30203/198303 objects misplaced (15.231%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:24:20.625 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:20.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:20.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:20.765 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:20.819 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:20.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:20.872 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:20.923 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:20.984 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:24:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:24:20] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-10-09T04:24:20.990 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:21.097 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:21.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:21.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:21.202 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:21.246 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:21.444 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T04:24:21.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:21.531 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4002778756' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:24:21.531 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:21 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:24:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:21.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:21.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4002778756' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:24:21.532 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:21 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:24:21.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:24:21.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:21.665 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4002778756' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:24:21.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:21 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:24:21.983 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:22.120 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:22.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:22.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:22.252 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:22.425 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:22 smithi181 ceph-mon[196979]: failed to encode map e433 with expected crc 2023-10-09T04:24:22.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:22.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:22.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:22.617 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:22.672 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:22.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:22.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:24:22.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[221067]: failed to encode map e433 with expected crc 2023-10-09T04:24:22.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T04:24:22.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[221067]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T04:24:22.731 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[221067]: pgmap v1152: 129 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 252 op/s; 4778/195630 objects degraded (2.442%); 31263/195630 objects misplaced (15.981%); 17 MiB/s, 265 objects/s recovering 2023-10-09T04:24:22.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[213841]: failed to encode map e433 with expected crc 2023-10-09T04:24:22.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T04:24:22.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[213841]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T04:24:22.732 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:22 smithi163 ceph-mon[213841]: pgmap v1152: 129 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 252 op/s; 4778/195630 objects degraded (2.442%); 31263/195630 objects misplaced (15.981%); 17 MiB/s, 265 objects/s recovering 2023-10-09T04:24:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:22 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T04:24:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:22 smithi181 ceph-mon[196979]: osdmap e433: 8 total, 8 up, 7 in 2023-10-09T04:24:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:22 smithi181 ceph-mon[196979]: pgmap v1152: 129 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 252 op/s; 4778/195630 objects degraded (2.442%); 31263/195630 objects misplaced (15.981%); 17 MiB/s, 265 objects/s recovering 2023-10-09T04:24:22.776 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:22.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:22.965 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:22.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:23.070 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:23.136 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:23.239 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:23.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:23.346 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:23.373 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:23.374 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:23.424 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:23.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:23.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:23.430 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.430 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.430 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: osdmap e434: 8 total, 8 up, 7 in 2023-10-09T04:24:23.430 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: 10.15 continuing backfill to osd.0 from (0'0,432'5253] MIN to 432'5253 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: 10.5 continuing backfill to osd.0 from (0'0,432'5253] MIN to 432'5253 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4778/195630 objects degraded (2.442%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: osdmap e434: 8 total, 8 up, 7 in 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.431 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.432 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.432 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: 10.15 continuing backfill to osd.0 from (0'0,432'5253] MIN to 432'5253 2023-10-09T04:24:23.432 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: 10.5 continuing backfill to osd.0 from (0'0,432'5253] MIN to 432'5253 2023-10-09T04:24:23.432 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:23 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4778/195630 objects degraded (2.442%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:23.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:23.562 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:23.612 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:23.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:23.716 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:23.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: osdmap e434: 8 total, 8 up, 7 in 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: failed to encode map e434 with expected crc 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: 10.15 continuing backfill to osd.0 from (0'0,432'5253] MIN to 432'5253 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: 10.5 continuing backfill to osd.0 from (0'0,432'5253] MIN to 432'5253 2023-10-09T04:24:23.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:23 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4778/195630 objects degraded (2.442%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:23.753 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:23.985 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:23.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:24.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:24.128 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:24.195 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:24.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:24.302 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:24.331 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:24.375 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:24.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:24.436 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:24:24.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:24.506 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:24.563 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:24.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2023636192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:24:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:24:24.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:24 smithi163 ceph-mon[213841]: pgmap v1154: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 4778/194454 objects degraded (2.457%); 29708/194454 objects misplaced (15.278%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:24:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2023636192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:24:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:24:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:24 smithi163 ceph-mon[221067]: pgmap v1154: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 4778/194454 objects degraded (2.457%); 29708/194454 objects misplaced (15.278%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:24:24.588 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:24:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:24:24.588 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:24:24.588 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:24:24.616 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2023636192' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:24:24.617 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:24:24.617 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:24 smithi181 ceph-mon[196979]: pgmap v1154: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 243 op/s; 4778/194454 objects degraded (2.457%); 29708/194454 objects misplaced (15.278%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:24:24.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:24.735 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:24.898 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:24.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:24.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:25.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:25.044 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:25.534 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:25.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:25.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[221067]: failed to encode map e435 with expected crc 2023-10-09T04:24:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[221067]: failed to encode map e435 with expected crc 2023-10-09T04:24:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:24:25.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[221067]: osdmap e435: 8 total, 8 up, 7 in 2023-10-09T04:24:25.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:25.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[213841]: failed to encode map e435 with expected crc 2023-10-09T04:24:25.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[213841]: failed to encode map e435 with expected crc 2023-10-09T04:24:25.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:24:25.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:25 smithi163 ceph-mon[213841]: osdmap e435: 8 total, 8 up, 7 in 2023-10-09T04:24:25.641 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:25.676 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:25.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:25 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:25.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:25 smithi181 ceph-mon[196979]: failed to encode map e435 with expected crc 2023-10-09T04:24:25.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:25 smithi181 ceph-mon[196979]: failed to encode map e435 with expected crc 2023-10-09T04:24:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:25 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:24:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:25 smithi181 ceph-mon[196979]: osdmap e435: 8 total, 8 up, 7 in 2023-10-09T04:24:25.913 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:25.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:25.973 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:25.976 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:24:26.009 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:26.017 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:26.039 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:26.051 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:26.076 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:26.103 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:26.120 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:26.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:26.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:26.163 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:26.198 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.200 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.201 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:24:26.227 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:26.281 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:26.399 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:26.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:26.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:26.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:26.566 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:26.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/921805555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:26.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:26 smithi163 ceph-mon[221067]: pgmap v1156: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 298 op/s; 4724/192936 objects degraded (2.448%); 29212/192936 objects misplaced (15.141%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:24:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/921805555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:26.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:26 smithi163 ceph-mon[213841]: pgmap v1156: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 298 op/s; 4724/192936 objects degraded (2.448%); 29212/192936 objects misplaced (15.141%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:24:26.593 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:26.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:26.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:26.740 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/921805555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:26.740 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:26 smithi181 ceph-mon[196979]: pgmap v1156: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 111 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 298 op/s; 4724/192936 objects degraded (2.448%); 29212/192936 objects misplaced (15.141%); 14 MiB/s, 228 objects/s recovering 2023-10-09T04:24:26.742 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:26.767 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:26.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:26.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:26.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:26.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:26.979 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:27.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:27.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:27.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:27.130 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:27.170 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:27.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:27.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:27.332 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:27.368 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:27.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:27.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:27.536 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:27.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:24:27.567 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:27.669 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:24:27.669 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:24:27.669 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:24:27.669 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:24:27.715 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:27.717 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:27 smithi181 ceph-mon[196979]: failed to encode map e436 with expected crc 2023-10-09T04:24:27.717 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:27 smithi181 ceph-mon[196979]: osdmap e436: 8 total, 8 up, 7 in 2023-10-09T04:24:27.717 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:27 smithi181 ceph-mon[196979]: failed to encode map e436 with expected crc 2023-10-09T04:24:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:27.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:27.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:27 smithi163 ceph-mon[213841]: failed to encode map e436 with expected crc 2023-10-09T04:24:27.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:27 smithi163 ceph-mon[213841]: osdmap e436: 8 total, 8 up, 7 in 2023-10-09T04:24:27.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:27 smithi163 ceph-mon[213841]: failed to encode map e436 with expected crc 2023-10-09T04:24:27.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:27 smithi163 ceph-mon[221067]: failed to encode map e436 with expected crc 2023-10-09T04:24:27.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:27 smithi163 ceph-mon[221067]: osdmap e436: 8 total, 8 up, 7 in 2023-10-09T04:24:27.739 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:27 smithi163 ceph-mon[221067]: failed to encode map e436 with expected crc 2023-10-09T04:24:27.819 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:27.876 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:27.972 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:27.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:27.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:28.081 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:28.181 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:28.248 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:24:28.249 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:24:28.293 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:28.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:28.376 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:28.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:28.429 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:28.474 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:28.617 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:28.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:28.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: osdmap e437: 8 total, 8 up, 7 in 2023-10-09T04:24:28.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.704 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.705 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4724/192936 objects degraded (2.448%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:28.705 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[213841]: pgmap v1159: 129 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.1 KiB/s rd, 13 KiB/s wr, 362 op/s; 4627/189702 objects degraded (2.439%); 27561/189702 objects misplaced (14.529%); 22 MiB/s, 352 objects/s recovering 2023-10-09T04:24:28.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: osdmap e437: 8 total, 8 up, 7 in 2023-10-09T04:24:28.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.705 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4724/192936 objects degraded (2.448%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:28.706 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:28 smithi163 ceph-mon[221067]: pgmap v1159: 129 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.1 KiB/s rd, 13 KiB/s wr, 362 op/s; 4627/189702 objects degraded (2.439%); 27561/189702 objects misplaced (14.529%); 22 MiB/s, 352 objects/s recovering 2023-10-09T04:24:28.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:28.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: osdmap e437: 8 total, 8 up, 7 in 2023-10-09T04:24:28.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.756 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: failed to encode map e437 with expected crc 2023-10-09T04:24:28.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4724/192936 objects degraded (2.448%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:28.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:28 smithi181 ceph-mon[196979]: pgmap v1159: 129 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 5.1 KiB/s rd, 13 KiB/s wr, 362 op/s; 4627/189702 objects degraded (2.439%); 27561/189702 objects misplaced (14.529%); 22 MiB/s, 352 objects/s recovering 2023-10-09T04:24:28.759 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:28.811 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:28.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:28.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:28.943 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:29.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:29.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:24:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:29.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:29.178 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:24:29.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:29.299 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:29.346 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:29.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:29.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:29.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:29.482 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:29.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:29.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:29.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:29.604 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:29.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:29.730 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:30.910 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:24:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:24:30] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-10-09T04:24:31.211 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:24:31.211 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:24:31.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:31 smithi181 ceph-mon[196979]: pgmap v1160: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 342 op/s; 4627/188358 objects degraded (2.456%); 26739/188358 objects misplaced (14.196%); 16 MiB/s, 258 objects/s recovering 2023-10-09T04:24:31.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:31 smithi163 ceph-mon[213841]: pgmap v1160: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 342 op/s; 4627/188358 objects degraded (2.456%); 26739/188358 objects misplaced (14.196%); 16 MiB/s, 258 objects/s recovering 2023-10-09T04:24:31.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:31 smithi163 ceph-mon[221067]: pgmap v1160: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 342 op/s; 4627/188358 objects degraded (2.456%); 26739/188358 objects misplaced (14.196%); 16 MiB/s, 258 objects/s recovering 2023-10-09T04:24:31.635 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:31.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:31.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:31.739 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:31.776 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:32.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:24:32.009 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:32.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:32.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:32.114 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:32.144 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:32.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:32.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:32.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:32.339 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:32.341 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:32.389 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:32.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:32.412 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:32.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:32.521 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:32.566 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:32.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:32.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:32.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:32.772 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:32.878 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:32.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:32.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:33.021 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:33.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:33.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:33.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[213841]: pgmap v1161: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 3.7 KiB/s rd, 9.7 KiB/s wr, 400 op/s; 4461/185493 objects degraded (2.405%); 26323/185493 objects misplaced (14.191%); 16 MiB/s, 258 objects/s recovering 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4461/185493 objects degraded (2.405%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3778834011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[221067]: pgmap v1161: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 3.7 KiB/s rd, 9.7 KiB/s wr, 400 op/s; 4461/185493 objects degraded (2.405%); 26323/185493 objects misplaced (14.191%); 16 MiB/s, 258 objects/s recovering 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4461/185493 objects degraded (2.405%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3778834011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:33.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:33 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:33.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:33.178 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:24:33.185 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:24:33.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:33 smithi181 ceph-mon[196979]: pgmap v1161: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 3.7 KiB/s rd, 9.7 KiB/s wr, 400 op/s; 4461/185493 objects degraded (2.405%); 26323/185493 objects misplaced (14.191%); 16 MiB/s, 258 objects/s recovering 2023-10-09T04:24:33.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:33 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4461/185493 objects degraded (2.405%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:33.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3778834011' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:33.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:33 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:33.200 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:33.251 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:33.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:33.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:33.383 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:33.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:33.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:33.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:33.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:33.646 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:24:33.669 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:33.759 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:33.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:33.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:33.932 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:34.066 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:24:34.067 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:24:34.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:34 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:34.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:34 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:34.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:34 smithi181 ceph-mon[196979]: osdmap e438: 8 total, 8 up, 7 in 2023-10-09T04:24:34.095 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:34 smithi181 ceph-mon[196979]: failed to encode map e438 with expected crc 2023-10-09T04:24:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[221067]: osdmap e438: 8 total, 8 up, 7 in 2023-10-09T04:24:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[221067]: failed to encode map e438 with expected crc 2023-10-09T04:24:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[213841]: osdmap e438: 8 total, 8 up, 7 in 2023-10-09T04:24:34.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:34 smithi163 ceph-mon[213841]: failed to encode map e438 with expected crc 2023-10-09T04:24:34.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:24:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: from='client.56038 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:24:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: pgmap v1163: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 4461/184023 objects degraded (2.424%); 26206/184023 objects misplaced (14.241%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:24:35.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:35.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: from='client.56038 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:24:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: pgmap v1163: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 4461/184023 objects degraded (2.424%); 26206/184023 objects misplaced (14.241%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: from='client.56038 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: failed to encode map e438 with expected crc 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: pgmap v1163: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 4461/184023 objects degraded (2.424%); 26206/184023 objects misplaced (14.241%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:24:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:35.363 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:36.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:36.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:36.203 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1161,"stamp":"2023-10-09T04:24:31.944205+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4051173605,"num_objects":61831,"num_object_clones":0,"num_object_copies":185493,"num_objects_missing_on_primary":3442,"num_objects_missing":3442,"num_objects_degraded":4461,"num_objects_misplaced":26323,"num_objects_unfound":0,"num_objects_dirty":61831,"num_whiteouts":0,"num_read":105,"num_read_kb":196,"num_write":86333,"num_write_kb":4737375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24995,"num_bytes_recovered":1634265173,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142046,"ondisk_log_size":142046,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":392,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16110004,"kb_used_data":11043644,"kb_used_omap":126,"kb_used_meta":5066177,"kb_avail":640106060,"statfs":{"total":671965249536,"available":655468605440,"internally_reserved":0,"allocated":11308691456,"data_stored":11296899475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":5187765883},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-195738142,"num_objects":-2987,"num_object_clones":0,"num_object_copies":-8961,"num_objects_missing_on_primary":-320,"num_objects_missing":-320,"num_objects_degraded":-317,"num_objects_misplaced":-3385,"num_objects_unfound":0,"num_objects_dirty":-2987,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":2994,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1944,"num_bytes_recovered":127455654,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.513658"},"pg_stats":[{"pgid":"10.13","version":"437'5204","reported_seq":5289,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:30.899203+0000","last_change":"2023-10-09T04:24:22.451557+0000","last_active":"2023-10-09T04:24:30.899203+0000","last_peered":"2023-10-09T04:24:30.899203+0000","last_clean":"2023-10-09T04:24:30.899203+0000","last_became_active":"2023-10-09T04:24:22.446790+0000","last_became_peered":"2023-10-09T04:24:22.446790+0000","last_unstale":"2023-10-09T04:24:30.899203+0000","last_undegraded":"2023-10-09T04:24:30.899203+0000","last_fullsized":"2023-10-09T04:24:30.899203+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5204,"log_dups_size":0,"ondisk_log_size":5204,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124583936,"num_objects":1901,"num_object_clones":0,"num_object_copies":5703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2685,"num_write_kb":146752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"80'161","reported_seq":561,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519612+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:24:26.519612+0000","last_peered":"2023-10-09T04:24:26.519612+0000","last_clean":"2023-10-09T04:24:26.519612+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:24:26.519612+0000","last_undegraded":"2023-10-09T04:24:26.519612+0000","last_fullsized":"2023-10-09T04:24:26.519612+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":203,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.464082+0000","last_change":"2023-10-09T04:24:22.452403+0000","last_active":"2023-10-09T04:24:26.464082+0000","last_peered":"2023-10-09T04:24:26.464082+0000","last_clean":"2023-10-09T04:24:26.464082+0000","last_became_active":"2023-10-09T04:24:22.452080+0000","last_became_peered":"2023-10-09T04:24:22.452080+0000","last_unstale":"2023-10-09T04:24:26.464082+0000","last_undegraded":"2023-10-09T04:24:26.464082+0000","last_fullsized":"2023-10-09T04:24:26.464082+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:34.882631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":314,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443613+0000","last_change":"2023-10-09T04:23:22.368166+0000","last_active":"2023-10-09T04:24:26.443613+0000","last_peered":"2023-10-09T04:24:26.443613+0000","last_clean":"2023-10-09T04:24:26.443613+0000","last_became_active":"2023-10-09T04:23:22.367876+0000","last_became_peered":"2023-10-09T04:23:22.367876+0000","last_unstale":"2023-10-09T04:24:26.443613+0000","last_undegraded":"2023-10-09T04:24:26.443613+0000","last_fullsized":"2023-10-09T04:24:26.443613+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:39.268330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"437'5207","reported_seq":13578,"reported_epoch":437,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:24:30.901802+0000","last_change":"2023-10-09T04:24:12.720669+0000","last_active":"2023-10-09T04:24:30.901802+0000","last_peered":"2023-10-09T04:24:30.901802+0000","last_clean":"2023-10-09T04:23:21.324155+0000","last_became_active":"2023-10-09T04:24:12.711657+0000","last_became_peered":"2023-10-09T04:24:12.711657+0000","last_unstale":"2023-10-09T04:24:30.901802+0000","last_undegraded":"2023-10-09T04:24:12.697781+0000","last_fullsized":"2023-10-09T04:24:30.901802+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125796352,"num_objects":1920,"num_object_clones":0,"num_object_copies":5760,"num_objects_missing_on_primary":99,"num_objects_missing":99,"num_objects_degraded":99,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2689,"num_write_kb":147456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2078,"num_bytes_recovered":136183808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":99},{"shards":"1,6,7","objects":1821}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":614,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520150+0000","last_change":"2023-10-09T04:23:22.379164+0000","last_active":"2023-10-09T04:24:26.520150+0000","last_peered":"2023-10-09T04:24:26.520150+0000","last_clean":"2023-10-09T04:24:26.520150+0000","last_became_active":"2023-10-09T04:23:22.378812+0000","last_became_peered":"2023-10-09T04:23:22.378812+0000","last_unstale":"2023-10-09T04:24:26.520150+0000","last_undegraded":"2023-10-09T04:24:26.520150+0000","last_fullsized":"2023-10-09T04:24:26.520150+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:58.326150+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":434,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443269+0000","last_change":"2023-10-09T04:23:37.010935+0000","last_active":"2023-10-09T04:24:26.443269+0000","last_peered":"2023-10-09T04:24:26.443269+0000","last_clean":"2023-10-09T04:24:26.443269+0000","last_became_active":"2023-10-09T04:23:37.010628+0000","last_became_peered":"2023-10-09T04:23:37.010628+0000","last_unstale":"2023-10-09T04:24:26.443269+0000","last_undegraded":"2023-10-09T04:24:26.443269+0000","last_fullsized":"2023-10-09T04:24:26.443269+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:28.587185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":40,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.464107+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:24:26.464107+0000","last_peered":"2023-10-09T04:24:26.464107+0000","last_clean":"2023-10-09T04:24:26.464107+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:24:26.464107+0000","last_undegraded":"2023-10-09T04:24:26.464107+0000","last_fullsized":"2023-10-09T04:24:26.464107+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"437'5490","reported_seq":8276,"reported_epoch":437,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:24:30.952024+0000","last_change":"2023-10-09T04:24:12.738961+0000","last_active":"2023-10-09T04:24:30.952024+0000","last_peered":"2023-10-09T04:24:30.952024+0000","last_clean":"2023-10-09T04:22:25.940554+0000","last_became_active":"2023-10-09T04:24:12.719423+0000","last_became_peered":"2023-10-09T04:24:12.719423+0000","last_unstale":"2023-10-09T04:24:30.952024+0000","last_undegraded":"2023-10-09T04:24:12.713840+0000","last_fullsized":"2023-10-09T04:24:30.952024+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130940928,"num_objects":1998,"num_object_clones":0,"num_object_copies":5994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1523,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2829,"num_write_kb":154464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":755,"num_bytes_recovered":49479680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"0,1,7","objects":717},{"shards":"0,4,7","objects":599},{"shards":"1,4,7","objects":682}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":440,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519709+0000","last_change":"2023-10-09T04:23:25.382713+0000","last_active":"2023-10-09T04:24:26.519709+0000","last_peered":"2023-10-09T04:24:26.519709+0000","last_clean":"2023-10-09T04:24:26.519709+0000","last_became_active":"2023-10-09T04:23:25.382298+0000","last_became_peered":"2023-10-09T04:23:25.382298+0000","last_unstale":"2023-10-09T04:24:26.519709+0000","last_undegraded":"2023-10-09T04:24:26.519709+0000","last_fullsized":"2023-10-09T04:24:26.519709+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":40,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.462643+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:24:26.462643+0000","last_peered":"2023-10-09T04:24:26.462643+0000","last_clean":"2023-10-09T04:24:26.462643+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:24:26.462643+0000","last_undegraded":"2023-10-09T04:24:26.462643+0000","last_fullsized":"2023-10-09T04:24:26.462643+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":390,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520402+0000","last_change":"2023-10-09T04:22:13.634165+0000","last_active":"2023-10-09T04:24:26.520402+0000","last_peered":"2023-10-09T04:24:26.520402+0000","last_clean":"2023-10-09T04:24:26.520402+0000","last_became_active":"2023-10-09T04:22:13.632882+0000","last_became_peered":"2023-10-09T04:22:13.632882+0000","last_unstale":"2023-10-09T04:24:26.520402+0000","last_undegraded":"2023-10-09T04:24:26.520402+0000","last_fullsized":"2023-10-09T04:24:26.520402+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:28.354644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":836,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466382+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:24:26.466382+0000","last_peered":"2023-10-09T04:24:26.466382+0000","last_clean":"2023-10-09T04:24:26.466382+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:24:26.466382+0000","last_undegraded":"2023-10-09T04:24:26.466382+0000","last_fullsized":"2023-10-09T04:24:26.466382+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":40,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463745+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:24:26.463745+0000","last_peered":"2023-10-09T04:24:26.463745+0000","last_clean":"2023-10-09T04:24:26.463745+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:24:26.463745+0000","last_undegraded":"2023-10-09T04:24:26.463745+0000","last_fullsized":"2023-10-09T04:24:26.463745+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":157,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443647+0000","last_change":"2023-10-09T04:22:13.630073+0000","last_active":"2023-10-09T04:24:26.443647+0000","last_peered":"2023-10-09T04:24:26.443647+0000","last_clean":"2023-10-09T04:24:26.443647+0000","last_became_active":"2023-10-09T04:22:13.629719+0000","last_became_peered":"2023-10-09T04:22:13.629719+0000","last_unstale":"2023-10-09T04:24:26.443647+0000","last_undegraded":"2023-10-09T04:24:26.443647+0000","last_fullsized":"2023-10-09T04:24:26.443647+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:03.686524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":677,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466665+0000","last_change":"2023-10-09T04:24:22.451141+0000","last_active":"2023-10-09T04:24:26.466665+0000","last_peered":"2023-10-09T04:24:26.466665+0000","last_clean":"2023-10-09T04:24:26.466665+0000","last_became_active":"2023-10-09T04:24:22.450872+0000","last_became_peered":"2023-10-09T04:24:22.450872+0000","last_unstale":"2023-10-09T04:24:26.466665+0000","last_undegraded":"2023-10-09T04:24:26.466665+0000","last_fullsized":"2023-10-09T04:24:26.466665+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":396,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.521252+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:24:26.521252+0000","last_peered":"2023-10-09T04:24:26.521252+0000","last_clean":"2023-10-09T04:24:26.521252+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:24:26.521252+0000","last_undegraded":"2023-10-09T04:24:26.521252+0000","last_fullsized":"2023-10-09T04:24:26.521252+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443291+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:24:26.443291+0000","last_peered":"2023-10-09T04:24:26.443291+0000","last_clean":"2023-10-09T04:24:26.443291+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:24:26.443291+0000","last_undegraded":"2023-10-09T04:24:26.443291+0000","last_fullsized":"2023-10-09T04:24:26.443291+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":641,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520359+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:24:26.520359+0000","last_peered":"2023-10-09T04:24:26.520359+0000","last_clean":"2023-10-09T04:24:26.520359+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:24:26.520359+0000","last_undegraded":"2023-10-09T04:24:26.520359+0000","last_fullsized":"2023-10-09T04:24:26.520359+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":495,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466791+0000","last_change":"2023-10-09T04:23:22.464995+0000","last_active":"2023-10-09T04:24:26.466791+0000","last_peered":"2023-10-09T04:24:26.466791+0000","last_clean":"2023-10-09T04:24:26.466791+0000","last_became_active":"2023-10-09T04:23:22.377288+0000","last_became_peered":"2023-10-09T04:23:22.377288+0000","last_unstale":"2023-10-09T04:24:26.466791+0000","last_undegraded":"2023-10-09T04:24:26.466791+0000","last_fullsized":"2023-10-09T04:24:26.466791+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:06.312840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.450886+0000","last_change":"2023-10-09T04:22:13.629778+0000","last_active":"2023-10-09T04:24:26.450886+0000","last_peered":"2023-10-09T04:24:26.450886+0000","last_clean":"2023-10-09T04:24:26.450886+0000","last_became_active":"2023-10-09T04:22:13.629473+0000","last_became_peered":"2023-10-09T04:22:13.629473+0000","last_unstale":"2023-10-09T04:24:26.450886+0000","last_undegraded":"2023-10-09T04:24:26.450886+0000","last_fullsized":"2023-10-09T04:24:26.450886+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:42.753957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":614,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.461684+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:24:26.461684+0000","last_peered":"2023-10-09T04:24:26.461684+0000","last_clean":"2023-10-09T04:24:26.461684+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:24:26.461684+0000","last_undegraded":"2023-10-09T04:24:26.461684+0000","last_fullsized":"2023-10-09T04:24:26.461684+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":124,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466535+0000","last_change":"2023-10-09T04:23:22.462142+0000","last_active":"2023-10-09T04:24:26.466535+0000","last_peered":"2023-10-09T04:24:26.466535+0000","last_clean":"2023-10-09T04:24:26.466535+0000","last_became_active":"2023-10-09T04:23:22.376278+0000","last_became_peered":"2023-10-09T04:23:22.376278+0000","last_unstale":"2023-10-09T04:24:26.466535+0000","last_undegraded":"2023-10-09T04:24:26.466535+0000","last_fullsized":"2023-10-09T04:24:26.466535+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:36.530321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":32,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519835+0000","last_change":"2023-10-09T04:23:22.376371+0000","last_active":"2023-10-09T04:24:26.519835+0000","last_peered":"2023-10-09T04:24:26.519835+0000","last_clean":"2023-10-09T04:24:26.519835+0000","last_became_active":"2023-10-09T04:23:22.376005+0000","last_became_peered":"2023-10-09T04:23:22.376005+0000","last_unstale":"2023-10-09T04:24:26.519835+0000","last_undegraded":"2023-10-09T04:24:26.519835+0000","last_fullsized":"2023-10-09T04:24:26.519835+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:49.927677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":455,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463335+0000","last_change":"2023-10-09T04:23:08.629976+0000","last_active":"2023-10-09T04:24:26.463335+0000","last_peered":"2023-10-09T04:24:26.463335+0000","last_clean":"2023-10-09T04:24:26.463335+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:24:26.463335+0000","last_undegraded":"2023-10-09T04:24:26.463335+0000","last_fullsized":"2023-10-09T04:24:26.463335+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":446,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.465756+0000","last_change":"2023-10-09T04:23:22.455486+0000","last_active":"2023-10-09T04:24:26.465756+0000","last_peered":"2023-10-09T04:24:26.465756+0000","last_clean":"2023-10-09T04:24:26.465756+0000","last_became_active":"2023-10-09T04:23:22.373684+0000","last_became_peered":"2023-10-09T04:23:22.373684+0000","last_unstale":"2023-10-09T04:24:26.465756+0000","last_undegraded":"2023-10-09T04:24:26.465756+0000","last_fullsized":"2023-10-09T04:24:26.465756+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:38.278137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":212,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466291+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:24:26.466291+0000","last_peered":"2023-10-09T04:24:26.466291+0000","last_clean":"2023-10-09T04:24:26.466291+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:24:26.466291+0000","last_undegraded":"2023-10-09T04:24:26.466291+0000","last_fullsized":"2023-10-09T04:24:26.466291+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":601,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519624+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:24:26.519624+0000","last_peered":"2023-10-09T04:24:26.519624+0000","last_clean":"2023-10-09T04:24:26.519624+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:24:26.519624+0000","last_undegraded":"2023-10-09T04:24:26.519624+0000","last_fullsized":"2023-10-09T04:24:26.519624+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":40,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.464232+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:24:26.464232+0000","last_peered":"2023-10-09T04:24:26.464232+0000","last_clean":"2023-10-09T04:24:26.464232+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:24:26.464232+0000","last_undegraded":"2023-10-09T04:24:26.464232+0000","last_fullsized":"2023-10-09T04:24:26.464232+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":415,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520211+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:24:26.520211+0000","last_peered":"2023-10-09T04:24:26.520211+0000","last_clean":"2023-10-09T04:24:26.520211+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:24:26.520211+0000","last_undegraded":"2023-10-09T04:24:26.520211+0000","last_fullsized":"2023-10-09T04:24:26.520211+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"437'5238","reported_seq":7824,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:29.040176+0000","last_change":"2023-10-09T04:24:14.648810+0000","last_active":"2023-10-09T04:24:29.040176+0000","last_peered":"2023-10-09T04:24:29.040176+0000","last_clean":"2023-10-09T04:23:05.325743+0000","last_became_active":"2023-10-09T04:24:14.642895+0000","last_became_peered":"2023-10-09T04:24:14.642895+0000","last_unstale":"2023-10-09T04:24:29.040176+0000","last_undegraded":"2023-10-09T04:24:29.040176+0000","last_fullsized":"2023-10-09T04:24:29.040176+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5238,"log_dups_size":0,"ondisk_log_size":5238,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125140992,"num_objects":1910,"num_object_clones":0,"num_object_copies":5730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1895,"num_objects_unfound":0,"num_objects_dirty":1910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2702,"num_write_kb":147552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":621,"num_bytes_recovered":40665088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1910}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":564,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.444074+0000","last_change":"2023-10-09T04:23:25.386262+0000","last_active":"2023-10-09T04:24:26.444074+0000","last_peered":"2023-10-09T04:24:26.444074+0000","last_clean":"2023-10-09T04:24:26.444074+0000","last_became_active":"2023-10-09T04:23:25.385887+0000","last_became_peered":"2023-10-09T04:23:25.385887+0000","last_unstale":"2023-10-09T04:24:26.444074+0000","last_undegraded":"2023-10-09T04:24:26.444074+0000","last_fullsized":"2023-10-09T04:24:26.444074+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:48.918844+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":33,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.462957+0000","last_change":"2023-10-09T04:23:22.379011+0000","last_active":"2023-10-09T04:24:26.462957+0000","last_peered":"2023-10-09T04:24:26.462957+0000","last_clean":"2023-10-09T04:24:26.462957+0000","last_became_active":"2023-10-09T04:23:22.378391+0000","last_became_peered":"2023-10-09T04:23:22.378391+0000","last_unstale":"2023-10-09T04:24:26.462957+0000","last_undegraded":"2023-10-09T04:24:26.462957+0000","last_fullsized":"2023-10-09T04:24:26.462957+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:22.951460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":40,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.464042+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:24:26.464042+0000","last_peered":"2023-10-09T04:24:26.464042+0000","last_clean":"2023-10-09T04:24:26.464042+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:24:26.464042+0000","last_undegraded":"2023-10-09T04:24:26.464042+0000","last_fullsized":"2023-10-09T04:24:26.464042+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"437'5354","reported_seq":5429,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:30.663983+0000","last_change":"2023-10-09T04:24:12.658965+0000","last_active":"2023-10-09T04:24:30.663983+0000","last_peered":"2023-10-09T04:24:30.663983+0000","last_clean":"2023-10-09T04:24:30.663983+0000","last_became_active":"2023-10-09T04:24:12.655745+0000","last_became_peered":"2023-10-09T04:24:12.655745+0000","last_unstale":"2023-10-09T04:24:30.663983+0000","last_undegraded":"2023-10-09T04:24:30.663983+0000","last_fullsized":"2023-10-09T04:24:30.663983+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5354,"log_dups_size":0,"ondisk_log_size":5354,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127336448,"num_objects":1943,"num_object_clones":0,"num_object_copies":5829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2766,"num_write_kb":150688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":601,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520085+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:24:26.520085+0000","last_peered":"2023-10-09T04:24:26.520085+0000","last_clean":"2023-10-09T04:24:26.520085+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:24:26.520085+0000","last_undegraded":"2023-10-09T04:24:26.520085+0000","last_fullsized":"2023-10-09T04:24:26.520085+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":50,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.462698+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:24:26.462698+0000","last_peered":"2023-10-09T04:24:26.462698+0000","last_clean":"2023-10-09T04:24:26.462698+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:24:26.462698+0000","last_undegraded":"2023-10-09T04:24:26.462698+0000","last_fullsized":"2023-10-09T04:24:26.462698+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":100,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443651+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:24:26.443651+0000","last_peered":"2023-10-09T04:24:26.443651+0000","last_clean":"2023-10-09T04:24:26.443651+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:24:26.443651+0000","last_undegraded":"2023-10-09T04:24:26.443651+0000","last_fullsized":"2023-10-09T04:24:26.443651+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"437'5117","reported_seq":7512,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:30.904542+0000","last_change":"2023-10-09T04:24:12.664435+0000","last_active":"2023-10-09T04:24:30.904542+0000","last_peered":"2023-10-09T04:24:30.904542+0000","last_clean":"2023-10-09T04:23:05.250360+0000","last_became_active":"2023-10-09T04:24:12.661247+0000","last_became_peered":"2023-10-09T04:24:12.661247+0000","last_unstale":"2023-10-09T04:24:30.904542+0000","last_undegraded":"2023-10-09T04:24:30.904542+0000","last_fullsized":"2023-10-09T04:24:30.904542+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122421248,"num_objects":1868,"num_object_clones":0,"num_object_copies":5604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1868,"num_objects_unfound":0,"num_objects_dirty":1868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2646,"num_write_kb":144448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":567,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,1,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1868}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":859,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466465+0000","last_change":"2023-10-09T04:23:25.383275+0000","last_active":"2023-10-09T04:24:26.466465+0000","last_peered":"2023-10-09T04:24:26.466465+0000","last_clean":"2023-10-09T04:24:26.466465+0000","last_became_active":"2023-10-09T04:23:25.382932+0000","last_became_peered":"2023-10-09T04:23:25.382932+0000","last_unstale":"2023-10-09T04:24:26.466465+0000","last_undegraded":"2023-10-09T04:24:26.466465+0000","last_fullsized":"2023-10-09T04:24:26.466465+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:24.086212+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":467,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466070+0000","last_change":"2023-10-09T04:23:22.469398+0000","last_active":"2023-10-09T04:24:26.466070+0000","last_peered":"2023-10-09T04:24:26.466070+0000","last_clean":"2023-10-09T04:24:26.466070+0000","last_became_active":"2023-10-09T04:23:22.382490+0000","last_became_peered":"2023-10-09T04:23:22.382490+0000","last_unstale":"2023-10-09T04:24:26.466070+0000","last_undegraded":"2023-10-09T04:24:26.466070+0000","last_fullsized":"2023-10-09T04:24:26.466070+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:20.439036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":398,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519534+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:24:26.519534+0000","last_peered":"2023-10-09T04:24:26.519534+0000","last_clean":"2023-10-09T04:24:26.519534+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:24:26.519534+0000","last_undegraded":"2023-10-09T04:24:26.519534+0000","last_fullsized":"2023-10-09T04:24:26.519534+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"437'5211","reported_seq":5297,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:30.763267+0000","last_change":"2023-10-09T04:24:12.681872+0000","last_active":"2023-10-09T04:24:30.763267+0000","last_peered":"2023-10-09T04:24:30.763267+0000","last_clean":"2023-10-09T04:24:30.763267+0000","last_became_active":"2023-10-09T04:24:12.676248+0000","last_became_peered":"2023-10-09T04:24:12.676248+0000","last_unstale":"2023-10-09T04:24:30.763267+0000","last_undegraded":"2023-10-09T04:24:30.763267+0000","last_fullsized":"2023-10-09T04:24:30.763267+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5211,"log_dups_size":0,"ondisk_log_size":5211,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125927424,"num_objects":1921,"num_object_clones":0,"num_object_copies":5763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2685,"num_write_kb":147424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":668,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.444873+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:24:26.444873+0000","last_peered":"2023-10-09T04:24:26.444873+0000","last_clean":"2023-10-09T04:24:26.444873+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:24:26.444873+0000","last_undegraded":"2023-10-09T04:24:26.444873+0000","last_fullsized":"2023-10-09T04:24:26.444873+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":443,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466073+0000","last_change":"2023-10-09T04:23:22.467275+0000","last_active":"2023-10-09T04:24:26.466073+0000","last_peered":"2023-10-09T04:24:26.466073+0000","last_clean":"2023-10-09T04:24:26.466073+0000","last_became_active":"2023-10-09T04:23:22.380682+0000","last_became_peered":"2023-10-09T04:23:22.380682+0000","last_unstale":"2023-10-09T04:24:26.466073+0000","last_undegraded":"2023-10-09T04:24:26.466073+0000","last_fullsized":"2023-10-09T04:24:26.466073+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:18.953922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":148,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.459455+0000","last_change":"2023-10-09T04:23:22.371153+0000","last_active":"2023-10-09T04:24:26.459455+0000","last_peered":"2023-10-09T04:24:26.459455+0000","last_clean":"2023-10-09T04:24:26.459455+0000","last_became_active":"2023-10-09T04:23:22.370388+0000","last_became_peered":"2023-10-09T04:23:22.370388+0000","last_unstale":"2023-10-09T04:24:26.459455+0000","last_undegraded":"2023-10-09T04:24:26.459455+0000","last_fullsized":"2023-10-09T04:24:26.459455+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:50.346844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"437'5226","reported_seq":8415,"reported_epoch":437,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:24:30.987565+0000","last_change":"2023-10-09T04:24:14.965495+0000","last_active":"2023-10-09T04:24:30.987565+0000","last_peered":"2023-10-09T04:24:30.987565+0000","last_clean":"2023-10-09T04:22:25.963778+0000","last_became_active":"2023-10-09T04:24:14.557230+0000","last_became_peered":"2023-10-09T04:24:14.557230+0000","last_unstale":"2023-10-09T04:24:30.987565+0000","last_undegraded":"2023-10-09T04:24:14.534051+0000","last_fullsized":"2023-10-09T04:24:13.704209+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5226,"log_dups_size":0,"ondisk_log_size":5226,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125566976,"num_objects":1916,"num_object_clones":0,"num_object_copies":5748,"num_objects_missing_on_primary":710,"num_objects_missing":710,"num_objects_degraded":1308,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2698,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":858,"num_bytes_recovered":56262656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,1],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"0,1","objects":623},{"shards":"0,3,4","objects":583},{"shards":"1,3,4","objects":710}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":621,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520333+0000","last_change":"2023-10-09T04:23:25.386738+0000","last_active":"2023-10-09T04:24:26.520333+0000","last_peered":"2023-10-09T04:24:26.520333+0000","last_clean":"2023-10-09T04:24:26.520333+0000","last_became_active":"2023-10-09T04:23:25.386422+0000","last_became_peered":"2023-10-09T04:23:25.386422+0000","last_unstale":"2023-10-09T04:24:26.520333+0000","last_undegraded":"2023-10-09T04:24:26.520333+0000","last_fullsized":"2023-10-09T04:24:26.520333+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:35.084506+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":408,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466629+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:24:26.466629+0000","last_peered":"2023-10-09T04:24:26.466629+0000","last_clean":"2023-10-09T04:24:26.466629+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:24:26.466629+0000","last_undegraded":"2023-10-09T04:24:26.466629+0000","last_fullsized":"2023-10-09T04:24:26.466629+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":63,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.450730+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:24:26.450730+0000","last_peered":"2023-10-09T04:24:26.450730+0000","last_clean":"2023-10-09T04:24:26.450730+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:24:26.450730+0000","last_undegraded":"2023-10-09T04:24:26.450730+0000","last_fullsized":"2023-10-09T04:24:26.450730+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"437'5337","reported_seq":7861,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:29.405948+0000","last_change":"2023-10-09T04:24:22.480653+0000","last_active":"2023-10-09T04:24:29.405948+0000","last_peered":"2023-10-09T04:24:29.405948+0000","last_clean":"2023-10-09T04:23:05.287910+0000","last_became_active":"2023-10-09T04:24:22.474880+0000","last_became_peered":"2023-10-09T04:24:22.474880+0000","last_unstale":"2023-10-09T04:24:29.405948+0000","last_undegraded":"2023-10-09T04:24:29.405948+0000","last_fullsized":"2023-10-09T04:24:29.405948+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129204224,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1971,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2747,"num_write_kb":151008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":603,"num_bytes_recovered":39550976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1971}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":716,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.465921+0000","last_change":"2023-10-09T04:23:25.380799+0000","last_active":"2023-10-09T04:24:26.465921+0000","last_peered":"2023-10-09T04:24:26.465921+0000","last_clean":"2023-10-09T04:24:26.465921+0000","last_became_active":"2023-10-09T04:23:25.380434+0000","last_became_peered":"2023-10-09T04:23:25.380434+0000","last_unstale":"2023-10-09T04:24:26.465921+0000","last_undegraded":"2023-10-09T04:24:26.465921+0000","last_fullsized":"2023-10-09T04:24:26.465921+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":436,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520048+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:24:26.520048+0000","last_peered":"2023-10-09T04:24:26.520048+0000","last_clean":"2023-10-09T04:24:26.520048+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:24:26.520048+0000","last_undegraded":"2023-10-09T04:24:26.520048+0000","last_fullsized":"2023-10-09T04:24:26.520048+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":49,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463237+0000","last_change":"2023-10-09T04:23:22.367231+0000","last_active":"2023-10-09T04:24:26.463237+0000","last_peered":"2023-10-09T04:24:26.463237+0000","last_clean":"2023-10-09T04:24:26.463237+0000","last_became_active":"2023-10-09T04:23:22.366606+0000","last_became_peered":"2023-10-09T04:23:22.366606+0000","last_unstale":"2023-10-09T04:24:26.463237+0000","last_undegraded":"2023-10-09T04:24:26.463237+0000","last_fullsized":"2023-10-09T04:24:26.463237+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:57.425211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"437'5256","reported_seq":7765,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:28.530696+0000","last_change":"2023-10-09T04:24:12.699405+0000","last_active":"2023-10-09T04:24:28.530696+0000","last_peered":"2023-10-09T04:24:28.530696+0000","last_clean":"2023-10-09T04:23:05.307068+0000","last_became_active":"2023-10-09T04:24:12.691077+0000","last_became_peered":"2023-10-09T04:24:12.691077+0000","last_unstale":"2023-10-09T04:24:28.530696+0000","last_undegraded":"2023-10-09T04:24:28.530696+0000","last_fullsized":"2023-10-09T04:24:28.530696+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5256,"log_dups_size":0,"ondisk_log_size":5256,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126222336,"num_objects":1926,"num_object_clones":0,"num_object_copies":5778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1926,"num_objects_unfound":0,"num_objects_dirty":1926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2704,"num_write_kb":148160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":596,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":1926}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":860,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.465634+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:24:26.465634+0000","last_peered":"2023-10-09T04:24:26.465634+0000","last_clean":"2023-10-09T04:24:26.465634+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:24:26.465634+0000","last_undegraded":"2023-10-09T04:24:26.465634+0000","last_fullsized":"2023-10-09T04:24:26.465634+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":57,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463182+0000","last_change":"2023-10-09T04:23:22.384794+0000","last_active":"2023-10-09T04:24:26.463182+0000","last_peered":"2023-10-09T04:24:26.463182+0000","last_clean":"2023-10-09T04:24:26.463182+0000","last_became_active":"2023-10-09T04:23:22.370199+0000","last_became_peered":"2023-10-09T04:23:22.370199+0000","last_unstale":"2023-10-09T04:24:26.463182+0000","last_undegraded":"2023-10-09T04:24:26.463182+0000","last_fullsized":"2023-10-09T04:24:26.463182+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:35.104932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.444612+0000","last_change":"2023-10-09T04:23:37.016405+0000","last_active":"2023-10-09T04:24:26.444612+0000","last_peered":"2023-10-09T04:24:26.444612+0000","last_clean":"2023-10-09T04:24:26.444612+0000","last_became_active":"2023-10-09T04:23:37.016108+0000","last_became_peered":"2023-10-09T04:23:37.016108+0000","last_unstale":"2023-10-09T04:24:26.444612+0000","last_undegraded":"2023-10-09T04:24:26.444612+0000","last_fullsized":"2023-10-09T04:24:26.444612+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:54.303768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"437'5192","reported_seq":5278,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:30.836546+0000","last_change":"2023-10-09T04:24:22.451555+0000","last_active":"2023-10-09T04:24:30.836546+0000","last_peered":"2023-10-09T04:24:30.836546+0000","last_clean":"2023-10-09T04:24:30.836546+0000","last_became_active":"2023-10-09T04:24:22.446741+0000","last_became_peered":"2023-10-09T04:24:22.446741+0000","last_unstale":"2023-10-09T04:24:30.836546+0000","last_undegraded":"2023-10-09T04:24:30.836546+0000","last_fullsized":"2023-10-09T04:24:30.836546+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5192,"log_dups_size":0,"ondisk_log_size":5192,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125370368,"num_objects":1913,"num_object_clones":0,"num_object_copies":5739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2673,"num_write_kb":146752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":606,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519702+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:24:26.519702+0000","last_peered":"2023-10-09T04:24:26.519702+0000","last_clean":"2023-10-09T04:24:26.519702+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:24:26.519702+0000","last_undegraded":"2023-10-09T04:24:26.519702+0000","last_fullsized":"2023-10-09T04:24:26.519702+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":174,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.450587+0000","last_change":"2023-10-09T04:23:22.368751+0000","last_active":"2023-10-09T04:24:26.450587+0000","last_peered":"2023-10-09T04:24:26.450587+0000","last_clean":"2023-10-09T04:24:26.450587+0000","last_became_active":"2023-10-09T04:23:22.368404+0000","last_became_peered":"2023-10-09T04:23:22.368404+0000","last_unstale":"2023-10-09T04:24:26.450587+0000","last_undegraded":"2023-10-09T04:24:26.450587+0000","last_fullsized":"2023-10-09T04:24:26.450587+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:11.713600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":243,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.467122+0000","last_change":"2023-10-09T04:23:22.367561+0000","last_active":"2023-10-09T04:24:26.467122+0000","last_peered":"2023-10-09T04:24:26.467122+0000","last_clean":"2023-10-09T04:24:26.467122+0000","last_became_active":"2023-10-09T04:23:22.367246+0000","last_became_peered":"2023-10-09T04:23:22.367246+0000","last_unstale":"2023-10-09T04:24:26.467122+0000","last_undegraded":"2023-10-09T04:24:26.467122+0000","last_fullsized":"2023-10-09T04:24:26.467122+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:52.238902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"437'5215","reported_seq":13586,"reported_epoch":437,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:24:31.000942+0000","last_change":"2023-10-09T04:24:13.719393+0000","last_active":"2023-10-09T04:24:31.000942+0000","last_peered":"2023-10-09T04:24:31.000942+0000","last_clean":"2023-10-09T04:23:21.324155+0000","last_became_active":"2023-10-09T04:24:13.711662+0000","last_became_peered":"2023-10-09T04:24:13.711662+0000","last_unstale":"2023-10-09T04:24:31.000942+0000","last_undegraded":"2023-10-09T04:24:13.623117+0000","last_fullsized":"2023-10-09T04:24:31.000942+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5215,"log_dups_size":0,"ondisk_log_size":5215,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125272064,"num_objects":1911,"num_object_clones":0,"num_object_copies":5733,"num_objects_missing_on_primary":77,"num_objects_missing":77,"num_objects_degraded":77,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2696,"num_write_kb":147456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2078,"num_bytes_recovered":136183808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":77},{"shards":"1,6,7","objects":1834}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":541,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463283+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:24:26.463283+0000","last_peered":"2023-10-09T04:24:26.463283+0000","last_clean":"2023-10-09T04:24:26.463283+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:24:26.463283+0000","last_undegraded":"2023-10-09T04:24:26.463283+0000","last_fullsized":"2023-10-09T04:24:26.463283+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.467007+0000","last_change":"2023-10-09T04:24:22.441420+0000","last_active":"2023-10-09T04:24:26.467007+0000","last_peered":"2023-10-09T04:24:26.467007+0000","last_clean":"2023-10-09T04:24:26.467007+0000","last_became_active":"2023-10-09T04:24:22.441132+0000","last_became_peered":"2023-10-09T04:24:22.441132+0000","last_unstale":"2023-10-09T04:24:26.467007+0000","last_undegraded":"2023-10-09T04:24:26.467007+0000","last_fullsized":"2023-10-09T04:24:26.467007+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:25.795213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":212,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.450973+0000","last_change":"2023-10-09T04:23:22.369235+0000","last_active":"2023-10-09T04:24:26.450973+0000","last_peered":"2023-10-09T04:24:26.450973+0000","last_clean":"2023-10-09T04:24:26.450973+0000","last_became_active":"2023-10-09T04:23:22.368922+0000","last_became_peered":"2023-10-09T04:23:22.368922+0000","last_unstale":"2023-10-09T04:24:26.450973+0000","last_undegraded":"2023-10-09T04:24:26.450973+0000","last_fullsized":"2023-10-09T04:24:26.450973+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:42.940762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"437'5482","reported_seq":8255,"reported_epoch":437,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:24:30.780632+0000","last_change":"2023-10-09T04:24:13.726995+0000","last_active":"2023-10-09T04:24:30.780632+0000","last_peered":"2023-10-09T04:24:30.780632+0000","last_clean":"2023-10-09T04:22:25.940554+0000","last_became_active":"2023-10-09T04:24:13.721805+0000","last_became_peered":"2023-10-09T04:24:13.721805+0000","last_unstale":"2023-10-09T04:24:30.780632+0000","last_undegraded":"2023-10-09T04:24:13.659663+0000","last_fullsized":"2023-10-09T04:24:30.780632+0000","mapping_epoch":429,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5482,"log_dups_size":0,"ondisk_log_size":5482,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131465216,"num_objects":2006,"num_object_clones":0,"num_object_copies":6018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1454,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":154464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":747,"num_bytes_recovered":48955392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"0,1,7","objects":805},{"shards":"0,4,7","objects":566},{"shards":"1,4,7","objects":635}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":601,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519712+0000","last_change":"2023-10-09T04:22:16.643512+0000","last_active":"2023-10-09T04:24:26.519712+0000","last_peered":"2023-10-09T04:24:26.519712+0000","last_clean":"2023-10-09T04:24:26.519712+0000","last_became_active":"2023-10-09T04:22:16.643150+0000","last_became_peered":"2023-10-09T04:22:16.643150+0000","last_unstale":"2023-10-09T04:24:26.519712+0000","last_undegraded":"2023-10-09T04:24:26.519712+0000","last_fullsized":"2023-10-09T04:24:26.519712+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:51.421460+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443197+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:24:26.443197+0000","last_peered":"2023-10-09T04:24:26.443197+0000","last_clean":"2023-10-09T04:24:26.443197+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:24:26.443197+0000","last_undegraded":"2023-10-09T04:24:26.443197+0000","last_fullsized":"2023-10-09T04:24:26.443197+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":99,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.465988+0000","last_change":"2023-10-09T04:24:22.441246+0000","last_active":"2023-10-09T04:24:26.465988+0000","last_peered":"2023-10-09T04:24:26.465988+0000","last_clean":"2023-10-09T04:24:26.465988+0000","last_became_active":"2023-10-09T04:24:22.441082+0000","last_became_peered":"2023-10-09T04:24:22.441082+0000","last_unstale":"2023-10-09T04:24:26.465988+0000","last_undegraded":"2023-10-09T04:24:26.465988+0000","last_fullsized":"2023-10-09T04:24:26.465988+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:45.385118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.c","version":"437'5326","reported_seq":7710,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:30.885448+0000","last_change":"2023-10-09T04:23:28.776984+0000","last_active":"2023-10-09T04:24:30.885448+0000","last_peered":"2023-10-09T04:24:30.885448+0000","last_clean":"2023-10-09T04:23:05.321351+0000","last_became_active":"2023-10-09T04:23:28.772370+0000","last_became_peered":"2023-10-09T04:23:28.772370+0000","last_unstale":"2023-10-09T04:24:30.885448+0000","last_undegraded":"2023-10-09T04:24:30.885448+0000","last_fullsized":"2023-10-09T04:24:30.885448+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5326,"log_dups_size":0,"ondisk_log_size":5326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:48.147071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527168,"num_objects":3838,"num_object_clones":0,"num_object_copies":11514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3838,"num_objects_unfound":0,"num_objects_dirty":3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5326,"num_write_kb":293248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1137,"num_bytes_recovered":74514432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,3,4],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"3,4,7","objects":3838}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":563,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.451252+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:24:26.451252+0000","last_peered":"2023-10-09T04:24:26.451252+0000","last_clean":"2023-10-09T04:24:26.451252+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:24:26.451252+0000","last_undegraded":"2023-10-09T04:24:26.451252+0000","last_fullsized":"2023-10-09T04:24:26.451252+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":38,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.474238+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:24:26.474238+0000","last_peered":"2023-10-09T04:24:26.474238+0000","last_clean":"2023-10-09T04:24:26.474238+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:24:26.474238+0000","last_undegraded":"2023-10-09T04:24:26.474238+0000","last_fullsized":"2023-10-09T04:24:26.474238+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":433,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520105+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:24:26.520105+0000","last_peered":"2023-10-09T04:24:26.520105+0000","last_clean":"2023-10-09T04:24:26.520105+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:24:26.520105+0000","last_undegraded":"2023-10-09T04:24:26.520105+0000","last_fullsized":"2023-10-09T04:24:26.520105+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":296,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520000+0000","last_change":"2023-10-09T04:24:22.449375+0000","last_active":"2023-10-09T04:24:26.520000+0000","last_peered":"2023-10-09T04:24:26.520000+0000","last_clean":"2023-10-09T04:24:26.520000+0000","last_became_active":"2023-10-09T04:24:22.448978+0000","last_became_peered":"2023-10-09T04:24:22.448978+0000","last_unstale":"2023-10-09T04:24:26.520000+0000","last_undegraded":"2023-10-09T04:24:26.520000+0000","last_fullsized":"2023-10-09T04:24:26.520000+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:08.615425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"437'5355","reported_seq":9051,"reported_epoch":437,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:24:29.630013+0000","last_change":"2023-10-09T04:24:26.421981+0000","last_active":"2023-10-09T04:24:29.630013+0000","last_peered":"2023-10-09T04:24:29.630013+0000","last_clean":"2023-10-09T04:23:05.256815+0000","last_became_active":"2023-10-09T04:23:07.467082+0000","last_became_peered":"2023-10-09T04:23:07.467082+0000","last_unstale":"2023-10-09T04:24:29.630013+0000","last_undegraded":"2023-10-09T04:24:29.630013+0000","last_fullsized":"2023-10-09T04:24:29.630013+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5355,"log_dups_size":0,"ondisk_log_size":5355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.977596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248578048,"num_objects":3793,"num_object_clones":0,"num_object_copies":11379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3248,"num_objects_unfound":0,"num_objects_dirty":3793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5355,"num_write_kb":292736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1794,"num_bytes_recovered":117571584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":3793}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":604,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520192+0000","last_change":"2023-10-09T04:23:08.631094+0000","last_active":"2023-10-09T04:24:26.520192+0000","last_peered":"2023-10-09T04:24:26.520192+0000","last_clean":"2023-10-09T04:24:26.520192+0000","last_became_active":"2023-10-09T04:23:08.630831+0000","last_became_peered":"2023-10-09T04:23:08.630831+0000","last_unstale":"2023-10-09T04:24:26.520192+0000","last_undegraded":"2023-10-09T04:24:26.520192+0000","last_fullsized":"2023-10-09T04:24:26.520192+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:17.921309+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":420,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519863+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:24:26.519863+0000","last_peered":"2023-10-09T04:24:26.519863+0000","last_clean":"2023-10-09T04:24:26.519863+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:24:26.519863+0000","last_undegraded":"2023-10-09T04:24:26.519863+0000","last_fullsized":"2023-10-09T04:24:26.519863+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":217,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463044+0000","last_change":"2023-10-09T04:23:22.370474+0000","last_active":"2023-10-09T04:24:26.463044+0000","last_peered":"2023-10-09T04:24:26.463044+0000","last_clean":"2023-10-09T04:24:26.463044+0000","last_became_active":"2023-10-09T04:23:22.370173+0000","last_became_peered":"2023-10-09T04:23:22.370173+0000","last_unstale":"2023-10-09T04:24:26.463044+0000","last_undegraded":"2023-10-09T04:24:26.463044+0000","last_fullsized":"2023-10-09T04:24:26.463044+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:25.871804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":398,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.461073+0000","last_change":"2023-10-09T04:24:22.445259+0000","last_active":"2023-10-09T04:24:26.461073+0000","last_peered":"2023-10-09T04:24:26.461073+0000","last_clean":"2023-10-09T04:24:26.461073+0000","last_became_active":"2023-10-09T04:24:22.444960+0000","last_became_peered":"2023-10-09T04:24:22.444960+0000","last_unstale":"2023-10-09T04:24:26.461073+0000","last_undegraded":"2023-10-09T04:24:26.461073+0000","last_fullsized":"2023-10-09T04:24:26.461073+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:22.200720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"437'5350","reported_seq":5426,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:30.887704+0000","last_change":"2023-10-09T04:24:12.638694+0000","last_active":"2023-10-09T04:24:30.887704+0000","last_peered":"2023-10-09T04:24:30.887704+0000","last_clean":"2023-10-09T04:24:30.887704+0000","last_became_active":"2023-10-09T04:24:12.633090+0000","last_became_peered":"2023-10-09T04:24:12.633090+0000","last_unstale":"2023-10-09T04:24:30.887704+0000","last_undegraded":"2023-10-09T04:24:30.887704+0000","last_fullsized":"2023-10-09T04:24:30.887704+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5350,"log_dups_size":0,"ondisk_log_size":5350,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127598592,"num_objects":1947,"num_object_clones":0,"num_object_copies":5841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2762,"num_write_kb":150688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":575,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.464001+0000","last_change":"2023-10-09T04:23:25.379708+0000","last_active":"2023-10-09T04:24:26.464001+0000","last_peered":"2023-10-09T04:24:26.464001+0000","last_clean":"2023-10-09T04:24:26.464001+0000","last_became_active":"2023-10-09T04:23:25.379318+0000","last_became_peered":"2023-10-09T04:23:25.379318+0000","last_unstale":"2023-10-09T04:24:26.464001+0000","last_undegraded":"2023-10-09T04:24:26.464001+0000","last_fullsized":"2023-10-09T04:24:26.464001+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:36.564798+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":268,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463167+0000","last_change":"2023-10-09T04:24:22.449470+0000","last_active":"2023-10-09T04:24:26.463167+0000","last_peered":"2023-10-09T04:24:26.463167+0000","last_clean":"2023-10-09T04:24:26.463167+0000","last_became_active":"2023-10-09T04:24:22.448762+0000","last_became_peered":"2023-10-09T04:24:22.448762+0000","last_unstale":"2023-10-09T04:24:26.463167+0000","last_undegraded":"2023-10-09T04:24:26.463167+0000","last_fullsized":"2023-10-09T04:24:26.463167+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:13.213314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":373,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.459061+0000","last_change":"2023-10-09T04:24:22.444595+0000","last_active":"2023-10-09T04:24:26.459061+0000","last_peered":"2023-10-09T04:24:26.459061+0000","last_clean":"2023-10-09T04:24:26.459061+0000","last_became_active":"2023-10-09T04:24:22.444387+0000","last_became_peered":"2023-10-09T04:24:22.444387+0000","last_unstale":"2023-10-09T04:24:26.459061+0000","last_undegraded":"2023-10-09T04:24:26.459061+0000","last_fullsized":"2023-10-09T04:24:26.459061+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:37.195469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520441+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:24:26.520441+0000","last_peered":"2023-10-09T04:24:26.520441+0000","last_clean":"2023-10-09T04:24:26.520441+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:24:26.520441+0000","last_undegraded":"2023-10-09T04:24:26.520441+0000","last_fullsized":"2023-10-09T04:24:26.520441+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"437'5404","reported_seq":13132,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:27.499566+0000","last_change":"2023-10-09T04:24:27.462956+0000","last_active":"2023-10-09T04:24:27.499566+0000","last_peered":"2023-10-09T04:24:27.499566+0000","last_clean":"2023-10-09T04:24:27.499566+0000","last_became_active":"2023-10-09T04:24:27.454212+0000","last_became_peered":"2023-10-09T04:24:27.454212+0000","last_unstale":"2023-10-09T04:24:27.499566+0000","last_undegraded":"2023-10-09T04:24:27.499566+0000","last_fullsized":"2023-10-09T04:24:27.499566+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:13.486516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260308992,"num_objects":3972,"num_object_clones":0,"num_object_copies":11916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5404,"num_write_kb":300032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3903,"num_bytes_recovered":255787008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":420,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520069+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:24:26.520069+0000","last_peered":"2023-10-09T04:24:26.520069+0000","last_clean":"2023-10-09T04:24:26.520069+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:24:26.520069+0000","last_undegraded":"2023-10-09T04:24:26.520069+0000","last_fullsized":"2023-10-09T04:24:26.520069+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":891,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466074+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:24:26.466074+0000","last_peered":"2023-10-09T04:24:26.466074+0000","last_clean":"2023-10-09T04:24:26.466074+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:24:26.466074+0000","last_undegraded":"2023-10-09T04:24:26.466074+0000","last_fullsized":"2023-10-09T04:24:26.466074+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":373,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520124+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:24:26.520124+0000","last_peered":"2023-10-09T04:24:26.520124+0000","last_clean":"2023-10-09T04:24:26.520124+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:24:26.520124+0000","last_undegraded":"2023-10-09T04:24:26.520124+0000","last_fullsized":"2023-10-09T04:24:26.520124+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":174,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466892+0000","last_change":"2023-10-09T04:23:22.381833+0000","last_active":"2023-10-09T04:24:26.466892+0000","last_peered":"2023-10-09T04:24:26.466892+0000","last_clean":"2023-10-09T04:24:26.466892+0000","last_became_active":"2023-10-09T04:23:22.381088+0000","last_became_peered":"2023-10-09T04:23:22.381088+0000","last_unstale":"2023-10-09T04:24:26.466892+0000","last_undegraded":"2023-10-09T04:24:26.466892+0000","last_fullsized":"2023-10-09T04:24:26.466892+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:32.847976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"437'5220","reported_seq":7807,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:29.449514+0000","last_change":"2023-10-09T04:24:12.654876+0000","last_active":"2023-10-09T04:24:29.449514+0000","last_peered":"2023-10-09T04:24:29.449514+0000","last_clean":"2023-10-09T04:23:05.325743+0000","last_became_active":"2023-10-09T04:24:12.649119+0000","last_became_peered":"2023-10-09T04:24:12.649119+0000","last_unstale":"2023-10-09T04:24:29.449514+0000","last_undegraded":"2023-10-09T04:24:29.449514+0000","last_fullsized":"2023-10-09T04:24:29.449514+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5220,"log_dups_size":0,"ondisk_log_size":5220,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126320640,"num_objects":1927,"num_object_clones":0,"num_object_copies":5781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1912,"num_objects_unfound":0,"num_objects_dirty":1927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2683,"num_write_kb":147552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":620,"num_bytes_recovered":40665088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1927}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":553,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519919+0000","last_change":"2023-10-09T04:23:25.384702+0000","last_active":"2023-10-09T04:24:26.519919+0000","last_peered":"2023-10-09T04:24:26.519919+0000","last_clean":"2023-10-09T04:24:26.519919+0000","last_became_active":"2023-10-09T04:23:25.384324+0000","last_became_peered":"2023-10-09T04:23:25.384324+0000","last_unstale":"2023-10-09T04:24:26.519919+0000","last_undegraded":"2023-10-09T04:24:26.519919+0000","last_fullsized":"2023-10-09T04:24:26.519919+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:03.536732+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":437,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.465991+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:24:26.465991+0000","last_peered":"2023-10-09T04:24:26.465991+0000","last_clean":"2023-10-09T04:24:26.465991+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:24:26.465991+0000","last_undegraded":"2023-10-09T04:24:26.465991+0000","last_fullsized":"2023-10-09T04:24:26.465991+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":49,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463448+0000","last_change":"2023-10-09T04:23:22.378846+0000","last_active":"2023-10-09T04:24:26.463448+0000","last_peered":"2023-10-09T04:24:26.463448+0000","last_clean":"2023-10-09T04:24:26.463448+0000","last_became_active":"2023-10-09T04:23:22.377975+0000","last_became_peered":"2023-10-09T04:23:22.377975+0000","last_unstale":"2023-10-09T04:24:26.463448+0000","last_undegraded":"2023-10-09T04:24:26.463448+0000","last_fullsized":"2023-10-09T04:24:26.463448+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:47.753526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":40,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463974+0000","last_change":"2023-10-09T04:23:06.341197+0000","last_active":"2023-10-09T04:24:26.463974+0000","last_peered":"2023-10-09T04:24:26.463974+0000","last_clean":"2023-10-09T04:24:26.463974+0000","last_became_active":"2023-10-09T04:23:06.340773+0000","last_became_peered":"2023-10-09T04:23:06.340773+0000","last_unstale":"2023-10-09T04:24:26.463974+0000","last_undegraded":"2023-10-09T04:24:26.463974+0000","last_fullsized":"2023-10-09T04:24:26.463974+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:55.764215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"437'5281","reported_seq":7880,"reported_epoch":437,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:24:30.988521+0000","last_change":"2023-10-09T04:24:12.440192+0000","last_active":"2023-10-09T04:24:30.988521+0000","last_peered":"2023-10-09T04:24:30.988521+0000","last_clean":"2023-10-09T04:22:25.956502+0000","last_became_active":"2023-10-09T04:23:28.336672+0000","last_became_peered":"2023-10-09T04:23:28.336672+0000","last_unstale":"2023-10-09T04:24:30.988521+0000","last_undegraded":"2023-10-09T04:24:30.988521+0000","last_fullsized":"2023-10-09T04:23:23.362505+0000","mapping_epoch":415,"log_start":"388'200","ondisk_log_start":"388'200","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5081,"log_dups_size":0,"ondisk_log_size":5081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:35.497725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246153240,"num_objects":3757,"num_object_clones":0,"num_object_copies":11271,"num_objects_missing_on_primary":2240,"num_objects_missing":2240,"num_objects_degraded":0,"num_objects_misplaced":2977,"num_objects_unfound":0,"num_objects_dirty":3757,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5281,"num_write_kb":289153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1243,"num_bytes_recovered":81461248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,3,4,6","objects":1},{"shards":"0,4,6","objects":737},{"shards":"0,6","objects":779},{"shards":"3,4,6","objects":2240}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":555,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.451367+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:24:26.451367+0000","last_peered":"2023-10-09T04:24:26.451367+0000","last_clean":"2023-10-09T04:24:26.451367+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:24:26.451367+0000","last_undegraded":"2023-10-09T04:24:26.451367+0000","last_fullsized":"2023-10-09T04:24:26.451367+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":450,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466704+0000","last_change":"2023-10-09T04:23:22.381998+0000","last_active":"2023-10-09T04:24:26.466704+0000","last_peered":"2023-10-09T04:24:26.466704+0000","last_clean":"2023-10-09T04:24:26.466704+0000","last_became_active":"2023-10-09T04:23:22.381540+0000","last_became_peered":"2023-10-09T04:23:22.381540+0000","last_unstale":"2023-10-09T04:24:26.466704+0000","last_undegraded":"2023-10-09T04:24:26.466704+0000","last_fullsized":"2023-10-09T04:24:26.466704+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:21.053870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":175,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520252+0000","last_change":"2023-10-09T04:24:22.447108+0000","last_active":"2023-10-09T04:24:26.520252+0000","last_peered":"2023-10-09T04:24:26.520252+0000","last_clean":"2023-10-09T04:24:26.520252+0000","last_became_active":"2023-10-09T04:24:22.446733+0000","last_became_peered":"2023-10-09T04:24:22.446733+0000","last_unstale":"2023-10-09T04:24:26.520252+0000","last_undegraded":"2023-10-09T04:24:26.520252+0000","last_fullsized":"2023-10-09T04:24:26.520252+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:33.132499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":156,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520002+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:24:26.520002+0000","last_peered":"2023-10-09T04:24:26.520002+0000","last_clean":"2023-10-09T04:24:26.520002+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:24:26.520002+0000","last_undegraded":"2023-10-09T04:24:26.520002+0000","last_fullsized":"2023-10-09T04:24:26.520002+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"437'5268","reported_seq":13948,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:27.507927+0000","last_change":"2023-10-09T04:24:11.095905+0000","last_active":"2023-10-09T04:24:27.507927+0000","last_peered":"2023-10-09T04:24:27.507927+0000","last_clean":"2023-10-09T04:24:27.507927+0000","last_became_active":"2023-10-09T04:24:11.073289+0000","last_became_peered":"2023-10-09T04:24:11.073289+0000","last_unstale":"2023-10-09T04:24:27.507927+0000","last_undegraded":"2023-10-09T04:24:27.507927+0000","last_fullsized":"2023-10-09T04:24:27.507927+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:31.773778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253100032,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5268,"num_write_kb":292160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4291,"num_bytes_recovered":281214976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":425,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519934+0000","last_change":"2023-10-09T04:24:22.447712+0000","last_active":"2023-10-09T04:24:26.519934+0000","last_peered":"2023-10-09T04:24:26.519934+0000","last_clean":"2023-10-09T04:24:26.519934+0000","last_became_active":"2023-10-09T04:24:22.447439+0000","last_became_peered":"2023-10-09T04:24:22.447439+0000","last_unstale":"2023-10-09T04:24:26.519934+0000","last_undegraded":"2023-10-09T04:24:26.519934+0000","last_fullsized":"2023-10-09T04:24:26.519934+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":397,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466224+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:24:26.466224+0000","last_peered":"2023-10-09T04:24:26.466224+0000","last_clean":"2023-10-09T04:24:26.466224+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:24:26.466224+0000","last_undegraded":"2023-10-09T04:24:26.466224+0000","last_fullsized":"2023-10-09T04:24:26.466224+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":376,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.461817+0000","last_change":"2023-10-09T04:23:22.371790+0000","last_active":"2023-10-09T04:24:26.461817+0000","last_peered":"2023-10-09T04:24:26.461817+0000","last_clean":"2023-10-09T04:24:26.461817+0000","last_became_active":"2023-10-09T04:23:22.371464+0000","last_became_peered":"2023-10-09T04:23:22.371464+0000","last_unstale":"2023-10-09T04:24:26.461817+0000","last_undegraded":"2023-10-09T04:24:26.461817+0000","last_fullsized":"2023-10-09T04:24:26.461817+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:38.674475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":335,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466162+0000","last_change":"2023-10-09T04:24:22.449209+0000","last_active":"2023-10-09T04:24:26.466162+0000","last_peered":"2023-10-09T04:24:26.466162+0000","last_clean":"2023-10-09T04:24:26.466162+0000","last_became_active":"2023-10-09T04:24:22.448886+0000","last_became_peered":"2023-10-09T04:24:22.448886+0000","last_unstale":"2023-10-09T04:24:26.466162+0000","last_undegraded":"2023-10-09T04:24:26.466162+0000","last_fullsized":"2023-10-09T04:24:26.466162+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:03.309571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"437'5365","reported_seq":5428,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:27.537567+0000","last_change":"2023-10-09T04:22:13.630026+0000","last_active":"2023-10-09T04:24:27.537567+0000","last_peered":"2023-10-09T04:24:27.537567+0000","last_clean":"2023-10-09T04:24:27.537567+0000","last_became_active":"2023-10-09T04:22:13.629701+0000","last_became_peered":"2023-10-09T04:22:13.629701+0000","last_unstale":"2023-10-09T04:24:27.537567+0000","last_undegraded":"2023-10-09T04:24:27.537567+0000","last_fullsized":"2023-10-09T04:24:27.537567+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5365,"log_dups_size":0,"ondisk_log_size":5365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:47.148119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259325952,"num_objects":3957,"num_object_clones":0,"num_object_copies":11871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5365,"num_write_kb":298304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":610,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.444732+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:24:26.444732+0000","last_peered":"2023-10-09T04:24:26.444732+0000","last_clean":"2023-10-09T04:24:26.444732+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:24:26.444732+0000","last_undegraded":"2023-10-09T04:24:26.444732+0000","last_fullsized":"2023-10-09T04:24:26.444732+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":232,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.451402+0000","last_change":"2023-10-09T04:24:22.444161+0000","last_active":"2023-10-09T04:24:26.451402+0000","last_peered":"2023-10-09T04:24:26.451402+0000","last_clean":"2023-10-09T04:24:26.451402+0000","last_became_active":"2023-10-09T04:24:22.443860+0000","last_became_peered":"2023-10-09T04:24:22.443860+0000","last_unstale":"2023-10-09T04:24:26.451402+0000","last_undegraded":"2023-10-09T04:24:26.451402+0000","last_fullsized":"2023-10-09T04:24:26.451402+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:35.445455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":57,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.464200+0000","last_change":"2023-10-09T04:23:22.459821+0000","last_active":"2023-10-09T04:24:26.464200+0000","last_peered":"2023-10-09T04:24:26.464200+0000","last_clean":"2023-10-09T04:24:26.464200+0000","last_became_active":"2023-10-09T04:23:22.374691+0000","last_became_peered":"2023-10-09T04:23:22.374691+0000","last_unstale":"2023-10-09T04:24:26.464200+0000","last_undegraded":"2023-10-09T04:24:26.464200+0000","last_fullsized":"2023-10-09T04:24:26.464200+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:36.436501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":174,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.519964+0000","last_change":"2023-10-09T04:23:22.376671+0000","last_active":"2023-10-09T04:24:26.519964+0000","last_peered":"2023-10-09T04:24:26.519964+0000","last_clean":"2023-10-09T04:24:26.519964+0000","last_became_active":"2023-10-09T04:23:22.376262+0000","last_became_peered":"2023-10-09T04:23:22.376262+0000","last_unstale":"2023-10-09T04:24:26.519964+0000","last_undegraded":"2023-10-09T04:24:26.519964+0000","last_fullsized":"2023-10-09T04:24:26.519964+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:04.864080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.12","version":"437'5255","reported_seq":7761,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:28.669673+0000","last_change":"2023-10-09T04:24:14.646544+0000","last_active":"2023-10-09T04:24:28.669673+0000","last_peered":"2023-10-09T04:24:28.669673+0000","last_clean":"2023-10-09T04:23:05.307068+0000","last_became_active":"2023-10-09T04:24:14.642949+0000","last_became_peered":"2023-10-09T04:24:14.642949+0000","last_unstale":"2023-10-09T04:24:28.669673+0000","last_undegraded":"2023-10-09T04:24:28.669673+0000","last_fullsized":"2023-10-09T04:24:28.669673+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5255,"log_dups_size":0,"ondisk_log_size":5255,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126287872,"num_objects":1927,"num_object_clones":0,"num_object_copies":5781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1927,"num_objects_unfound":0,"num_objects_dirty":1927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2703,"num_write_kb":148160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":596,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":1927}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":466,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463248+0000","last_change":"2023-10-09T04:23:08.628990+0000","last_active":"2023-10-09T04:24:26.463248+0000","last_peered":"2023-10-09T04:24:26.463248+0000","last_clean":"2023-10-09T04:24:26.463248+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:24:26.463248+0000","last_undegraded":"2023-10-09T04:24:26.463248+0000","last_fullsized":"2023-10-09T04:24:26.463248+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":49,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.466079+0000","last_change":"2023-10-09T04:23:22.374059+0000","last_active":"2023-10-09T04:24:26.466079+0000","last_peered":"2023-10-09T04:24:26.466079+0000","last_clean":"2023-10-09T04:24:26.466079+0000","last_became_active":"2023-10-09T04:23:22.373732+0000","last_became_peered":"2023-10-09T04:23:22.373732+0000","last_unstale":"2023-10-09T04:24:26.466079+0000","last_undegraded":"2023-10-09T04:24:26.466079+0000","last_fullsized":"2023-10-09T04:24:26.466079+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:37.083009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":273,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.467471+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:24:26.467471+0000","last_peered":"2023-10-09T04:24:26.467471+0000","last_clean":"2023-10-09T04:24:26.467471+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:24:26.467471+0000","last_undegraded":"2023-10-09T04:24:26.467471+0000","last_fullsized":"2023-10-09T04:24:26.467471+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"437'5330","reported_seq":7855,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:29.586604+0000","last_change":"2023-10-09T04:24:22.480882+0000","last_active":"2023-10-09T04:24:29.586604+0000","last_peered":"2023-10-09T04:24:29.586604+0000","last_clean":"2023-10-09T04:23:05.287910+0000","last_became_active":"2023-10-09T04:24:22.475675+0000","last_became_peered":"2023-10-09T04:24:22.475675+0000","last_unstale":"2023-10-09T04:24:29.586604+0000","last_undegraded":"2023-10-09T04:24:29.586604+0000","last_fullsized":"2023-10-09T04:24:29.586604+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5330,"log_dups_size":0,"ondisk_log_size":5330,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129662976,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1979,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2741,"num_write_kb":151008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":604,"num_bytes_recovered":39550976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1979}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":511,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.451254+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:24:26.451254+0000","last_peered":"2023-10-09T04:24:26.451254+0000","last_clean":"2023-10-09T04:24:26.451254+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:24:26.451254+0000","last_undegraded":"2023-10-09T04:24:26.451254+0000","last_fullsized":"2023-10-09T04:24:26.451254+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":440,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520134+0000","last_change":"2023-10-09T04:23:22.457283+0000","last_active":"2023-10-09T04:24:26.520134+0000","last_peered":"2023-10-09T04:24:26.520134+0000","last_clean":"2023-10-09T04:24:26.520134+0000","last_became_active":"2023-10-09T04:23:22.376038+0000","last_became_peered":"2023-10-09T04:23:22.376038+0000","last_unstale":"2023-10-09T04:24:26.520134+0000","last_undegraded":"2023-10-09T04:24:26.520134+0000","last_fullsized":"2023-10-09T04:24:26.520134+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:17:27.509695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":110,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.444250+0000","last_change":"2023-10-09T04:24:22.441556+0000","last_active":"2023-10-09T04:24:26.444250+0000","last_peered":"2023-10-09T04:24:26.444250+0000","last_clean":"2023-10-09T04:24:26.444250+0000","last_became_active":"2023-10-09T04:24:22.441272+0000","last_became_peered":"2023-10-09T04:24:22.441272+0000","last_unstale":"2023-10-09T04:24:26.444250+0000","last_undegraded":"2023-10-09T04:24:26.444250+0000","last_fullsized":"2023-10-09T04:24:26.444250+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:56.817127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.14","version":"437'5211","reported_seq":9193,"reported_epoch":437,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:24:30.969135+0000","last_change":"2023-10-09T04:24:16.142550+0000","last_active":"2023-10-09T04:24:30.969135+0000","last_peered":"2023-10-09T04:24:30.969135+0000","last_clean":"2023-10-09T04:22:25.963778+0000","last_became_active":"2023-10-09T04:24:15.721070+0000","last_became_peered":"2023-10-09T04:24:15.721070+0000","last_unstale":"2023-10-09T04:24:30.969135+0000","last_undegraded":"2023-10-09T04:24:30.969135+0000","last_fullsized":"2023-10-09T04:24:14.975934+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5211,"log_dups_size":0,"ondisk_log_size":5211,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126550016,"num_objects":1931,"num_object_clones":0,"num_object_copies":5793,"num_objects_missing_on_primary":316,"num_objects_missing":316,"num_objects_degraded":0,"num_objects_misplaced":889,"num_objects_unfound":0,"num_objects_dirty":1931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2683,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1254,"num_bytes_recovered":82018304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,1],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"0,1","objects":1041},{"shards":"0,1,3,4","objects":1},{"shards":"0,3,4","objects":573},{"shards":"1,3,4","objects":316}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":451,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.520029+0000","last_change":"2023-10-09T04:23:22.463818+0000","last_active":"2023-10-09T04:24:26.520029+0000","last_peered":"2023-10-09T04:24:26.520029+0000","last_clean":"2023-10-09T04:24:26.520029+0000","last_became_active":"2023-10-09T04:23:22.376113+0000","last_became_peered":"2023-10-09T04:23:22.376113+0000","last_unstale":"2023-10-09T04:24:26.520029+0000","last_undegraded":"2023-10-09T04:24:26.520029+0000","last_fullsized":"2023-10-09T04:24:26.520029+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:12.436065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":217,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463908+0000","last_change":"2023-10-09T04:23:22.375144+0000","last_active":"2023-10-09T04:24:26.463908+0000","last_peered":"2023-10-09T04:24:26.463908+0000","last_clean":"2023-10-09T04:24:26.463908+0000","last_became_active":"2023-10-09T04:23:22.374905+0000","last_became_peered":"2023-10-09T04:23:22.374905+0000","last_unstale":"2023-10-09T04:24:26.463908+0000","last_undegraded":"2023-10-09T04:24:26.463908+0000","last_fullsized":"2023-10-09T04:24:26.463908+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:32.802152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.17","version":"437'5214","reported_seq":5299,"reported_epoch":437,"state":"active+clean","last_fresh":"2023-10-09T04:24:30.760631+0000","last_change":"2023-10-09T04:24:12.686316+0000","last_active":"2023-10-09T04:24:30.760631+0000","last_peered":"2023-10-09T04:24:30.760631+0000","last_clean":"2023-10-09T04:24:30.760631+0000","last_became_active":"2023-10-09T04:24:12.682272+0000","last_became_peered":"2023-10-09T04:24:12.682272+0000","last_unstale":"2023-10-09T04:24:30.760631+0000","last_undegraded":"2023-10-09T04:24:30.760631+0000","last_fullsized":"2023-10-09T04:24:30.760631+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5214,"log_dups_size":0,"ondisk_log_size":5214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125730816,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2689,"num_write_kb":147424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":785,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.467210+0000","last_change":"2023-10-09T04:22:16.629747+0000","last_active":"2023-10-09T04:24:26.467210+0000","last_peered":"2023-10-09T04:24:26.467210+0000","last_clean":"2023-10-09T04:24:26.467210+0000","last_became_active":"2023-10-09T04:22:16.629376+0000","last_became_peered":"2023-10-09T04:22:16.629376+0000","last_unstale":"2023-10-09T04:24:26.467210+0000","last_undegraded":"2023-10-09T04:24:26.467210+0000","last_fullsized":"2023-10-09T04:24:26.467210+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:21:55.959478+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:23.285011+0000","scrub_duration":0.0016555039999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":151,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.443822+0000","last_change":"2023-10-09T04:24:22.443922+0000","last_active":"2023-10-09T04:24:26.443822+0000","last_peered":"2023-10-09T04:24:26.443822+0000","last_clean":"2023-10-09T04:24:26.443822+0000","last_became_active":"2023-10-09T04:24:22.443562+0000","last_became_peered":"2023-10-09T04:24:22.443562+0000","last_unstale":"2023-10-09T04:24:26.443822+0000","last_undegraded":"2023-10-09T04:24:26.443822+0000","last_fullsized":"2023-10-09T04:24:26.443822+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:04.129668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":268,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463093+0000","last_change":"2023-10-09T04:24:22.449672+0000","last_active":"2023-10-09T04:24:26.463093+0000","last_peered":"2023-10-09T04:24:26.463093+0000","last_clean":"2023-10-09T04:24:26.463093+0000","last_became_active":"2023-10-09T04:24:22.449279+0000","last_became_peered":"2023-10-09T04:24:22.449279+0000","last_unstale":"2023-10-09T04:24:26.463093+0000","last_undegraded":"2023-10-09T04:24:26.463093+0000","last_fullsized":"2023-10-09T04:24:26.463093+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:56.633978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.16","version":"437'5092","reported_seq":7487,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:30.685399+0000","last_change":"2023-10-09T04:24:14.647659+0000","last_active":"2023-10-09T04:24:30.685399+0000","last_peered":"2023-10-09T04:24:30.685399+0000","last_clean":"2023-10-09T04:23:05.250360+0000","last_became_active":"2023-10-09T04:24:14.644336+0000","last_became_peered":"2023-10-09T04:24:14.644336+0000","last_unstale":"2023-10-09T04:24:30.685399+0000","last_undegraded":"2023-10-09T04:24:30.685399+0000","last_fullsized":"2023-10-09T04:24:30.685399+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124059648,"num_objects":1893,"num_object_clones":0,"num_object_copies":5679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1893,"num_objects_unfound":0,"num_objects_dirty":1893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2621,"num_write_kb":144448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":568,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,1,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1893}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":540,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.459776+0000","last_change":"2023-10-09T04:24:22.444652+0000","last_active":"2023-10-09T04:24:26.459776+0000","last_peered":"2023-10-09T04:24:26.459776+0000","last_clean":"2023-10-09T04:24:26.459776+0000","last_became_active":"2023-10-09T04:24:22.444410+0000","last_became_peered":"2023-10-09T04:24:22.444410+0000","last_unstale":"2023-10-09T04:24:26.459776+0000","last_undegraded":"2023-10-09T04:24:26.459776+0000","last_fullsized":"2023-10-09T04:24:26.459776+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:58.649986+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":46,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.463825+0000","last_change":"2023-10-09T04:23:06.385526+0000","last_active":"2023-10-09T04:24:26.463825+0000","last_peered":"2023-10-09T04:24:26.463825+0000","last_clean":"2023-10-09T04:24:26.463825+0000","last_became_active":"2023-10-09T04:23:06.345850+0000","last_became_peered":"2023-10-09T04:23:06.345850+0000","last_unstale":"2023-10-09T04:24:26.463825+0000","last_undegraded":"2023-10-09T04:24:26.463825+0000","last_fullsized":"2023-10-09T04:24:26.463825+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":174,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-10-09T04:24:26.450329+0000","last_change":"2023-10-09T04:23:22.374292+0000","last_active":"2023-10-09T04:24:26.450329+0000","last_peered":"2023-10-09T04:24:26.450329+0000","last_clean":"2023-10-09T04:24:26.450329+0000","last_became_active":"2023-10-09T04:23:22.373904+0000","last_became_peered":"2023-10-09T04:23:22.373904+0000","last_unstale":"2023-10-09T04:24:26.450329+0000","last_undegraded":"2023-10-09T04:24:26.450329+0000","last_fullsized":"2023-10-09T04:24:26.450329+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:20.848759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":4050452504,"num_objects":61806,"num_object_clones":0,"num_object_copies":185418,"num_objects_missing_on_primary":3442,"num_objects_missing":3442,"num_objects_degraded":4461,"num_objects_misplaced":26323,"num_objects_unfound":0,"num_objects_dirty":61806,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":86222,"num_write_kb":4736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24913,"num_bytes_recovered":1632567296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13624029184,"data_stored":13624016968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137000,"ondisk_log_size":137000,"up":78,"acting":75,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":11468,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":715597,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":104,"num_read_kb":195,"num_write":50,"num_write_kb":513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1686409,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201664,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2398780,"kb_used_data":1490676,"kb_used_omap":0,"kb_used_meta":908096,"kb_avail":91346372,"statfs":{"total":95995035648,"available":93538684928,"internally_reserved":0,"allocated":1526452224,"data_stored":1525148461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929890304},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":3.4470000000000001,"15min":3.4260000000000002},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":4.601,"5min":148.65899999999999,"15min":148.65899999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.4630000000000001,"5min":3.6720000000000002,"15min":3.4449999999999998},"min":{"1min":0.26400000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.2919999999999998,"5min":148.642,"15min":148.642},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:14 2023","interfaces":[{"interface":"back","average":{"1min":4.8010000000000002,"5min":10.42,"15min":6.851},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":74.933999999999997,"5min":261.935,"15min":261.935},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.8789999999999996,"5min":10.02,"15min":6.6669999999999998},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":69.355000000000004,"5min":262.07600000000002,"15min":262.07600000000002},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":3.706,"15min":2.5459999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.3470000000000004,"5min":253.41200000000001,"15min":253.41200000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.516,"5min":4.2130000000000001,"15min":2.9630000000000001},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":8.5280000000000005,"5min":253.39400000000001,"15min":253.39400000000001},"last":0.54900000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:24:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":4.2320000000000002,"15min":2.9100000000000001},"min":{"1min":0.42299999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.4859999999999998,"5min":255.97300000000001,"15min":255.97300000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.4119999999999999,"5min":4.5650000000000004,"15min":3.1560000000000001},"min":{"1min":0.42499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.5,"5min":256.06099999999998,"15min":256.06099999999998},"last":0.40699999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.89100000000000001,"15min":0.82999999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23400000000000001},"max":{"1min":3.4630000000000001,"5min":4.8529999999999998,"15min":16.195},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.01,"5min":1.3260000000000001,"15min":1.179},"min":{"1min":0.27900000000000003,"5min":0.251,"15min":0.251},"max":{"1min":7.2549999999999999,"5min":12.566000000000001,"15min":12.566000000000001},"last":0.35399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.6339999999999999,"15min":1.5449999999999999},"min":{"1min":0.29699999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.1440000000000001,"5min":13.122999999999999,"15min":120.066},"last":17.882999999999999},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.234,"15min":1.379},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":15.795,"5min":15.795,"15min":120.09699999999999},"last":17.568999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.51,"15min":1.159},"min":{"1min":0.311,"5min":0.311,"15min":0.307},"max":{"1min":5.0599999999999996,"5min":8.4550000000000001,"15min":8.4550000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.359,"5min":1.0289999999999999,"15min":0.91200000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":4.2430000000000003,"5min":5.968,"15min":8.9009999999999998},"last":0.378}]}]},{"osd":6,"up_from":47,"seq":201863463299,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2067556,"kb_used_data":1757980,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":91677596,"statfs":{"total":95995035648,"available":93877858304,"internally_reserved":0,"allocated":1800171520,"data_stored":1798882715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":4.6429999999999998,"15min":4.0469999999999997},"min":{"1min":0.215,"5min":0.19800000000000001,"15min":0.193},"max":{"1min":2.3250000000000002,"5min":133.78999999999999,"15min":133.78999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.2989999999999999,"5min":4.6360000000000001,"15min":4.2050000000000001},"min":{"1min":0.26200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":10.039,"5min":133.69300000000001,"15min":133.69300000000001},"last":0.44800000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":5.5119999999999996,"5min":9.2569999999999997,"15min":5.3200000000000003},"min":{"1min":0.30099999999999999,"5min":0.24299999999999999,"15min":0.22800000000000001},"max":{"1min":64.738,"5min":315.73599999999999,"15min":315.73599999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":5.2629999999999999,"5min":9.6010000000000009,"15min":5.4169999999999998},"min":{"1min":0.26900000000000002,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":64.789000000000001,"5min":316.91500000000002,"15min":316.91500000000002},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 04:23:42 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.22,"15min":2.681},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23899999999999999},"max":{"1min":4.431,"5min":6.266,"15min":139.06899999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.292,"5min":1.2609999999999999,"15min":2.722},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.28599999999999998},"max":{"1min":4.4619999999999997,"5min":6.3040000000000003,"15min":139.173},"last":0.42199999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":2.0920000000000001,"15min":2.2719999999999998},"min":{"1min":0.308,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.0759999999999996,"5min":43.475999999999999,"15min":53.497999999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.8480000000000001,"15min":2.0960000000000001},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6059999999999999,"5min":43.369,"15min":53.460999999999999},"last":7.657}]},{"osd":4,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":2.5259999999999998,"15min":1.3560000000000001},"min":{"1min":0.31900000000000001,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":3.161,"5min":161.27500000000001,"15min":161.27500000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":2.6739999999999999,"15min":1.472},"min":{"1min":0.34799999999999998,"5min":0.27100000000000002,"15min":0.22600000000000001},"max":{"1min":0.92500000000000004,"5min":160.32599999999999,"15min":160.32599999999999},"last":0.372}]},{"osd":5,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.95899999999999996,"15min":1.03},"min":{"1min":0.36899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.6309999999999998,"5min":7.032,"15min":22.879000000000001},"last":0.44},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95999999999999996,"15min":1.0720000000000001},"min":{"1min":0.34799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.369,"5min":6.4109999999999996,"15min":23.006},"last":0.33500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.5950000000000002,"15min":1.5740000000000001},"min":{"1min":0.36799999999999999,"5min":0.32600000000000001,"15min":0.313},"max":{"1min":4.4939999999999998,"5min":161.39400000000001,"15min":161.39400000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.6259999999999999,"15min":1.6160000000000001},"min":{"1min":0.375,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":2.25,"5min":161.41200000000001,"15min":161.41200000000001},"last":0.41199999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089609,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2056908,"kb_used_data":1355908,"kb_used_omap":9,"kb_used_meta":700982,"kb_avail":91688244,"statfs":{"total":95995035648,"available":93888761856,"internally_reserved":0,"allocated":1388449792,"data_stored":1386499599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":717806569},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":5.9279999999999999,"15min":5.1459999999999999},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17199999999999999},"max":{"1min":8.2149999999999999,"5min":259.26799999999997,"15min":259.26799999999997},"last":1.474},{"interface":"front","average":{"1min":1.3340000000000001,"5min":5.8769999999999998,"15min":5.867},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":7.8929999999999998,"5min":258.46899999999999,"15min":258.46899999999999},"last":1.4419999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:23:47 2023","interfaces":[{"interface":"back","average":{"1min":14.804,"5min":5.133,"15min":3.3380000000000001},"min":{"1min":0.23699999999999999,"5min":0.20699999999999999,"15min":0.191},"max":{"1min":259.315,"5min":259.315,"15min":259.315},"last":1.3029999999999999},{"interface":"front","average":{"1min":15.205,"5min":5.6239999999999997,"15min":3.7759999999999998},"min":{"1min":0.25,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":259.733,"5min":259.733,"15min":259.733},"last":0.39200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":2.246,"15min":3.48},"min":{"1min":0.30499999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.401999999999999,"5min":19.640000000000001,"15min":138.87299999999999},"last":1.3360000000000001},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.169,"15min":3.944},"min":{"1min":0.29799999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.2629999999999999,"5min":12.353999999999999,"15min":138.49799999999999},"last":0.27300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":5.0069999999999997,"15min":3.028},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.1910000000000007,"5min":257.83300000000003,"15min":257.83300000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.121,"5min":4.4930000000000003,"15min":2.7869999999999999},"min":{"1min":0.35699999999999998,"5min":0.30499999999999999,"15min":0.25900000000000001},"max":{"1min":4.4809999999999999,"5min":257.863,"15min":257.863},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":2.1429999999999998,"15min":2.3170000000000002},"min":{"1min":0.313,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.2300000000000004,"5min":25.696999999999999,"15min":38.848999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":2.0910000000000002,"15min":2.3380000000000001},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":2.8849999999999998,"5min":21.893999999999998,"15min":39.593000000000004},"last":0.375}]},{"osd":6,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.6100000000000001,"15min":1.952},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.30299999999999999},"max":{"1min":3.0169999999999999,"5min":8.3529999999999998,"15min":25.931000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.593,"5min":1.633,"15min":1.6659999999999999},"min":{"1min":0.373,"5min":0.29999999999999999,"15min":0.29799999999999999},"max":{"1min":8.1630000000000003,"5min":19.440000000000001,"15min":19.440000000000001},"last":1.3779999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":6.8390000000000004,"15min":3.8420000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.9390000000000001,"5min":259.08499999999998,"15min":259.08499999999998},"last":0.44},{"interface":"front","average":{"1min":1.8009999999999999,"5min":6.6509999999999998,"15min":3.5110000000000001},"min":{"1min":0.40699999999999997,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":8.1899999999999995,"5min":259.13299999999998,"15min":259.13299999999998},"last":1.3620000000000001}]}]},{"osd":0,"up_from":84,"seq":360777253135,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1718060,"kb_used_data":1047204,"kb_used_omap":28,"kb_used_meta":670819,"kb_avail":92027092,"statfs":{"total":95995035648,"available":94235742208,"internally_reserved":0,"allocated":1072336896,"data_stored":1070297166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":686919504},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:24:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.8650000000000002,"15min":4.7210000000000001},"min":{"1min":0.224,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":5.9139999999999997,"5min":96.305000000000007,"15min":209.215},"last":2.976},{"interface":"front","average":{"1min":2.2130000000000001,"5min":4.4500000000000002,"15min":4.8159999999999998},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.383,"5min":95.861000000000004,"15min":120.934},"last":3.1850000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":3.8170000000000002,"15min":3.0819999999999999},"min":{"1min":0.22500000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.01,"5min":88.088999999999999,"15min":88.088999999999999},"last":4.4779999999999998},{"interface":"front","average":{"1min":2.3399999999999999,"5min":3.9540000000000002,"15min":3.2839999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":17.655000000000001,"5min":88.004999999999995,"15min":88.004999999999995},"last":3.694}]},{"osd":3,"last update":"Mon Oct 9 04:24:20 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":3.306,"15min":3.7719999999999998},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.192},"max":{"1min":5.8710000000000004,"5min":87.891999999999996,"15min":156.488},"last":3.0070000000000001},{"interface":"front","average":{"1min":2.278,"5min":3.3130000000000002,"15min":3.863},"min":{"1min":0.39300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":7.1379999999999999,"5min":88.147999999999996,"15min":156.50700000000001},"last":3.5019999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":3.1619999999999999,"15min":2.4489999999999998},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":7.468,"5min":88.058000000000007,"15min":88.058000000000007},"last":3.214},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.6339999999999999,"15min":2.3399999999999999},"min":{"1min":0.29899999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.6079999999999997,"5min":49.607999999999997,"15min":49.607999999999997},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.5329999999999999,"15min":3.2280000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":5.6050000000000004,"5min":49.526000000000003,"15min":156.96100000000001},"last":0.504},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.2029999999999998,"15min":2.7320000000000002},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":1.99,"5min":49.356999999999999,"15min":156.97300000000001},"last":3.0800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.742,"15min":2.4670000000000001},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.30599999999999999},"max":{"1min":2.984,"5min":7.8700000000000001,"15min":137.36099999999999},"last":3.1499999999999999},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.6110000000000002,"15min":2.9300000000000002},"min":{"1min":0.41199999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.3140000000000001,"5min":88.180000000000007,"15min":136.88999999999999},"last":3.266}]},{"osd":7,"last update":"Mon Oct 9 04:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.8859999999999999,"15min":2.4020000000000001},"min":{"1min":0.40899999999999997,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":4.1790000000000003,"5min":11.298999999999999,"15min":121.324},"last":3.1230000000000002},{"interface":"front","average":{"1min":1.883,"5min":2.9710000000000001,"15min":2.6120000000000001},"min":{"1min":0.435,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.8010000000000002,"5min":88.120000000000005,"15min":88.120000000000005},"last":3.2400000000000002}]}]},{"osd":2,"up_from":94,"seq":403726926083,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2451552,"kb_used_data":1743128,"kb_used_omap":81,"kb_used_meta":708334,"kb_avail":91293600,"statfs":{"total":95995035648,"available":93484646400,"internally_reserved":0,"allocated":1784963072,"data_stored":1782997351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":725334234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.718,"15min":3.7839999999999998},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17399999999999999},"max":{"1min":5.8499999999999996,"5min":21.792000000000002,"15min":81.385999999999996},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.9930000000000001,"5min":2.0329999999999999,"15min":3.7469999999999999},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.20000000000000001},"max":{"1min":8.0800000000000001,"5min":21.603000000000002,"15min":79.453999999999994},"last":1.6639999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":5.718,"15min":4.4560000000000004},"min":{"1min":0.28100000000000003,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":6.0529999999999999,"5min":158.13399999999999,"15min":158.13399999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.0590000000000002,"5min":5.6210000000000004,"15min":4.6139999999999999},"min":{"1min":0.314,"5min":0.253,"15min":0.20799999999999999},"max":{"1min":5.8840000000000003,"5min":161.12,"15min":161.12},"last":0.39200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.877,"15min":2.5720000000000001},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.20599999999999999},"max":{"1min":4.093,"5min":28.611999999999998,"15min":58.997},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.4940000000000002,"5min":2.54,"15min":2.7610000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23499999999999999},"max":{"1min":8.0549999999999997,"5min":27.733000000000001,"15min":66.135000000000005},"last":1.883}]},{"osd":4,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.9119999999999999,"15min":2.1349999999999998},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":4.7889999999999997,"5min":24.814,"15min":58.798000000000002},"last":0.438},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.2689999999999999,"15min":1.9119999999999999},"min":{"1min":0.34200000000000003,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":3.629,"5min":24.021000000000001,"15min":56.616},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":3.1299999999999999,"15min":2.7429999999999999},"min":{"1min":0.42699999999999999,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":7.984,"5min":82.030000000000001,"15min":82.030000000000001},"last":1.7030000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":3.3319999999999999,"15min":2.734},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":4.484,"5min":82.016000000000005,"15min":82.016000000000005},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.2210000000000001,"15min":2.4009999999999998},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.308},"max":{"1min":5.6269999999999998,"5min":7.3230000000000004,"15min":114.14},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.476,"5min":1.9970000000000001,"15min":2.6360000000000001},"min":{"1min":0.497,"5min":0.32100000000000001,"15min":0.26800000000000002},"max":{"1min":8.0280000000000005,"5min":15.456,"15min":114.123},"last":1.7669999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:23:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.3929999999999998,"15min":2.8679999999999999},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.33700000000000002},"max":{"1min":2.6930000000000001,"5min":101.675,"15min":117.32599999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":2.3599999999999999,"5min":3.177,"15min":3.145},"min":{"1min":0.54200000000000004,"5min":0.34499999999999997,"15min":0.34399999999999997},"max":{"1min":10.839,"5min":101.72499999999999,"15min":117.28700000000001},"last":1.7350000000000001}]}]},{"osd":3,"up_from":99,"seq":425201762557,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2811160,"kb_used_data":1935312,"kb_used_omap":7,"kb_used_meta":875832,"kb_avail":90933992,"statfs":{"total":95995035648,"available":93116407808,"internally_reserved":0,"allocated":1981759488,"data_stored":1979799555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":896852584},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.0379999999999998,"15min":4.1760000000000002},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.16900000000000001},"max":{"1min":9.3049999999999997,"5min":10.672000000000001,"15min":125.765},"last":1.911},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.738,"15min":4.7190000000000003},"min":{"1min":0.25700000000000001,"5min":0.185,"15min":0.185},"max":{"1min":4.7679999999999998,"5min":22.446999999999999,"15min":125.46899999999999},"last":1.246}]},{"osd":1,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":4.117,"15min":5.2699999999999996},"min":{"1min":0.22,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.7889999999999997,"5min":170.196,"15min":186.21299999999999},"last":1.119},{"interface":"front","average":{"1min":2.1819999999999999,"5min":3.9830000000000001,"15min":5.1689999999999996},"min":{"1min":0.23000000000000001,"5min":0.221,"15min":0.193},"max":{"1min":14.042,"5min":166.953,"15min":191.78},"last":1.966}]},{"osd":2,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.4910000000000001,"15min":2.8759999999999999},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.187},"max":{"1min":9.2720000000000002,"5min":9.2720000000000002,"15min":127.599},"last":3.9079999999999999},{"interface":"front","average":{"1min":2.1190000000000002,"5min":2.3130000000000002,"15min":4.101},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.214},"max":{"1min":12.032999999999999,"5min":12.032999999999999,"15min":129.98599999999999},"last":0.996}]},{"osd":4,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":3.6699999999999999,"15min":2.5579999999999998},"min":{"1min":0.34200000000000003,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":4.8280000000000003,"5min":211.84700000000001,"15min":211.84700000000001},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":3.8879999999999999,"15min":2.673},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.7149999999999999,"5min":211.17500000000001,"15min":211.17500000000001},"last":1.054}]},{"osd":5,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.036,"15min":2.161},"min":{"1min":0.35599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.6709999999999994,"5min":11.772,"15min":17.901},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.419,"15min":1.6919999999999999},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.248},"max":{"1min":7.7530000000000001,"5min":8.4670000000000005,"15min":18.068000000000001},"last":2.0510000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.218,"15min":1.5880000000000001},"min":{"1min":0.376,"5min":0.33500000000000002,"15min":0.26500000000000001},"max":{"1min":7.7050000000000001,"5min":7.7050000000000001,"15min":14.976000000000001},"last":2.0299999999999998},{"interface":"front","average":{"1min":1.288,"5min":1.6799999999999999,"15min":1.988},"min":{"1min":0.371,"5min":0.371,"15min":0.29999999999999999},"max":{"1min":4.7960000000000003,"5min":10.071,"15min":14.529},"last":1.1850000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":4.1429999999999998,"15min":2.7759999999999998},"min":{"1min":0.38,"5min":0.35399999999999998,"15min":0.34399999999999997},"max":{"1min":4.7290000000000001,"5min":212.62700000000001,"15min":212.62700000000001},"last":1.544},{"interface":"front","average":{"1min":2.1360000000000001,"5min":4.3620000000000001,"15min":2.956},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.7579999999999991,"5min":212.04599999999999,"15min":212.04599999999999},"last":1.0169999999999999}]}]},{"osd":4,"up_from":0,"seq":133143986573,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724937,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2605988,"kb_used_data":1713436,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":91139164,"statfs":{"total":95995035648,"available":93326503936,"internally_reserved":0,"allocated":1754558464,"data_stored":1753274628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.2480000000000002,"15min":4.9980000000000002},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.193},"max":{"1min":24.076000000000001,"5min":63.914999999999999,"15min":151.23599999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.4860000000000002,"5min":2.585,"15min":5.383},"min":{"1min":0.27500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":22.629999999999999,"5min":61.281999999999996,"15min":151.566},"last":0.22900000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":8.0739999999999998,"15min":5.4889999999999999},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":22.722999999999999,"5min":296.23099999999999,"15min":296.23099999999999},"last":0.24099999999999999},{"interface":"front","average":{"1min":2.5510000000000002,"5min":8.3330000000000002,"15min":5.7380000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":22.437000000000001,"5min":295.065,"15min":295.065},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":12.301,"5min":5.3419999999999996,"15min":3.827},"min":{"1min":0.36799999999999999,"5min":0.28199999999999997,"15min":0.24399999999999999},"max":{"1min":168.779,"5min":168.779,"15min":168.779},"last":0.36599999999999999},{"interface":"front","average":{"1min":11.641,"5min":5.0430000000000001,"15min":3.597},"min":{"1min":0.33000000000000002,"5min":0.27000000000000002,"15min":0.25600000000000001},"max":{"1min":168.03100000000001,"5min":168.03100000000001,"15min":168.03100000000001},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:24:07 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":2.274,"15min":2.093},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":8.9350000000000005,"5min":41,"15min":41},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.3280000000000001,"5min":2.258,"15min":2.1840000000000002},"min":{"1min":0.38,"5min":0.317,"15min":0.23499999999999999},"max":{"1min":8.9169999999999998,"5min":40.988,"15min":40.988},"last":0.35499999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.1000000000000001,"15min":0.98099999999999998},"min":{"1min":0.29399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.3740000000000001,"5min":9.2579999999999991,"15min":10.162000000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":1.125,"15min":0.94299999999999995},"min":{"1min":0.31,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":2.0339999999999998,"5min":7.5890000000000004,"15min":11.297000000000001},"last":0.31}]},{"osd":6,"last update":"Mon Oct 9 04:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":1.0229999999999999,"15min":1.341},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":0.91200000000000003,"5min":9.4090000000000007,"15min":123.239},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.99299999999999999,"15min":1.4399999999999999},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":2.0489999999999999,"5min":6.1689999999999996,"15min":123.014},"last":0.33600000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:23:49 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":1.236,"15min":1.01},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":2.8300000000000001,"5min":9.3800000000000008,"15min":12.096},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.252,"15min":1.0800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.307},"max":{"1min":3.302,"5min":9.3019999999999996,"15min":9.9760000000000009},"last":0.39000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1067061248,"data_stored":1067057176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1378156544,"data_stored":1378156544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1774059520,"data_stored":1774059520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1971392512,"data_stored":1971388440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2382888960,"data_stored":2382888960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1746337792,"data_stored":1746337792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1789333504,"data_stored":1789329432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1514799104,"data_stored":1514799104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:24:36.211 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:24:36.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-10-09T04:24:36.217 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:36.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:36.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:36.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:36.399 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:36.537 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:36.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:36.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:36.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:36.682 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:36.754 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:36.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:36.785 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:36.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:36.827 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:36.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:36.879 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:36.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:36.889 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:36.918 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:36.922 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:36.952 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:36.973 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:36.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:36.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:36.983 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:36.987 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.987 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.987 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.987 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.987 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.987 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.988 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:24:36.988 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:24:37.054 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:24:37.054 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:24:37.054 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:24:37.054 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:24:37.054 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:24:37.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:37.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:37 smithi181 ceph-mon[196979]: pgmap v1164: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4461/182889 objects degraded (2.439%); 25144/182889 objects misplaced (13.748%); 17 MiB/s, 271 objects/s recovering 2023-10-09T04:24:37.078 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3584893129' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:37.112 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:37.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:37.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:37 smithi163 ceph-mon[213841]: pgmap v1164: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4461/182889 objects degraded (2.439%); 25144/182889 objects misplaced (13.748%); 17 MiB/s, 271 objects/s recovering 2023-10-09T04:24:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3584893129' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:37 smithi163 ceph-mon[221067]: pgmap v1164: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4461/182889 objects degraded (2.439%); 25144/182889 objects misplaced (13.748%); 17 MiB/s, 271 objects/s recovering 2023-10-09T04:24:37.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3584893129' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:37.375 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:37.411 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:37.854 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:24:37.877 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:37.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:37.982 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:38.044 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:38.172 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:24:38.215 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:38 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4461/182889 objects degraded (2.439%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:38.216 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:38 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4461/182889 objects degraded (2.439%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:38.248 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:24:38.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:38 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4461/182889 objects degraded (2.439%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:39 smithi181 ceph-mon[196979]: pgmap v1165: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 4272/178575 objects degraded (2.392%); 24653/178575 objects misplaced (13.805%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:24:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:39 smithi163 ceph-mon[213841]: pgmap v1165: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 4272/178575 objects degraded (2.392%); 24653/178575 objects misplaced (13.805%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:24:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:39 smithi163 ceph-mon[221067]: pgmap v1165: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 370 op/s; 4272/178575 objects degraded (2.392%); 24653/178575 objects misplaced (13.805%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:24:39.673 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:39.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:39.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:39.753 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:39.785 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:39.821 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:39.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:39.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:39.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:39.922 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:24:39.922 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:24:39.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:24:39.922 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28475 9.1 9:8009e758:::benchmark_data_smithi163_464650_object28474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 372.11888563299999, 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.096956874, 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.923 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28475 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:24:39.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801180+0000", 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801209+0000", 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8646e-05 2023-10-09T04:24:39.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801226+0000", 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.715e-05 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801234+0000", 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3459999999999996e-06 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801291+0000", 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7438999999999997e-05 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801409+0000", 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011781399999999999 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801670+0000", 2023-10-09T04:24:39.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026062299999999997 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809884+0000", 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082140670000000002 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809890+0000", 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6460000000000004e-06 2023-10-09T04:24:39.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812682+0000", 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002791615 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812694+0000", 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.171e-05 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.884929+0000", 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0722356550000001 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.885962+0000", 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010329860000000001 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898005+0000", 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012042361999999999 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898138+0000", 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013387100000000001 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28492 9.1 9:80d8c0c4:::benchmark_data_smithi163_464650_object28491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 372.00789061699999, 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9871532940000001, 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28492 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.932 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912175+0000", 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912211+0000", 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6192000000000002e-05 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912224+0000", 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2965999999999999e-05 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912230+0000", 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6540000000000004e-06 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912283+0000", 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3115999999999998e-05 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912289+0000", 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2700000000000001e-06 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812717+0000", 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90042778800000001 2023-10-09T04:24:39.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812726+0000", 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5189999999999992e-06 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886022+0000", 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.073296604 2023-10-09T04:24:39.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886114+0000", 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1397999999999995e-05 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886285+0000", 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171807 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:39.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891737+0000", 2023-10-09T04:24:39.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005451661 2023-10-09T04:24:39.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898177+0000", 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064403500000000001 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898550+0000", 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037258199999999999 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898598+0000", 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8214000000000003e-05 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899330+0000", 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000731593 2023-10-09T04:24:39.942 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28496 9.1 9:834685fc:::benchmark_data_smithi163_464650_object28495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 372.00023986000002, 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9805368130000001, 2023-10-09T04:24:39.943 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28496 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919880+0000", 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3053000000000002e-05 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919893+0000", 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3439e-05 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919899+0000", 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.161e-06 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919957+0000", 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8118999999999997e-05 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919969+0000", 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2065000000000001e-05 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812748+0000", 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89277856099999997 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812756+0000", 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6079999999999998e-06 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887444+0000", 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.074688672 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887596+0000", 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000151533 2023-10-09T04:24:39.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887736+0000", 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013991200000000001 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:39.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891747+0000", 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040110479999999997 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898230+0000", 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064832809999999996 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899391+0000", 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001160863 2023-10-09T04:24:39.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899443+0000", 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1969000000000003e-05 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.900364+0000", 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00092131999999999999 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.953 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87402 9.d 9:b58a4af0:::benchmark_data_smithi163_464650_object87401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 314.02346254399998, 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9327528470000002, 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.954 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87402 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896603+0000", 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896662+0000", 2023-10-09T04:24:39.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8721999999999998e-05 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896672+0000", 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0342e-05 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896678+0000", 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5890000000000002e-06 2023-10-09T04:24:39.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897746+0000", 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001067901 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897884+0000", 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013837699999999999 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.898027+0000", 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014309899999999999 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.903780+0000", 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057525900000000001 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913854+0000", 2023-10-09T04:24:39.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010074411 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913865+0000", 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0667e-05 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904284+0000", 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99041863600000002 2023-10-09T04:24:39.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904295+0000", 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1622e-05 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829288+0000", 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9249930480000002 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829357+0000", 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9139000000000003e-05 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87427 9.d 9:b85b518c:::benchmark_data_smithi163_464650_object87426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.99032353400003, 2023-10-09T04:24:39.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9083351629999998, 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87427 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929742+0000", 2023-10-09T04:24:39.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929812+0000", 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9973000000000002e-05 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929833+0000", 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0372000000000001e-05 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929840+0000", 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0380000000000002e-06 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929955+0000", 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115658 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929964+0000", 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.02e-06 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904451+0000", 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97448704799999997 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904459+0000", 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:24:39.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829570+0000", 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9251109199999998 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829657+0000", 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6693000000000005e-05 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.968 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829772+0000", 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115109 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837839+0000", 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080666330000000001 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:39.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837847+0000", 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5720000000000003e-06 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837892+0000", 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4297e-05 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837929+0000", 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7622e-05 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837980+0000", 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0816000000000003e-05 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838079+0000", 2023-10-09T04:24:39.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8758000000000006e-05 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87460 9.d 9:be691015:::benchmark_data_smithi163_464650_object87459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:24:39.972 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.94939591899998, 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8737964319999998, 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87460 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970670+0000", 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970704+0000", 2023-10-09T04:24:39.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3708999999999999e-05 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970712+0000", 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2910000000000007e-06 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970718+0000", 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2909999999999998e-06 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971579+0000", 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086131599999999999 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971589+0000", 2023-10-09T04:24:39.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0108e-05 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904489+0000", 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93289962699999995 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904496+0000", 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3939999999999998e-06 2023-10-09T04:24:39.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832602+0000", 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9281065480000001 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832830+0000", 2023-10-09T04:24:39.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022731300000000001 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.833032+0000", 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202256 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837853+0000", 2023-10-09T04:24:39.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048209000000000004 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838118+0000", 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000265083 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838174+0000", 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5485999999999997e-05 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844377+0000", 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062031949999999999 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844410+0000", 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3266999999999998e-05 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844468+0000", 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7698000000000002e-05 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.983 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87469 9.d 9:b0d8b7f7:::benchmark_data_smithi163_464650_object87468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.94618726499999, 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8824357369999998, 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.984 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87469 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973879+0000", 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973909+0000", 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0063999999999999e-05 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973921+0000", 2023-10-09T04:24:39.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.204e-05 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973927+0000", 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2049999999999999e-06 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974432+0000", 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050552799999999999 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974441+0000", 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904518+0000", 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93007710099999996 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904525+0000", 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9280000000000004e-06 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835034+0000", 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9305090869999999 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835189+0000", 2023-10-09T04:24:39.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015497800000000001 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835319+0000", 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012983499999999999 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838122+0000", 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028034420000000002 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838149+0000", 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6636999999999999e-05 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838193+0000", 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4202999999999997e-05 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:39.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856143+0000", 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017949975 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856198+0000", 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4968e-05 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856316+0000", 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117589 2023-10-09T04:24:39.993 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87510 9.d 9:b1e25812:::benchmark_data_smithi163_464650_object87509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.896829852, 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8391414909999999, 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:39.994 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87510 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023236+0000", 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:39.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023264+0000", 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7639000000000001e-05 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023280+0000", 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6189999999999999e-05 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023285+0000", 2023-10-09T04:24:39.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4380000000000003e-06 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023338+0000", 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2713999999999997e-05 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023350+0000", 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1912999999999999e-05 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904548+0000", 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88119805299999998 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904555+0000", 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1690000000000003e-06 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:39.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838206+0000", 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9336506519999999 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838292+0000", 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6051999999999995e-05 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838426+0000", 2023-10-09T04:24:40.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013363900000000001 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844098+0000", 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056724699999999998 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844218+0000", 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012015000000000001 2023-10-09T04:24:40.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844519+0000", 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000300842 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862218+0000", 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017698935999999998 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862272+0000", 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4085999999999997e-05 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862379+0000", 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.003 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87520 9.d 9:b10313ad:::benchmark_data_smithi163_464650_object87519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.88820202699998, 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.830771683, 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.004 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87520 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031864+0000", 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031940+0000", 2023-10-09T04:24:40.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6348999999999997e-05 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031956+0000", 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5733e-05 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031962+0000", 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1330000000000004e-06 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032020+0000", 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8168999999999998e-05 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032031+0000", 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0546e-05 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904577+0000", 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87254678500000005 2023-10-09T04:24:40.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904584+0000", 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8279999999999997e-06 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841092+0000", 2023-10-09T04:24:40.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.936507684 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841230+0000", 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138245 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841385+0000", 2023-10-09T04:24:40.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015503399999999999 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844512+0000", 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031265519999999999 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850232+0000", 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057197300000000001 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850519+0000", 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000287342 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862488+0000", 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011968807999999999 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862529+0000", 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1087999999999997e-05 2023-10-09T04:24:40.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862637+0000", 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108094 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87523 9.d 9:b2b1ff3b:::benchmark_data_smithi163_464650_object87522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.88235327699999, 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8252149690000001, 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.016 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87523 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037713+0000", 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:40.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037758+0000", 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5336000000000001e-05 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037769+0000", 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1139e-05 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037774+0000", 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6249999999999998e-06 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037811+0000", 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7234999999999998e-05 2023-10-09T04:24:40.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037819+0000", 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8599999999999993e-06 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904623+0000", 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86680350299999998 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904630+0000", 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5780000000000001e-06 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844552+0000", 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9399218949999999 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844685+0000", 2023-10-09T04:24:40.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013337900000000001 2023-10-09T04:24:40.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844852+0000", 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166682 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850445+0000", 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055933379999999998 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850487+0000", 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1508000000000003e-05 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856104+0000", 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056174129999999999 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862788+0000", 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066833480000000004 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862832+0000", 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000002e-05 2023-10-09T04:24:40.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862929+0000", 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7040999999999995e-05 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87524 9.d 9:bf5a5ce2:::benchmark_data_smithi163_464650_object87523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:24:40.028 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.87962916599997, 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8227152589999998, 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87524 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040437+0000", 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040544+0000", 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010666399999999999 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040556+0000", 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697999999999999e-05 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040562+0000", 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0839999999999997e-06 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040647+0000", 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4971999999999999e-05 2023-10-09T04:24:40.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040656+0000", 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9479999999999997e-06 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904660+0000", 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86400389600000005 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904667+0000", 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847526+0000", 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9428587660000001 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847651+0000", 2023-10-09T04:24:40.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125136 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847838+0000", 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018702700000000001 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850552+0000", 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002713853 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856113+0000", 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055611580000000001 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862179+0000", 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060656729999999997 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863021+0000", 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000842759 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863060+0000", 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8272000000000002e-05 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863153+0000", 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3684000000000006e-05 2023-10-09T04:24:40.037 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87531 9.d 9:b4d1cf98:::benchmark_data_smithi163_464650_object87530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.87860165500001, 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.836533765, 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.038 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87531 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041465+0000", 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041513+0000", 2023-10-09T04:24:40.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8566999999999998e-05 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041522+0000", 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9020000000000005e-06 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041528+0000", 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:24:40.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041602+0000", 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4530000000000006e-05 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041611+0000", 2023-10-09T04:24:40.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2819999999999996e-06 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904688+0000", 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86307752000000004 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904695+0000", 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:24:40.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850572+0000", 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9458773759999999 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850685+0000", 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011277699999999999 2023-10-09T04:24:40.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850827+0000", 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014244099999999999 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862146+0000", 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011318701 2023-10-09T04:24:40.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862451+0000", 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030508599999999999 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862741+0000", 2023-10-09T04:24:40.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029011400000000001 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877820+0000", 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015078775000000001 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877869+0000", 2023-10-09T04:24:40.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9407999999999999e-05 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877999+0000", 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129868 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87536 9.d 9:b13d2384:::benchmark_data_smithi163_464650_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.86811924199998, 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.826285253, 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.049 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87536 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:24:40.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051947+0000", 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052032+0000", 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5885999999999997e-05 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052049+0000", 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6867e-05 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052056+0000", 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9349999999999997e-06 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052117+0000", 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0758999999999999e-05 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052127+0000", 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301e-05 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904719+0000", 2023-10-09T04:24:40.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85259186799999997 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904726+0000", 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7989999999999996e-06 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853627+0000", 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9489006930000001 2023-10-09T04:24:40.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853769+0000", 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142619 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853901+0000", 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205399999999999 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862421+0000", 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085194190000000003 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862735+0000", 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031399200000000003 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862747+0000", 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2238e-05 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878064+0000", 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015316585000000001 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878120+0000", 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6338999999999997e-05 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878233+0000", 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113268 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.058 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87542 9.d 9:b8ae8bf6:::benchmark_data_smithi163_464650_object87541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.85861766199997, 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.817112244, 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.059 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87542 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061448+0000", 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061495+0000", 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6932999999999999e-05 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061515+0000", 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061522+0000", 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8499999999999996e-06 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061580+0000", 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8037e-05 2023-10-09T04:24:40.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061591+0000", 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1099e-05 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904748+0000", 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84315744100000001 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904755+0000", 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7880000000000001e-06 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856351+0000", 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9515959619999999 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856435+0000", 2023-10-09T04:24:40.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3497000000000004e-05 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856590+0000", 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015579299999999999 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862697+0000", 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0061066360000000004 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862753+0000", 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6106000000000001e-05 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862991+0000", 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023799100000000001 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878381+0000", 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015390452000000001 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878432+0000", 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0779999999999998e-05 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878562+0000", 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012949399999999999 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.067 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87544 9.d 9:b2959611:::benchmark_data_smithi163_464650_object87543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.85847263699998, 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8172143809999999, 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.068 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87544 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:24:40.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061594+0000", 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061631+0000", 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6940000000000002e-05 2023-10-09T04:24:40.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061641+0000", 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5650000000000007e-06 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061645+0000", 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9400000000000001e-06 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061692+0000", 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6638000000000003e-05 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061705+0000", 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2525e-05 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904777+0000", 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84307196799999995 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904783+0000", 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859252+0000", 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.954468756 2023-10-09T04:24:40.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859377+0000", 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124423 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859542+0000", 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016522800000000001 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862758+0000", 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032164730000000001 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862963+0000", 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000205105 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871931+0000", 2023-10-09T04:24:40.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089675880000000003 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878651+0000", 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067196959999999998 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878706+0000", 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5047999999999999e-05 2023-10-09T04:24:40.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878809+0000", 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103266 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87579 9.d 9:b6955889:::benchmark_data_smithi163_464650_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.801175634, 2023-10-09T04:24:40.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.760215788, 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87579 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:24:40.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118890+0000", 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118943+0000", 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2917000000000002e-05 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118957+0000", 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3553000000000001e-05 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118963+0000", 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9329999999999998e-06 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119027+0000", 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4450999999999997e-05 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119039+0000", 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1657999999999999e-05 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904808+0000", 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78576894799999997 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904814+0000", 2023-10-09T04:24:40.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.472e-06 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863179+0000", 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.958364789 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863297+0000", 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011818 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863439+0000", 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000141836 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871995+0000", 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085565290000000002 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872008+0000", 2023-10-09T04:24:40.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2779000000000001e-05 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878306+0000", 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062979710000000003 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878882+0000", 2023-10-09T04:24:40.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057614199999999997 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878977+0000", 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4944000000000004e-05 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879107+0000", 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013005199999999999 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.086 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87597 9.d 9:ba722c6f:::benchmark_data_smithi163_464650_object87596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.77279691799998, 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.741504747, 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.087 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87597 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147306+0000", 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6227999999999999e-05 2023-10-09T04:24:40.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147315+0000", 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4420000000000004e-06 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147321+0000", 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3900000000000001e-06 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147373+0000", 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1922000000000002e-05 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147383+0000", 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8160000000000005e-06 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904836+0000", 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75745349399999995 2023-10-09T04:24:40.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904843+0000", 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9360000000000002e-06 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866166+0000", 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9613231419999999 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866285+0000", 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011870399999999999 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866435+0000", 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015030299999999999 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.875034+0000", 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085985039999999999 2023-10-09T04:24:40.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878346+0000", 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033125379999999999 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878607+0000", 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000261158 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888598+0000", 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099901480000000008 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888652+0000", 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4688999999999998e-05 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888775+0000", 2023-10-09T04:24:40.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012282100000000001 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87606 9.d 9:bd98b1c5:::benchmark_data_smithi163_464650_object87605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.75194751200002, 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7216060710000001, 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.096 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87606 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168119+0000", 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168170+0000", 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1724000000000001e-05 2023-10-09T04:24:40.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168183+0000", 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2322e-05 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168188+0000", 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.011e-06 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168245+0000", 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7197999999999997e-05 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168257+0000", 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1742999999999999e-05 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904867+0000", 2023-10-09T04:24:40.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.73660977999999999 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904873+0000", 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3450000000000002e-06 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.868939+0000", 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9640662180000001 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869059+0000", 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012025 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869200+0000", 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014040799999999999 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878017+0000", 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0088170209999999995 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878849+0000", 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00083247799999999999 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879166+0000", 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000316373 2023-10-09T04:24:40.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889551+0000", 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010385553 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889609+0000", 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-05 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889726+0000", 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116743 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87628 9.d 9:b09d3eb4:::benchmark_data_smithi163_464650_object87627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 313.66748686099999, 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6374180369999998, 2023-10-09T04:24:40.105 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87628 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252579+0000", 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252624+0000", 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5071999999999998e-05 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252637+0000", 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.272e-05 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252642+0000", 2023-10-09T04:24:40.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8999999999999997e-06 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252685+0000", 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3056000000000002e-05 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252691+0000", 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3790000000000003e-06 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905181+0000", 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.65248916000000001 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905191+0000", 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0231000000000001e-05 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872040+0000", 2023-10-09T04:24:40.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.966849243 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872156+0000", 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115611 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872308+0000", 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152707 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878257+0000", 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059484389999999998 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879226+0000", 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096947500000000002 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888556+0000", 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093301310000000002 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889825+0000", 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001268948 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889882+0000", 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6885999999999997e-05 2023-10-09T04:24:40.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889998+0000", 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011613899999999999 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:81266 10.10 10:0ba0e01a:::benchmark_data_smithi163_523108_object7250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-10-09T04:24:40.114 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 27.751551745, 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.55363772300000003, 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 81266 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:24:40.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168512+0000", 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999971 2023-10-09T04:24:40.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168529+0000", 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6223999999999999e-05 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168551+0000", 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2459e-05 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168562+0000", 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1326e-05 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.313120+0000", 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14455803 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:24:40.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.313133+0000", 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2466000000000001e-05 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718112+0000", 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.404979266 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718376+0000", 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026423799999999997 2023-10-09T04:24:40.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718728+0000", 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00035156 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.720984+0000", 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022562839999999999 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721631+0000", 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064677500000000004 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721710+0000", 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8672999999999997e-05 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721809+0000", 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8904e-05 2023-10-09T04:24:40.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.722153+0000", 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00034452899999999999 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:24:40.122 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:24:40.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:40.159 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:40.213 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:40.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:40.319 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:40.344 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:40.388 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:40.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:40.492 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:40.520 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:40.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-10-09T04:24:40.640 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:24:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:24:40] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2023-10-09T04:24:40.761 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:40.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:40.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:40.834 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:24:40.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:40.931 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:41 smithi181 ceph-mon[196979]: pgmap v1166: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 4272/177165 objects degraded (2.411%); 23499/177165 objects misplaced (13.264%); 14 MiB/s, 226 objects/s recovering 2023-10-09T04:24:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3829438217' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T04:24:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:41 smithi163 ceph-mon[213841]: pgmap v1166: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 4272/177165 objects degraded (2.411%); 23499/177165 objects misplaced (13.264%); 14 MiB/s, 226 objects/s recovering 2023-10-09T04:24:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3829438217' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T04:24:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:41 smithi163 ceph-mon[221067]: pgmap v1166: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 4272/177165 objects degraded (2.411%); 23499/177165 objects misplaced (13.264%); 14 MiB/s, 226 objects/s recovering 2023-10-09T04:24:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3829438217' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-10-09T04:24:41.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:24:42.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:42.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:42.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:42.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:42.226 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:42.276 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:42.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:42.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:42.411 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:42.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:42.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:42.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:42.588 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:43.042 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:43.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:43.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:24:43.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:43.220 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:43.299 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:43.316 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[221067]: pgmap v1167: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 4084/173916 objects degraded (2.348%); 22738/173916 objects misplaced (13.074%); 16 MiB/s, 256 objects/s recovering 2023-10-09T04:24:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4084/173916 objects degraded (2.348%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[221067]: osdmap e439: 8 total, 8 up, 7 in 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[221067]: failed to encode map e439 with expected crc 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[221067]: failed to encode map e439 with expected crc 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[213841]: pgmap v1167: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 4084/173916 objects degraded (2.348%); 22738/173916 objects misplaced (13.074%); 16 MiB/s, 256 objects/s recovering 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4084/173916 objects degraded (2.348%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[213841]: osdmap e439: 8 total, 8 up, 7 in 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[213841]: failed to encode map e439 with expected crc 2023-10-09T04:24:43.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:43 smithi163 ceph-mon[213841]: failed to encode map e439 with expected crc 2023-10-09T04:24:43.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:43.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:43.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:43.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:43 smithi181 ceph-mon[196979]: pgmap v1167: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 385 op/s; 4084/173916 objects degraded (2.348%); 22738/173916 objects misplaced (13.074%); 16 MiB/s, 256 objects/s recovering 2023-10-09T04:24:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:43 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4084/173916 objects degraded (2.348%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:43 smithi181 ceph-mon[196979]: osdmap e439: 8 total, 8 up, 7 in 2023-10-09T04:24:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:43 smithi181 ceph-mon[196979]: failed to encode map e439 with expected crc 2023-10-09T04:24:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:43 smithi181 ceph-mon[196979]: failed to encode map e439 with expected crc 2023-10-09T04:24:43.524 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:43.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:43.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:43.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:43.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:43.910 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:43.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:43.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:44.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:44.097 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:44.148 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:44.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:44.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:44.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[213841]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[213841]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[213841]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[213841]: osdmap e440: 8 total, 8 up, 7 in 2023-10-09T04:24:44.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[213841]: failed to encode map e440 with expected crc 2023-10-09T04:24:44.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[221067]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[221067]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[221067]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[221067]: osdmap e440: 8 total, 8 up, 7 in 2023-10-09T04:24:44.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:44 smithi163 ceph-mon[221067]: failed to encode map e440 with expected crc 2023-10-09T04:24:44.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:44.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:44 smithi181 ceph-mon[196979]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:44 smithi181 ceph-mon[196979]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:44 smithi181 ceph-mon[196979]: failed to encode map e439 with expected crc 2023-10-09T04:24:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:44 smithi181 ceph-mon[196979]: osdmap e440: 8 total, 8 up, 7 in 2023-10-09T04:24:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:44 smithi181 ceph-mon[196979]: failed to encode map e440 with expected crc 2023-10-09T04:24:44.328 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:45.071 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:24:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[213841]: failed to encode map e440 with expected crc 2023-10-09T04:24:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[213841]: failed to encode map e440 with expected crc 2023-10-09T04:24:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[213841]: pgmap v1170: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 4084/172545 objects degraded (2.367%); 22644/172545 objects misplaced (13.124%); 14 MiB/s, 231 objects/s recovering 2023-10-09T04:24:45.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1263269759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[221067]: failed to encode map e440 with expected crc 2023-10-09T04:24:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[221067]: failed to encode map e440 with expected crc 2023-10-09T04:24:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[221067]: pgmap v1170: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 4084/172545 objects degraded (2.367%); 22644/172545 objects misplaced (13.124%); 14 MiB/s, 231 objects/s recovering 2023-10-09T04:24:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1263269759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:45 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:45.494 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:45.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:45.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:45 smithi181 ceph-mon[196979]: failed to encode map e440 with expected crc 2023-10-09T04:24:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:45 smithi181 ceph-mon[196979]: failed to encode map e440 with expected crc 2023-10-09T04:24:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:45 smithi181 ceph-mon[196979]: pgmap v1170: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 4084/172545 objects degraded (2.367%); 22644/172545 objects misplaced (13.124%); 14 MiB/s, 231 objects/s recovering 2023-10-09T04:24:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:45 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1263269759' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:45 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:24:45.599 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:45.648 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: osdmap e441: 8 total, 8 up, 7 in 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[221067]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:46.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:24:46.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: osdmap e441: 8 total, 8 up, 7 in 2023-10-09T04:24:46.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:46 smithi163 ceph-mon[213841]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:24:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:24:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: osdmap e441: 8 total, 8 up, 7 in 2023-10-09T04:24:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:46 smithi181 ceph-mon[196979]: failed to encode map e441 with expected crc 2023-10-09T04:24:46.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:46.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:46.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:46.715 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:46.766 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:46.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:46.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:46.872 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:46.900 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:46.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:46.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:46.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:47.049 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:47.113 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:47 smithi163 ceph-mon[221067]: pgmap v1172: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 4084/171480 objects degraded (2.382%); 21533/171480 objects misplaced (12.557%); 16 MiB/s, 250 objects/s recovering 2023-10-09T04:24:47.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:47 smithi163 ceph-mon[213841]: pgmap v1172: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 4084/171480 objects degraded (2.382%); 21533/171480 objects misplaced (12.557%); 16 MiB/s, 250 objects/s recovering 2023-10-09T04:24:47.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:47 smithi181 ceph-mon[196979]: pgmap v1172: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 112 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 4084/171480 objects degraded (2.382%); 21533/171480 objects misplaced (12.557%); 16 MiB/s, 250 objects/s recovering 2023-10-09T04:24:47.523 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:47.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:47.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:47.654 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:47.697 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:47.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:47.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:47.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92665 10.d 10.840120fd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.628692+0000", 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.19681337700000001, 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19684218000000001, 2023-10-09T04:24:47.829 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92665 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.628692+0000", 2023-10-09T04:24:47.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.628692+0000", 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.628690+0000", 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.831 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.628698+0000", 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6310000000000003e-06 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.628716+0000", 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8050000000000002e-05 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.628725+0000", 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4950000000000008e-06 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92715 10.d 10.e10ca29d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.792965+0000", 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.032540366000000001, 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032627365999999998, 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.833 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92715 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.792965+0000", 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.792965+0000", 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.792963+0000", 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.792972+0000", 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3310000000000004e-06 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.792985+0000", 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3546e-05 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.792995+0000", 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7180000000000008e-06 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46073.0:92716 10.5 e441/433)", 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.793486+0000", 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.032019767999999997, 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.032155702000000001, 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.837 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.793486+0000", 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.793486+0000", 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.793485+0000", 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.793492+0000", 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4999999999999996e-06 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.793502+0000", 2023-10-09T04:24:47.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0251000000000001e-05 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.793508+0000", 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2500000000000003e-06 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92600 10.d 10.4456162d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.462173+0000", 2023-10-09T04:24:47.840 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.36333232399999998, 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.36349969799999998, 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92600 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.462173+0000", 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.462173+0000", 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.462172+0000", 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.462179+0000", 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9530000000000002e-06 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.462189+0000", 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0868e-05 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.462195+0000", 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.801e-06 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92666 10.d 10.7be4da2d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.633549+0000", 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.191956869, 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.19215713800000001, 2023-10-09T04:24:47.844 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92666 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.633549+0000", 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.633549+0000", 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.633547+0000", 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.633558+0000", 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1257000000000001e-05 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.633571+0000", 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2459e-05 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.633579+0000", 2023-10-09T04:24:47.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6640000000000004e-06 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46073.0:92591 10.d e441/415)", 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.802043+0000", 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.023462738, 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.02369572, 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.848 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802043+0000", 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802043+0000", 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802041+0000", 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802048+0000", 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1210000000000003e-06 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802065+0000", 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7224e-05 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802075+0000", 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0010000000000001e-05 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.851 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92709 10.d 10.1e65e5ed (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.766227+0000", 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.059278193999999999, 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.059540430999999998, 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.852 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92709 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.766227+0000", 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.766227+0000", 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.766226+0000", 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.766232+0000", 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6119999999999998e-06 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.766243+0000", 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.134e-05 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.766251+0000", 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5830000000000001e-06 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.855 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92667 10.d 10.b57a731d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.638471+0000", 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.18703473600000001, 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.187336109, 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.856 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92667 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.638471+0000", 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.638471+0000", 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.638469+0000", 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.638477+0000", 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8310000000000007e-06 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.638495+0000", 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7220000000000001e-05 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.638505+0000", 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0581000000000001e-05 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46073.0:92591 10.d e441/415)", 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.802355+0000", 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.023150727999999999, 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023487131000000001, 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802355+0000", 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802355+0000", 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802354+0000", 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:47.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802370+0000", 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5815e-05 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802383+0000", 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3488e-05 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.802390+0000", 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0199999999999997e-06 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92697 10.d 10.18730d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.733343+0000", 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.092162690000000005, 2023-10-09T04:24:47.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.092551289999999994, 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92697 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.733343+0000", 2023-10-09T04:24:47.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.733343+0000", 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.733341+0000", 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.733349+0000", 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6550000000000004e-06 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.733361+0000", 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2421e-05 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.733369+0000", 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0989999999999999e-06 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92608 10.d 10.ca41d59d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.493816+0000", 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.33168893399999999, 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.33212987599999999, 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.867 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92608 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.493816+0000", 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.868 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.493816+0000", 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.493815+0000", 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.493822+0000", 2023-10-09T04:24:47.869 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7000000000000002e-06 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.493841+0000", 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8913999999999999e-05 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.493848+0000", 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7649999999999993e-06 2023-10-09T04:24:47.870 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92603 10.d 10.5b4aebfd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.469470+0000", 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.35603538600000001, 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.35651587299999998, 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.871 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92603 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.469470+0000", 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.469470+0000", 2023-10-09T04:24:47.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.469469+0000", 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.469475+0000", 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7769999999999999e-06 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.469488+0000", 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3387000000000001e-05 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.469496+0000", 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5920000000000003e-06 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.874 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92543 10.d 10:b2cb3430:::benchmark_data_smithi163_523108_object18527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.290507+0000", 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.53499848100000003, 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.53554062999999996, 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.875 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92543 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.290507+0000", 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.290507+0000", 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.290506+0000", 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.290513+0000", 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.973e-06 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.290523+0000", 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0506000000000001e-05 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.290529+0000", 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4219999999999999e-06 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:24:47.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.750469+0000", 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.459940393 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.758554+0000", 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080850769999999995 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.770086+0000", 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011532420999999999 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.808518+0000", 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.038431472000000001 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.809173+0000", 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00065459399999999999 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:24:47.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.811417+0000", 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022448239999999999 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.816288+0000", 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048705290000000002 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.881 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92605 10.d 10.b9184a5d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.471938+0000", 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.35356748199999999, 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.35420747600000002, 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92605 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.882 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.471938+0000", 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.471938+0000", 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.471937+0000", 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.471941+0000", 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0799999999999999e-06 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.471949+0000", 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7530000000000001e-06 2023-10-09T04:24:47.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.471954+0000", 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46073.0:92543 10.d e441/415 ondisk, result = 0)", 2023-10-09T04:24:47.885 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.781690+0000", 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.043815378000000002, 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.044516497000000002, 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.781690+0000", 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.886 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.781690+0000", 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.781689+0000", 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.781694+0000", 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2969999999999997e-06 2023-10-09T04:24:47.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.781707+0000", 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3042e-05 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.781714+0000", 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8009999999999998e-06 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:24:47.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.809851+0000", 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028137180000000001 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.810611+0000", 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00076019200000000003 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92591 10.d 10:b5f149ec:::benchmark_data_smithi163_523108_object18575:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.889 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.428343+0000", 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.39716246100000002, 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.39792771300000002, 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92591 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.890 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.428343+0000", 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.428343+0000", 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.428342+0000", 2023-10-09T04:24:47.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.428349+0000", 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6310000000000003e-06 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.428361+0000", 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2216e-05 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.892 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.428369+0000", 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3339999999999996e-06 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.779980+0000", 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.35161110499999998 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.786641+0000", 2023-10-09T04:24:47.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0066615149999999998 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.798073+0000", 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011432055999999999 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.894 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(10.4 441/429 [PushOp(10:21961181:::benchmark_data_smithi163_523108_object41050:head, version: 400'2595, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:21961181:::benchmark_data_smithi163_523108_object41050:head@400'2595, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.820551+0000", 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0049548049999999996, 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0058187680000000002, 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.820551+0000", 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.820551+0000", 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.820549+0000", 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.820592+0000", 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2809000000000003e-05 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.820615+0000", 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3044000000000002e-05 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.820631+0000", 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6181e-05 2023-10-09T04:24:47.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.821096+0000", 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046512600000000001 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.821273+0000", 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017709799999999999 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.898 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:92631 10.d 10.e6d215bd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.548021+0000", 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.27748474499999998, 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27841170700000001, 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:24:47.899 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 92631 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.548021+0000", 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.548021+0000", 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.548019+0000", 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.548026+0000", 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8369999999999999e-06 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.548038+0000", 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2313999999999999e-05 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.548046+0000", 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7109999999999995e-06 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "backfill_remove(10.a e441 10:5f7cf5b9:::benchmark_data_smithi163_523108_object10299:head,388'661)", 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.826276+0000", 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 4294967295.9992294, 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000151483, 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826276+0000", 2023-10-09T04:24:47.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826276+0000", 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826275+0000", 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826283+0000", 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0790000000000001e-06 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826296+0000", 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2981e-05 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826305+0000", 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6139999999999992e-06 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826356+0000", 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1294000000000001e-05 2023-10-09T04:24:47.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826374+0000", 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8128999999999999e-05 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826428+0000", 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3390999999999998e-05 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.907 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "pg_backfill(progress 10.a e 441/441 lb 10:5f7cf5b9:::benchmark_data_smithi163_523108_object10299:head)", 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:47.826321+0000", 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 4294967295.9991846, 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00022389400000000001, 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:24:47.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826321+0000", 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826321+0000", 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826320+0000", 2023-10-09T04:24:47.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826342+0000", 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1503999999999999e-05 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826350+0000", 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1380000000000007e-06 2023-10-09T04:24:47.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826356+0000", 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4320000000000004e-06 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826491+0000", 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013505900000000001 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826508+0000", 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6191000000000001e-05 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:47.826545+0000", 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6885999999999999e-05 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:24:47.912 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:24:47.913 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 20 2023-10-09T04:24:47.913 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:24:47.913 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:48.073 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:48 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 4084/171480 objects degraded (2.382%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:48.073 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:48 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 4084/171480 objects degraded (2.382%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:48.144 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:24:48.144 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:24:48.144 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T04:24:48.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T04:24:48.151 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:48.183 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:48.197 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:48.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:48.223 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:48.254 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:48.288 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:48.303 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:48.322 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:48.365 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:48.434 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:48.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:48.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:48 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 4084/171480 objects degraded (2.382%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:48.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:48.548 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:48.548 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:48.548 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.548 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.548 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.548 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.549 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.549 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.549 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.549 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:24:48.563 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:48.613 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:48.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:48.718 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:48.770 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:49.074 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:49 smithi163 ceph-mon[213841]: pgmap v1173: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 113 active+clean, 9 active+remapped+backfill_wait; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 2795/167901 objects degraded (1.665%); 22157/167901 objects misplaced (13.196%); 13 MiB/s, 215 objects/s recovering 2023-10-09T04:24:49.074 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3769887051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:49.208 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:49.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:49.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:49.279 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:24:49.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:49 smithi163 ceph-mon[221067]: pgmap v1173: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 113 active+clean, 9 active+remapped+backfill_wait; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 2795/167901 objects degraded (1.665%); 22157/167901 objects misplaced (13.196%); 13 MiB/s, 215 objects/s recovering 2023-10-09T04:24:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3769887051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:49.364 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:49.413 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:24:49.497 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:49.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:49.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:49 smithi181 ceph-mon[196979]: pgmap v1173: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 113 active+clean, 9 active+remapped+backfill_wait; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 2795/167901 objects degraded (1.665%); 22157/167901 objects misplaced (13.196%); 13 MiB/s, 215 objects/s recovering 2023-10-09T04:24:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3769887051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:49.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:49.631 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:49.677 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:49.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:49.786 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:49.814 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:49.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:49.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:49.964 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:50.005 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:50.009 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.095 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:50.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:50.197 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.201 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:50.291 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:50.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[221067]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[221067]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[221067]: osdmap e442: 8 total, 8 up, 7 in 2023-10-09T04:24:50.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[221067]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[221067]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:50.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[213841]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[213841]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[213841]: osdmap e442: 8 total, 8 up, 7 in 2023-10-09T04:24:50.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[213841]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[213841]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.356 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:50.439 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:50.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:50.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:50 smithi181 ceph-mon[196979]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:50 smithi181 ceph-mon[196979]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:50 smithi181 ceph-mon[196979]: osdmap e442: 8 total, 8 up, 7 in 2023-10-09T04:24:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:50 smithi181 ceph-mon[196979]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:50 smithi181 ceph-mon[196979]: failed to encode map e442 with expected crc 2023-10-09T04:24:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:50.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:50.596 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:50.699 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:50.860 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:50.866 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:24:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:24:50] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-10-09T04:24:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:50.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:50.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:50.999 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:51.057 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:51.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:51.161 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[213841]: pgmap v1175: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 113 active+clean, 9 active+remapped+backfill_wait; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 2795/167082 objects degraded (1.673%); 21884/167082 objects misplaced (13.098%); 13 MiB/s, 204 objects/s recovering 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[213841]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[213841]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[213841]: osdmap e443: 8 total, 8 up, 7 in 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[213841]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[213841]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[221067]: pgmap v1175: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 113 active+clean, 9 active+remapped+backfill_wait; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 2795/167082 objects degraded (1.673%); 21884/167082 objects misplaced (13.098%); 13 MiB/s, 204 objects/s recovering 2023-10-09T04:24:51.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[221067]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[221067]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[221067]: osdmap e443: 8 total, 8 up, 7 in 2023-10-09T04:24:51.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[221067]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:51 smithi163 ceph-mon[221067]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.214 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:51.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:51 smithi181 ceph-mon[196979]: pgmap v1175: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 113 active+clean, 9 active+remapped+backfill_wait; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 303 op/s; 2795/167082 objects degraded (1.673%); 21884/167082 objects misplaced (13.098%); 13 MiB/s, 204 objects/s recovering 2023-10-09T04:24:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:51 smithi181 ceph-mon[196979]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:51 smithi181 ceph-mon[196979]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:51 smithi181 ceph-mon[196979]: osdmap e443: 8 total, 8 up, 7 in 2023-10-09T04:24:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:51 smithi181 ceph-mon[196979]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:51 smithi181 ceph-mon[196979]: failed to encode map e443 with expected crc 2023-10-09T04:24:51.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:24:52.442 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:52.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:52.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:52.551 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:52.578 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:52.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:52.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:52.737 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:52.782 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:52.940 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:52.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:52.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:52.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:53.063 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:53.157 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:53.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:53 smithi163 ceph-mon[221067]: pgmap v1177: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 114 active+clean, 8 active+remapped+backfill_wait; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 2727/164052 objects degraded (1.662%); 20266/164052 objects misplaced (12.353%); 18 MiB/s, 286 objects/s recovering 2023-10-09T04:24:53.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:53 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 2727/164052 objects degraded (1.662%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:53.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:53 smithi163 ceph-mon[213841]: pgmap v1177: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 114 active+clean, 8 active+remapped+backfill_wait; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 2727/164052 objects degraded (1.662%); 20266/164052 objects misplaced (12.353%); 18 MiB/s, 286 objects/s recovering 2023-10-09T04:24:53.321 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:53 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 2727/164052 objects degraded (1.662%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:53 smithi181 ceph-mon[196979]: pgmap v1177: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 114 active+clean, 8 active+remapped+backfill_wait; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 358 op/s; 2727/164052 objects degraded (1.662%); 20266/164052 objects misplaced (12.353%); 18 MiB/s, 286 objects/s recovering 2023-10-09T04:24:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:53 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 2727/164052 objects degraded (1.662%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:53.601 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:24:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:53.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:24:53.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:53.735 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:53.785 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:53.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:53.890 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:53.917 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:53.958 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:53.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:54.067 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:54.095 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:54.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:54.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:54.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:54.253 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:54.304 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:54.354 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:24:54.354 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:24:54.748 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:54.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:54.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:24:54.878 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:54.931 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:24:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:54.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:24:55.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:55.065 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:55.069 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:54 smithi163 conmon[213817]: 2023-10-09T04:24:54.623+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e443 definitely_dead 0 2023-10-09T04:24:55.118 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:55.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:55.132 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.4. 2023-10-09T04:24:55.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:55 smithi181 ceph-mon[196979]: pgmap v1178: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 114 active+clean, 8 active+remapped+backfill_wait; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 2727/162837 objects degraded (1.675%); 20167/162837 objects misplaced (12.385%); 15 MiB/s, 247 objects/s recovering 2023-10-09T04:24:55.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1209292279' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:24:55.137 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:55 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:24:55.225 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:24:55.255 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:55.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:55 smithi163 ceph-mon[221067]: pgmap v1178: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 114 active+clean, 8 active+remapped+backfill_wait; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 2727/162837 objects degraded (1.675%); 20167/162837 objects misplaced (12.385%); 15 MiB/s, 247 objects/s recovering 2023-10-09T04:24:55.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1209292279' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:24:55.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:55 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:24:55.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:55 smithi163 ceph-mon[213841]: pgmap v1178: 129 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 114 active+clean, 8 active+remapped+backfill_wait; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 2727/162837 objects degraded (1.675%); 20167/162837 objects misplaced (12.385%); 15 MiB/s, 247 objects/s recovering 2023-10-09T04:24:55.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1209292279' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:24:55.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:55 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:24:55.741 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:24:55.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:55.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:24:55.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:55.873 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:55.916 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:55.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:55.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:56.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:56.052 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[221067]: failed to encode map e444 with expected crc 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[221067]: failed to encode map e444 with expected crc 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[221067]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[221067]: pgmap v1180: 129 pgs: 26 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 92 active+clean, 8 active+remapped+backfill_wait; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19919/170835 objects misplaced (11.660%) 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[213841]: failed to encode map e444 with expected crc 2023-10-09T04:24:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[213841]: failed to encode map e444 with expected crc 2023-10-09T04:24:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:24:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[213841]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T04:24:56.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:56 smithi163 ceph-mon[213841]: pgmap v1180: 129 pgs: 26 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 92 active+clean, 8 active+remapped+backfill_wait; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19919/170835 objects misplaced (11.660%) 2023-10-09T04:24:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:56 smithi181 ceph-mon[196979]: failed to encode map e444 with expected crc 2023-10-09T04:24:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:56 smithi181 ceph-mon[196979]: failed to encode map e444 with expected crc 2023-10-09T04:24:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:56 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:24:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:56 smithi181 ceph-mon[196979]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T04:24:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:56 smithi181 ceph-mon[196979]: pgmap v1180: 129 pgs: 26 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 92 active+clean, 8 active+remapped+backfill_wait; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 19919/170835 objects misplaced (11.660%) 2023-10-09T04:24:56.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:56.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:56.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:24:56.807 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:24:56.828 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T04:24:56.898 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:56.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:24:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:56.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:24:57.058 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:57.095 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:24:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:24:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[221067]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:24:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:57 smithi163 ceph-mon[213841]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.411 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:57.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:57.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:57.417 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:24:57.447 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:24:57.469 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:24:57.483 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:24:57.506 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:24:57.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:57.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:24:57.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:24:57.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:57 smithi181 ceph-mon[196979]: failed to encode map e445 with expected crc 2023-10-09T04:24:57.526 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:24:57.548 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:57.549 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:24:57.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:24:57.593 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:24:57.632 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:57.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:57.736 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:24:57.763 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:57.861 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:24:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:57.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:24:57.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:24:57.992 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:58.038 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:24:58.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:58.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:24:58.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:58.173 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:58.221 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:58.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T04:24:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.6 continuing backfill to osd.0 from (0'0,443'5384] MIN to 443'5384 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.e starting backfill to osd.4 from (0'0,413'4291] MIN to 443'5927 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.14 starting backfill to osd.4 from (0'0,413'4325] MIN to 443'5483 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.16 continuing backfill to osd.0 from (0'0,443'5326] MIN to 443'5326 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.c continuing backfill to osd.0 from (388'400,443'5873] 10:361d1342:::benchmark_data_smithi163_523108_object38259:head to 443'5873 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.12 continuing backfill to osd.0 from (0'0,443'5511] MIN to 443'5511 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: 10.2 continuing backfill to osd.0 from (0'0,443'5537] MIN to 443'5537 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3206753342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e447 with expected crc 2023-10-09T04:24:58.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T04:24:58.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: failed to encode map e447 with expected crc 2023-10-09T04:24:58.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2015724978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:58.271 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:58 smithi181 ceph-mon[196979]: pgmap v1184: 129 pgs: 6 remapped+peering, 38 peering, 1 active+recovering+undersized+remapped, 78 active+clean, 6 active+remapped+backfill_wait; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12931/168651 objects misplaced (7.667%) 2023-10-09T04:24:58.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:24:58.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T04:24:58.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.6 continuing backfill to osd.0 from (0'0,443'5384] MIN to 443'5384 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.e starting backfill to osd.4 from (0'0,413'4291] MIN to 443'5927 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.14 starting backfill to osd.4 from (0'0,413'4325] MIN to 443'5483 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.16 continuing backfill to osd.0 from (0'0,443'5326] MIN to 443'5326 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.c continuing backfill to osd.0 from (388'400,443'5873] 10:361d1342:::benchmark_data_smithi163_523108_object38259:head to 443'5873 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.12 continuing backfill to osd.0 from (0'0,443'5511] MIN to 443'5511 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: 10.2 continuing backfill to osd.0 from (0'0,443'5537] MIN to 443'5537 2023-10-09T04:24:58.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3206753342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e447 with expected crc 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: failed to encode map e447 with expected crc 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2015724978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[213841]: pgmap v1184: 129 pgs: 6 remapped+peering, 38 peering, 1 active+recovering+undersized+remapped, 78 active+clean, 6 active+remapped+backfill_wait; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12931/168651 objects misplaced (7.667%) 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e446 with expected crc 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.6 continuing backfill to osd.0 from (0'0,443'5384] MIN to 443'5384 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.e starting backfill to osd.4 from (0'0,413'4291] MIN to 443'5927 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.14 starting backfill to osd.4 from (0'0,413'4325] MIN to 443'5483 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.16 continuing backfill to osd.0 from (0'0,443'5326] MIN to 443'5326 2023-10-09T04:24:58.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.c continuing backfill to osd.0 from (388'400,443'5873] 10:361d1342:::benchmark_data_smithi163_523108_object38259:head to 443'5873 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.12 continuing backfill to osd.0 from (0'0,443'5511] MIN to 443'5511 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: 10.2 continuing backfill to osd.0 from (0'0,443'5537] MIN to 443'5537 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3206753342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e447 with expected crc 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: failed to encode map e447 with expected crc 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2015724978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:24:58.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:58 smithi163 ceph-mon[221067]: pgmap v1184: 129 pgs: 6 remapped+peering, 38 peering, 1 active+recovering+undersized+remapped, 78 active+clean, 6 active+remapped+backfill_wait; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12931/168651 objects misplaced (7.667%) 2023-10-09T04:24:58.355 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:58.401 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:24:58.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:58.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:24:58.506 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:58.533 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:58.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:58.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:58.641 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:24:58.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:24:58.710 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:58.757 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:24:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:24:58.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:24:58.866 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:24:58.893 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: failed to encode map e448 with expected crc 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2015724978' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:59.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[221067]: failed to encode map e448 with expected crc 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: failed to encode map e448 with expected crc 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2015724978' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T04:24:59.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:24:59 smithi163 ceph-mon[213841]: failed to encode map e448 with expected crc 2023-10-09T04:24:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: failed to encode map e447 with expected crc 2023-10-09T04:24:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:24:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: failed to encode map e448 with expected crc 2023-10-09T04:24:59.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2015724978' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:24:59.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T04:24:59.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:24:59 smithi181 ceph-mon[196979]: failed to encode map e448 with expected crc 2023-10-09T04:24:59.998 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:00.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:00.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:00.102 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:00.129 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:00.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:00.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:00.284 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:00.311 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:00.359 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:00.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:00.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[221067]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[221067]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[221067]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[221067]: pgmap v1186: 129 pgs: 2 activating+undersized+remapped, 6 remapped+peering, 35 peering, 1 active+recovering+undersized+remapped, 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.6 MiB/s wr, 992 op/s; 1583/162222 objects degraded (0.976%); 15485/162222 objects misplaced (9.546%); 159 MiB/s, 2.55k objects/s recovering 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[213841]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[213841]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[213841]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:00 smithi163 ceph-mon[213841]: pgmap v1186: 129 pgs: 2 activating+undersized+remapped, 6 remapped+peering, 35 peering, 1 active+recovering+undersized+remapped, 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.6 MiB/s wr, 992 op/s; 1583/162222 objects degraded (0.976%); 15485/162222 objects misplaced (9.546%); 159 MiB/s, 2.55k objects/s recovering 2023-10-09T04:25:00.495 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:00.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:00 smithi181 ceph-mon[196979]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:00 smithi181 ceph-mon[196979]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:00 smithi181 ceph-mon[196979]: failed to encode map e448 with expected crc 2023-10-09T04:25:00.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:00 smithi181 ceph-mon[196979]: pgmap v1186: 129 pgs: 2 activating+undersized+remapped, 6 remapped+peering, 35 peering, 1 active+recovering+undersized+remapped, 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.6 MiB/s wr, 992 op/s; 1583/162222 objects degraded (0.976%); 15485/162222 objects misplaced (9.546%); 159 MiB/s, 2.55k objects/s recovering 2023-10-09T04:25:00.591 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:00.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:00.695 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:00.722 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:00.767 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:00.769 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:25:00] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-10-09T04:25:00.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:00.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97814 9.8 9:1e2e17f2:::benchmark_data_smithi163_464650_object97813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:25:00.788 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 319.074920819, 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0107446179999999, 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97814 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.789 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710188+0000", 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710223+0000", 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5763000000000003e-05 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710243+0000", 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9536000000000001e-05 2023-10-09T04:25:00.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710260+0000", 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7271000000000001e-05 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710304+0000", 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3745000000000002e-05 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710446+0000", 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014181999999999999 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710552+0000", 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010643500000000001 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.713530+0000", 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029773289999999999 2023-10-09T04:25:00.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717690+0000", 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041607049999999998 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.742122+0000", 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024431905 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.742134+0000", 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1542000000000001e-05 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506765+0000", 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76463178499999995 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506780+0000", 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4603000000000001e-05 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.720842+0000", 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2140615079999999 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.720934+0000", 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2258999999999999e-05 2023-10-09T04:25:00.796 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97818 9.8 9:11ed80ac:::benchmark_data_smithi163_464650_object97817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 319.070489662, 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0067211999999999, 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.797 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97818 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:25:00.798 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714618+0000", 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714643+0000", 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4170999999999999e-05 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.799 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714655+0000", 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2646000000000001e-05 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714667+0000", 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1525000000000001e-05 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717718+0000", 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003051649 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.800 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717853+0000", 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013490299999999999 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717991+0000", 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137861 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.801 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.723501+0000", 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055102470000000002 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.726686+0000", 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031843050000000001 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.743057+0000", 2023-10-09T04:25:00.802 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016371198999999999 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.743067+0000", 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0241000000000001e-05 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506817+0000", 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76374998100000002 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.803 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506828+0000", 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0964999999999999e-05 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721244+0000", 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2144160310000001 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721342+0000", 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7343999999999995e-05 2023-10-09T04:25:00.804 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97832 9.8 9:1e55ad63:::benchmark_data_smithi163_464650_object97831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 319.05443382300001, 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.99079312, 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.805 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97832 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.806 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730695+0000", 2023-10-09T04:25:00.807 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8825999999999999e-05 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730702+0000", 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1600000000000001e-06 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730707+0000", 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8840000000000002e-06 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.808 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730736+0000", 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9824000000000001e-05 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730895+0000", 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015838500000000001 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.731058+0000", 2023-10-09T04:25:00.809 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016319699999999999 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.736028+0000", 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049698779999999996 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745215+0000", 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091867310000000001 2023-10-09T04:25:00.810 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745227+0000", 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1897e-05 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507474+0000", 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76224778000000004 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.811 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507486+0000", 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1707e-05 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721391+0000", 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.213904777 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721469+0000", 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8479999999999994e-05 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.812 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97836 9.8 9:11733f2c:::benchmark_data_smithi163_464650_object97835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 319.04101792400002, 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.996319304, 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.813 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97836 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.814 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744090+0000", 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744115+0000", 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4522999999999999e-05 2023-10-09T04:25:00.815 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744128+0000", 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3434999999999999e-05 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744138+0000", 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0118e-05 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745249+0000", 2023-10-09T04:25:00.816 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011104089999999999 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745254+0000", 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8379999999999998e-06 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507512+0000", 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.762257929 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.817 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507520+0000", 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9240000000000007e-06 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721492+0000", 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.213971412 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.818 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721620+0000", 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000128085 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721817+0000", 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000196839 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725055+0000", 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032383199999999998 2023-10-09T04:25:00.819 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732136+0000", 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070812619999999996 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732296+0000", 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015994499999999999 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.820 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740237+0000", 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079407409999999994 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740292+0000", 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5081999999999999e-05 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740411+0000", 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119461 2023-10-09T04:25:00.821 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97838 9.8 9:1b793ee6:::benchmark_data_smithi163_464650_object97837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 319.04091541899999, 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0048248380000002, 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.822 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97838 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.823 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744194+0000", 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744242+0000", 2023-10-09T04:25:00.824 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8263999999999997e-05 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744253+0000", 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0642000000000001e-05 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744259+0000", 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3040000000000001e-06 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.825 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745272+0000", 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010125170000000001 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745278+0000", 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.127e-06 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.826 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507541+0000", 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76226340000000004 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507549+0000", 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6620000000000005e-06 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725084+0000", 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2175345179999999 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.827 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725227+0000", 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000143406 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725417+0000", 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189956 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732176+0000", 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067589599999999996 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.828 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732240+0000", 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4319000000000005e-05 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740463+0000", 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082230529999999993 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.748849+0000", 2023-10-09T04:25:00.829 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083855000000000006 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.748896+0000", 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7463000000000003e-05 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749019+0000", 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012314499999999999 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.830 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97866 9.8 9:1cea5b31:::benchmark_data_smithi163_464650_object97865:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 319.009408011, 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.973609446, 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.831 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97866 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.832 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775722+0000", 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0279999999999999e-05 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.833 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775729+0000", 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775734+0000", 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1359999999999996e-06 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775796+0000", 2023-10-09T04:25:00.834 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2526000000000004e-05 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775811+0000", 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4799e-05 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507569+0000", 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.73175752299999997 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.835 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507576+0000", 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728482+0000", 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2209053270000001 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728643+0000", 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016166299999999999 2023-10-09T04:25:00.836 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728783+0000", 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000139185 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732184+0000", 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034017660000000001 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.837 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732369+0000", 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018494 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740499+0000", 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0081298159999999998 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749147+0000", 2023-10-09T04:25:00.838 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086481760000000005 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749200+0000", 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2543999999999997e-05 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749311+0000", 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111702 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.839 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97872 9.8 9:10a54521:::benchmark_data_smithi163_464650_object97871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 318.99940479200001, 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.963828709, 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.840 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97872 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.841 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785703+0000", 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785769+0000", 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6533999999999997e-05 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.842 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785789+0000", 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785799+0000", 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5719999999999992e-06 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785910+0000", 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111128 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.843 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785917+0000", 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3359999999999997e-06 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507597+0000", 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.72167993600000002 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507606+0000", 2023-10-09T04:25:00.844 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1589999999999993e-06 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732411+0000", 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.224805452 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732576+0000", 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016478 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.845 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732767+0000", 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019163200000000001 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740425+0000", 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076572660000000002 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740534+0000", 2023-10-09T04:25:00.846 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109341 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749083+0000", 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085489380000000007 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749377+0000", 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029441100000000001 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.847 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749423+0000", 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5676000000000001e-05 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749534+0000", 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110911 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97874 9.8 9:191c8b34:::benchmark_data_smithi163_464650_object97873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:25:00.848 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 318.99929527400002, 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9835905089999999, 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97874 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.849 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.850 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785814+0000", 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785855+0000", 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0932e-05 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785866+0000", 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0762e-05 2023-10-09T04:25:00.851 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785871+0000", 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7550000000000002e-06 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785935+0000", 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4851999999999995e-05 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.852 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785940+0000", 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2119999999999997e-06 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507627+0000", 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.72168717900000001 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507635+0000", 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3699999999999995e-06 2023-10-09T04:25:00.853 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736110+0000", 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2284743970000001 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736281+0000", 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017120699999999999 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.854 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736480+0000", 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019965799999999999 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740574+0000", 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0040938049999999998 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.855 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749102+0000", 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085278049999999994 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753493+0000", 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043906470000000001 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769243+0000", 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015749984000000002 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.856 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769290+0000", 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7129000000000002e-05 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769405+0000", 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115487 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.857 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97892 9.8 9:1aaa6a58:::benchmark_data_smithi163_464650_object97891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 318.97647993499999, 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9609627970000001, 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97892 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.858 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808629+0000", 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:25:00.859 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808672+0000", 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3235000000000002e-05 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808684+0000", 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2139e-05 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.860 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808692+0000", 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6420000000000007e-06 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808752+0000", 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9707999999999998e-05 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808760+0000", 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3569999999999998e-06 2023-10-09T04:25:00.861 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507656+0000", 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.69889576899999994 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507663+0000", 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6120000000000001e-06 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.862 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740602+0000", 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2329382529999999 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740744+0000", 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000142769 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740917+0000", 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017210500000000001 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.863 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753421+0000", 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012504678999999999 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753524+0000", 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102972 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.864 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.761433+0000", 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079086420000000004 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769451+0000", 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080184650000000007 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769498+0000", 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6409999999999998e-05 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.865 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769593+0000", 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5168999999999996e-05 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97904 9.8 9:1009d856:::benchmark_data_smithi163_464650_object97903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 318.96346493300001, 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9481932799999999, 2023-10-09T04:25:00.866 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97904 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.867 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821643+0000", 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821692+0000", 2023-10-09T04:25:00.868 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9016e-05 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821706+0000", 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3781000000000001e-05 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821713+0000", 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3730000000000004e-06 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.869 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821759+0000", 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5587999999999998e-05 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821769+0000", 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9369999999999999e-06 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507684+0000", 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.68591571799999995 2023-10-09T04:25:00.870 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507693+0000", 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1689999999999992e-06 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745049+0000", 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2373563329999999 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.871 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745227+0000", 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017849599999999999 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745426+0000", 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019907700000000001 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753455+0000", 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080286279999999995 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.872 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753558+0000", 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010336399999999999 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.761441+0000", 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078828909999999995 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769701+0000", 2023-10-09T04:25:00.873 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082594929999999997 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769746+0000", 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4821999999999999e-05 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769838+0000", 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2744000000000005e-05 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.874 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18935 10.b 10:da4c922f:::benchmark_data_smithi163_523108_object18934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.804695194, 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0962053709999999, 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.875 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18935 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.876 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980413+0000", 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980478+0000", 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4868000000000003e-05 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.877 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980501+0000", 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3396000000000001e-05 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980530+0000", 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9082999999999999e-05 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980605+0000", 2023-10-09T04:25:00.878 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5334999999999996e-05 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980613+0000", 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6529999999999994e-06 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985700+0000", 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050872310000000002 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.879 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985710+0000", 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0077e-05 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.054356+0000", 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0686453579999999 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.880 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.054995+0000", 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00063910900000000001 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.055173+0000", 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000178314 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058584+0000", 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034106679999999999 2023-10-09T04:25:00.881 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058630+0000", 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6400999999999998e-05 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060716+0000", 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020859139999999999 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.882 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.075527+0000", 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014810489 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.075578+0000", 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0995999999999998e-05 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076620+0000", 2023-10-09T04:25:00.883 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001042608 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18945 10.b 10:db72c492:::benchmark_data_smithi163_523108_object18944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.80455369699999, 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.097054776, 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.884 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18945 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.885 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980610+0000", 2023-10-09T04:25:00.886 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4863e-05 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980622+0000", 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1056000000000001e-05 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980628+0000", 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4520000000000002e-06 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.887 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980665+0000", 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7508000000000001e-05 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980671+0000", 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1889999999999998e-06 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985732+0000", 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005061767 2023-10-09T04:25:00.888 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985739+0000", 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8000000000000001e-06 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056187+0000", 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0704480949999999 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.889 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056380+0000", 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019281100000000001 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056554+0000", 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017387899999999999 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058639+0000", 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002084776 2023-10-09T04:25:00.890 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058684+0000", 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5627000000000002e-05 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060847+0000", 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002162298 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.891 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076681+0000", 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015834559000000002 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076730+0000", 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8708000000000002e-05 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077611+0000", 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088110100000000002 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.892 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18948 10.b 10:d3e53c2e:::benchmark_data_smithi163_523108_object18947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 153.80447515, 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0978110240000001, 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.893 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18948 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.894 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980634+0000", 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980675+0000", 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0608999999999997e-05 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.895 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980682+0000", 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7939999999999997e-06 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980686+0000", 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0589999999999996e-06 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.981834+0000", 2023-10-09T04:25:00.896 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011482910000000001 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.981840+0000", 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3459999999999999e-06 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985759+0000", 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039191210000000002 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.897 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985767+0000", 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4109999999999999e-06 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057486+0000", 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.071719098 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057617+0000", 2023-10-09T04:25:00.898 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013104999999999999 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057754+0000", 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137408 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058774+0000", 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001020137 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.899 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058886+0000", 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111945 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060884+0000", 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001997619 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077700+0000", 2023-10-09T04:25:00.900 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016816296000000001 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077760+0000", 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0031999999999997e-05 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.078446+0000", 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068554800000000002 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.901 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57312 10.8 10.a8007c28 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.334509+0000", 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 115.450601338, 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99230618599999998, 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.902 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57312 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334509+0000", 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334509+0000", 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.903 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334507+0000", 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334532+0000", 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5179999999999999e-05 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.904 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334542+0000", 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5999999999999996e-06 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334551+0000", 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1279999999999997e-06 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334879+0000", 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032774200000000001 2023-10-09T04:25:00.905 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334890+0000", 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0776e-05 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326799+0000", 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99190986299999995 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.906 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326815+0000", 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5413e-05 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57318 10.8 10.a56ae5f8 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 115.450547001, 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99228850099999999, 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.907 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57318 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.908 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334583+0000", 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0534e-05 2023-10-09T04:25:00.909 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334590+0000", 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1060000000000002e-06 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334596+0000", 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.694e-06 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.910 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334912+0000", 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031580300000000003 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334918+0000", 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7220000000000002e-06 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.911 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326841+0000", 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99192297100000004 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326852+0000", 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0145e-05 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.912 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57319 10.a 10.930077a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.334601+0000", 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 115.450509512, 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99232309100000005, 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57319 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.913 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334601+0000", 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334601+0000", 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.914 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334600+0000", 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334621+0000", 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0503999999999999e-05 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334628+0000", 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.968e-06 2023-10-09T04:25:00.915 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334634+0000", 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4999999999999996e-06 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334953+0000", 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000319303 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.916 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334961+0000", 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9719999999999992e-06 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326912+0000", 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99195034800000004 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326924+0000", 2023-10-09T04:25:00.917 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1855000000000001e-05 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:81169 10.d 10:b588674c:::benchmark_data_smithi163_523108_object7153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:11.752484+0000", 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 49.032625676000002, 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.81037368899999995, 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.918 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 81169 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:11.752484+0000", 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.919 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:11.752484+0000", 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:11.752482+0000", 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:11.752491+0000", 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3580000000000003e-06 2023-10-09T04:25:00.920 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:11.752508+0000", 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7683999999999999e-05 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:11.752518+0000", 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3360000000000001e-06 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.921 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.379772+0000", 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.62725468699999998 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.395435+0000", 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015662421999999999 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.418840+0000", 2023-10-09T04:25:00.922 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.023404872 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.463418+0000", 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.044578171999999999 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.514410+0000", 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.050991917999999997 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.923 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.548905+0000", 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034495476999999997 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.551979+0000", 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030738599999999999 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:12.562858+0000", 2023-10-09T04:25:00.924 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010878851 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(10.d 434/415 [PushOp(10:b0c90586:::benchmark_data_smithi163_523108_object20642:head, version: 392'1242, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:b0c90586:::benchmark_data_smithi163_523108_object20642:head@392'1242, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 38.205970671999999, 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.445266852, 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:25:00.925 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.926 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579201+0000", 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2989000000000003e-05 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579216+0000", 2023-10-09T04:25:00.927 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4229999999999999e-05 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579227+0000", 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1024e-05 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:25.003833+0000", 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.424606394 2023-10-09T04:25:00.928 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:25.004367+0000", 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053335800000000005 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:25.024406+0000", 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020039699000000001 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.929 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:91037 10.14 10:2b87ac6f:::benchmark_data_smithi163_523108_object17021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:42.555201+0000", 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.229908681000001, 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1080779919999999, 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 91037 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.930 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555201+0000", 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555201+0000", 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.931 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555200+0000", 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555206+0000", 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7009999999999999e-06 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555219+0000", 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2549e-05 2023-10-09T04:25:00.932 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555226+0000", 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2289999999999997e-06 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.559168+0000", 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039416590000000001 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.933 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.559289+0000", 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121478 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.559613+0000", 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032357399999999998 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.934 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.562434+0000", 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028212699999999999 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.562439+0000", 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8949999999999997e-06 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.662683+0000", 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.100244059 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.935 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.662697+0000", 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3648999999999999e-05 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663224+0000", 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0005272300000001 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.936 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663279+0000", 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5408000000000002e-05 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:91049 10.14 10:2bda9338:::benchmark_data_smithi163_523108_object17033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:42.680082+0000", 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 18.105027908, 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99102864800000001, 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:25:00.937 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 91049 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680082+0000", 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.938 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680082+0000", 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680081+0000", 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680087+0000", 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5880000000000003e-06 2023-10-09T04:25:00.939 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680097+0000", 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.842e-06 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680102+0000", 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3090000000000003e-06 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.940 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680717+0000", 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061489499999999996 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680732+0000", 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.473e-05 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663300+0000", 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98256744699999998 2023-10-09T04:25:00.941 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663401+0000", 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101123 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663699+0000", 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029785499999999997 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.942 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.669391+0000", 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005692679 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.670991+0000", 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015994620000000001 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.671025+0000", 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3853000000000003e-05 2023-10-09T04:25:00.943 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.671067+0000", 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2527000000000001e-05 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.671111+0000", 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3696999999999999e-05 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.944 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:25:00.945 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:25:00.945 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:25:00.945 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:00.973 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:01.026 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:01.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:01.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:01.160 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:01.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:01.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:01.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:25:01.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:01.494 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:01.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:01.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:01.650 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:01.791 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:01.816 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:01.828 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:25:01.828 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:25:01.829 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:25:01.829 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:25:01.880 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:01.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:01.988 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:02.015 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:02.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:25:02.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:02.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:02.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:02.170 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:02.201 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:02.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:25:02.356 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:02.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:02.463 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:02.494 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:02.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:02.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:02.647 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:02.677 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:02.730 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:02.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:02.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: pgmap v1188: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 MiB/s wr, 1.09k op/s; 7748/158739 objects degraded (4.881%); 19446/158739 objects misplaced (12.250%); 143 MiB/s, 2 keys/s, 2.34k objects/s recovering 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1583/162222 objects degraded (0.976%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:25:02.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:02 smithi181 ceph-mon[196979]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T04:25:02.836 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:02.863 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:02.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: pgmap v1188: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 MiB/s wr, 1.09k op/s; 7748/158739 objects degraded (4.881%); 19446/158739 objects misplaced (12.250%); 143 MiB/s, 2 keys/s, 2.34k objects/s recovering 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1583/162222 objects degraded (0.976%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:25:02.996 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[221067]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: failed to encode map e449 with expected crc 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: pgmap v1188: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 MiB/s wr, 1.09k op/s; 7748/158739 objects degraded (4.881%); 19446/158739 objects misplaced (12.250%); 143 MiB/s, 2 keys/s, 2.34k objects/s recovering 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1583/162222 objects degraded (0.976%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:25:02.997 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:02 smithi163 ceph-mon[213841]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T04:25:02.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:02.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:03.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:03.127 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:03.179 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:03.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:03.220 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:03.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:03.311 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:03.312 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:03.357 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:03.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:03.421 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:03.463 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:03.491 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:03.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:03.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:03.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:03.644 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:03.684 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:03.744 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:03.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:03 smithi181 ceph-mon[196979]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:03 smithi181 ceph-mon[196979]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.750 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:03 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:25:03.751 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:03 smithi181 ceph-mon[196979]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.751 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:03 smithi181 ceph-mon[196979]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:03.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:03.783 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:25:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:25:03.784 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:25:03.784 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:25:03.852 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[213841]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[213841]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.887 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[213841]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[213841]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[221067]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[221067]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[221067]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.888 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:03 smithi163 ceph-mon[221067]: failed to encode map e450 with expected crc 2023-10-09T04:25:03.892 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:04.653 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: pgmap v1191: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11 MiB/s wr, 963 op/s; 7748/158724 objects degraded (4.881%); 19446/158724 objects misplaced (12.251%); 130 MiB/s, 1 keys/s, 2.12k objects/s recovering 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:04.654 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: failed to encode map e451 with expected crc 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: pgmap v1191: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11 MiB/s wr, 963 op/s; 7748/158724 objects degraded (4.881%); 19446/158724 objects misplaced (12.251%); 130 MiB/s, 1 keys/s, 2.12k objects/s recovering 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:04.655 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:05.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:05.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: failed to encode map e451 with expected crc 2023-10-09T04:25:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: failed to encode map e451 with expected crc 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: failed to encode map e451 with expected crc 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: failed to encode map e451 with expected crc 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: failed to encode map e451 with expected crc 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: pgmap v1191: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11 MiB/s wr, 963 op/s; 7748/158724 objects degraded (4.881%); 19446/158724 objects misplaced (12.251%); 130 MiB/s, 1 keys/s, 2.12k objects/s recovering 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:05.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:05.143 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:05.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:25:05.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[221067]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[221067]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[221067]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T04:25:05.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[221067]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.772 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[221067]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[213841]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[213841]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[213841]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T04:25:05.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[213841]: failed to encode map e452 with expected crc 2023-10-09T04:25:05.773 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:05 smithi163 ceph-mon[213841]: failed to encode map e452 with expected crc 2023-10-09T04:25:06.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:06.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:05 smithi181 ceph-mon[196979]: failed to encode map e452 with expected crc 2023-10-09T04:25:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:05 smithi181 ceph-mon[196979]: failed to encode map e452 with expected crc 2023-10-09T04:25:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:05 smithi181 ceph-mon[196979]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T04:25:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:05 smithi181 ceph-mon[196979]: failed to encode map e452 with expected crc 2023-10-09T04:25:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:05 smithi181 ceph-mon[196979]: failed to encode map e452 with expected crc 2023-10-09T04:25:06.112 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:06.139 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:06.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:06 smithi163 ceph-mon[213841]: pgmap v1193: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 7748/158562 objects degraded (4.886%); 19396/158562 objects misplaced (12.232%) 2023-10-09T04:25:06.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:06 smithi163 ceph-mon[221067]: pgmap v1193: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 7748/158562 objects degraded (4.886%); 19396/158562 objects misplaced (12.232%) 2023-10-09T04:25:06.901 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:06.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:07.009 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:06 smithi181 ceph-mon[196979]: pgmap v1193: 129 pgs: 1 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 109 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 7748/158562 objects degraded (4.886%); 19396/158562 objects misplaced (12.232%) 2023-10-09T04:25:07.042 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:07.111 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:07.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:07.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:07.252 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:07.271 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:07.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:07.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:07.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:07.485 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:07.578 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:07.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:07.686 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:07.732 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:07 smithi181 ceph-mon[196979]: 10.7 deep-scrub starts 2023-10-09T04:25:07.732 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:07 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 7748/158562 objects degraded (4.886%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:07.737 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:07.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:07.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:07 smithi163 ceph-mon[213841]: 10.7 deep-scrub starts 2023-10-09T04:25:07.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:07 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 7748/158562 objects degraded (4.886%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:07 smithi163 ceph-mon[221067]: 10.7 deep-scrub starts 2023-10-09T04:25:07.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:07 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 7748/158562 objects degraded (4.886%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:07.846 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:07.872 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:07.899 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:25:07.903 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:25:07.906 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:07.931 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:07.932 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:07.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:07.982 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:07.991 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:08.015 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:08.046 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:08.052 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.053 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:25:08.056 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:08.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:08.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:08.203 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:08.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:08.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:08.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:08.409 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:08.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2614537213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:08.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[213841]: from='client.56077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:08.674 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[213841]: pgmap v1194: 129 pgs: 1 active+recovering+undersized+remapped, 114 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 5310/156834 objects degraded (3.386%); 19593/156834 objects misplaced (12.493%); 0 B/s, 64 objects/s recovering 2023-10-09T04:25:08.675 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1958324256' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:25:08.681 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:25:08.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:08.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:08.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:08.915 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:08.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2614537213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:08.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:08 smithi181 ceph-mon[196979]: from='client.56077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:08.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:08 smithi181 ceph-mon[196979]: pgmap v1194: 129 pgs: 1 active+recovering+undersized+remapped, 114 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 5310/156834 objects degraded (3.386%); 19593/156834 objects misplaced (12.493%); 0 B/s, 64 objects/s recovering 2023-10-09T04:25:08.947 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1958324256' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:25:08.951 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:09.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:09.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2614537213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[221067]: from='client.56077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:25:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[221067]: pgmap v1194: 129 pgs: 1 active+recovering+undersized+remapped, 114 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 5310/156834 objects degraded (3.386%); 19593/156834 objects misplaced (12.493%); 0 B/s, 64 objects/s recovering 2023-10-09T04:25:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1958324256' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:25:09.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:09.186 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:09.264 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:09.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:09.374 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:09.408 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:09.585 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1193,"stamp":"2023-10-09T04:25:05.955290+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3462856933,"num_objects":52854,"num_object_clones":0,"num_object_copies":158562,"num_objects_missing_on_primary":1555,"num_objects_missing":1555,"num_objects_degraded":7748,"num_objects_misplaced":19396,"num_objects_unfound":0,"num_objects_dirty":52854,"num_whiteouts":0,"num_read":105,"num_read_kb":196,"num_write":95310,"num_write_kb":4737375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33108,"num_bytes_recovered":2151278823,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148511,"ondisk_log_size":148511,"up":387,"acting":378,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":413,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17599576,"kb_used_data":11882776,"kb_used_omap":126,"kb_used_meta":5716609,"kb_avail":732361640,"statfs":{"total":767960285184,"available":749938319360,"internally_reserved":0,"allocated":12167962624,"data_stored":12154399406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":5853808251},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-3866624,"num_objects":-59,"num_object_clones":0,"num_object_copies":-177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-50,"num_objects_unfound":0,"num_objects_dirty":-59,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":59,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.314535"},"pg_stats":[{"pgid":"10.13","version":"446'5470","reported_seq":5566,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650302+0000","last_change":"2023-10-09T04:24:22.451557+0000","last_active":"2023-10-09T04:24:57.650302+0000","last_peered":"2023-10-09T04:24:57.650302+0000","last_clean":"2023-10-09T04:24:57.650302+0000","last_became_active":"2023-10-09T04:24:22.446790+0000","last_became_peered":"2023-10-09T04:24:22.446790+0000","last_unstale":"2023-10-09T04:24:57.650302+0000","last_undegraded":"2023-10-09T04:24:57.650302+0000","last_fullsized":"2023-10-09T04:24:57.650302+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107151360,"num_objects":1635,"num_object_clones":0,"num_object_copies":4905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2951,"num_write_kb":146752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"80'161","reported_seq":572,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.648878+0000","last_change":"2023-10-09T04:19:13.371216+0000","last_active":"2023-10-09T04:24:57.648878+0000","last_peered":"2023-10-09T04:24:57.648878+0000","last_clean":"2023-10-09T04:24:57.648878+0000","last_became_active":"2023-10-09T04:18:04.684429+0000","last_became_peered":"2023-10-09T04:18:04.684429+0000","last_unstale":"2023-10-09T04:24:57.648878+0000","last_undegraded":"2023-10-09T04:24:57.648878+0000","last_fullsized":"2023-10-09T04:24:57.648878+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:53.942632+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645738+0000","last_change":"2023-10-09T04:24:56.189047+0000","last_active":"2023-10-09T04:24:57.645738+0000","last_peered":"2023-10-09T04:24:57.645738+0000","last_clean":"2023-10-09T04:24:57.645738+0000","last_became_active":"2023-10-09T04:24:56.181560+0000","last_became_peered":"2023-10-09T04:24:56.181560+0000","last_unstale":"2023-10-09T04:24:57.645738+0000","last_undegraded":"2023-10-09T04:24:57.645738+0000","last_fullsized":"2023-10-09T04:24:57.645738+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:39.246218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T04:24:56.172256+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:24:56.172256+0000","last_peered":"2023-10-09T04:24:56.172256+0000","last_clean":"2023-10-09T04:24:56.172256+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:24:56.172256+0000","last_undegraded":"2023-10-09T04:24:56.172256+0000","last_fullsized":"2023-10-09T04:24:56.172256+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.10","version":"451'5484","reported_seq":13875,"reported_epoch":451,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:25:03.658633+0000","last_change":"2023-10-09T04:24:57.254084+0000","last_active":"2023-10-09T04:25:03.658633+0000","last_peered":"2023-10-09T04:25:03.658633+0000","last_clean":"2023-10-09T04:23:21.324155+0000","last_became_active":"2023-10-09T04:24:57.247132+0000","last_became_peered":"2023-10-09T04:24:57.247132+0000","last_unstale":"2023-10-09T04:25:03.658633+0000","last_undegraded":"2023-10-09T04:24:57.147915+0000","last_fullsized":"2023-10-09T04:24:57.143441+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107642880,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":806,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2966,"num_write_kb":147456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2078,"num_bytes_recovered":136183808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1502},{"shards":"1,6,7","objects":141}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":645,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.734627+0000","last_change":"2023-10-09T04:24:58.734627+0000","last_active":"2023-10-09T04:24:58.734627+0000","last_peered":"2023-10-09T04:24:58.734627+0000","last_clean":"2023-10-09T04:24:58.734627+0000","last_became_active":"2023-10-09T04:24:58.734248+0000","last_became_peered":"2023-10-09T04:24:58.734248+0000","last_unstale":"2023-10-09T04:24:58.734627+0000","last_undegraded":"2023-10-09T04:24:58.734627+0000","last_fullsized":"2023-10-09T04:24:58.734627+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:59.934563+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":458,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.659344+0000","last_change":"2023-10-09T04:24:56.158228+0000","last_active":"2023-10-09T04:25:03.659344+0000","last_peered":"2023-10-09T04:25:03.659344+0000","last_clean":"2023-10-09T04:25:03.659344+0000","last_became_active":"2023-10-09T04:24:56.157946+0000","last_became_peered":"2023-10-09T04:24:56.157946+0000","last_unstale":"2023-10-09T04:25:03.659344+0000","last_undegraded":"2023-10-09T04:25:03.659344+0000","last_fullsized":"2023-10-09T04:25:03.659344+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:37.279836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":51,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646112+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:24:57.646112+0000","last_peered":"2023-10-09T04:24:57.646112+0000","last_clean":"2023-10-09T04:24:57.646112+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:24:57.646112+0000","last_undegraded":"2023-10-09T04:24:57.646112+0000","last_fullsized":"2023-10-09T04:24:57.646112+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"443'5791","reported_seq":9028,"reported_epoch":447,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:24:57.643823+0000","last_change":"2023-10-09T04:24:57.252191+0000","last_active":"2023-10-09T04:24:57.643823+0000","last_peered":"2023-10-09T04:24:57.643823+0000","last_clean":"2023-10-09T04:22:25.940554+0000","last_became_active":"2023-10-09T04:24:57.244686+0000","last_became_peered":"2023-10-09T04:24:57.244686+0000","last_unstale":"2023-10-09T04:24:57.643823+0000","last_undegraded":"2023-10-09T04:24:57.209985+0000","last_fullsized":"2023-10-09T04:24:57.146869+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111214592,"num_objects":1697,"num_object_clones":0,"num_object_copies":5091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1571,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3130,"num_write_kb":154464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":971,"num_bytes_recovered":63635456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[7,0],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,1,7","objects":178},{"shards":"0,7","objects":837},{"shards":"1,4,7","objects":682}],"blocked_by":[],"up_primary":4,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":430,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.732163+0000","last_change":"2023-10-09T04:24:58.732163+0000","last_active":"2023-10-09T04:24:58.732163+0000","last_peered":"2023-10-09T04:24:58.732163+0000","last_clean":"2023-10-09T04:24:58.732163+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:24:58.732163+0000","last_undegraded":"2023-10-09T04:24:58.732163+0000","last_fullsized":"2023-10-09T04:24:58.732163+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.767969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.767969+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":51,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.641980+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:24:57.641980+0000","last_peered":"2023-10-09T04:24:57.641980+0000","last_clean":"2023-10-09T04:24:57.641980+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:24:57.641980+0000","last_undegraded":"2023-10-09T04:24:57.641980+0000","last_fullsized":"2023-10-09T04:24:57.641980+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":401,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650576+0000","last_change":"2023-10-09T04:22:13.634165+0000","last_active":"2023-10-09T04:24:57.650576+0000","last_peered":"2023-10-09T04:24:57.650576+0000","last_clean":"2023-10-09T04:24:57.650576+0000","last_became_active":"2023-10-09T04:22:13.632882+0000","last_became_peered":"2023-10-09T04:22:13.632882+0000","last_unstale":"2023-10-09T04:24:57.650576+0000","last_undegraded":"2023-10-09T04:24:57.650576+0000","last_fullsized":"2023-10-09T04:24:57.650576+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:28.354644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":847,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642701+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:24:57.642701+0000","last_peered":"2023-10-09T04:24:57.642701+0000","last_clean":"2023-10-09T04:24:57.642701+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:24:57.642701+0000","last_undegraded":"2023-10-09T04:24:57.642701+0000","last_fullsized":"2023-10-09T04:24:57.642701+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646273+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:24:57.646273+0000","last_peered":"2023-10-09T04:24:57.646273+0000","last_clean":"2023-10-09T04:24:57.646273+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:24:57.646273+0000","last_undegraded":"2023-10-09T04:24:57.646273+0000","last_fullsized":"2023-10-09T04:24:57.646273+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.657712+0000","last_change":"2023-10-09T04:22:13.630073+0000","last_active":"2023-10-09T04:25:03.657712+0000","last_peered":"2023-10-09T04:25:03.657712+0000","last_clean":"2023-10-09T04:25:03.657712+0000","last_became_active":"2023-10-09T04:22:13.629719+0000","last_became_peered":"2023-10-09T04:22:13.629719+0000","last_unstale":"2023-10-09T04:25:03.657712+0000","last_undegraded":"2023-10-09T04:25:03.657712+0000","last_fullsized":"2023-10-09T04:25:03.657712+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:03.686524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":708,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.732303+0000","last_change":"2023-10-09T04:24:58.732303+0000","last_active":"2023-10-09T04:24:58.732303+0000","last_peered":"2023-10-09T04:24:58.732303+0000","last_clean":"2023-10-09T04:24:58.732303+0000","last_became_active":"2023-10-09T04:24:58.731844+0000","last_became_peered":"2023-10-09T04:24:58.731844+0000","last_unstale":"2023-10-09T04:24:58.732303+0000","last_undegraded":"2023-10-09T04:24:58.732303+0000","last_fullsized":"2023-10-09T04:24:58.732303+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":407,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650840+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:24:57.650840+0000","last_peered":"2023-10-09T04:24:57.650840+0000","last_clean":"2023-10-09T04:24:57.650840+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:24:57.650840+0000","last_undegraded":"2023-10-09T04:24:57.650840+0000","last_fullsized":"2023-10-09T04:24:57.650840+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":94,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.658389+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:25:03.658389+0000","last_peered":"2023-10-09T04:25:03.658389+0000","last_clean":"2023-10-09T04:25:03.658389+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:25:03.658389+0000","last_undegraded":"2023-10-09T04:25:03.658389+0000","last_fullsized":"2023-10-09T04:25:03.658389+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":652,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650422+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:24:57.650422+0000","last_peered":"2023-10-09T04:24:57.650422+0000","last_clean":"2023-10-09T04:24:57.650422+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:24:57.650422+0000","last_undegraded":"2023-10-09T04:24:57.650422+0000","last_fullsized":"2023-10-09T04:24:57.650422+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":522,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642974+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:24:57.642974+0000","last_peered":"2023-10-09T04:24:57.642974+0000","last_clean":"2023-10-09T04:24:57.642974+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:24:57.642974+0000","last_undegraded":"2023-10-09T04:24:57.642974+0000","last_fullsized":"2023-10-09T04:24:57.642974+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.639847+0000","last_change":"2023-10-09T04:22:13.629778+0000","last_active":"2023-10-09T04:24:58.639847+0000","last_peered":"2023-10-09T04:24:58.639847+0000","last_clean":"2023-10-09T04:24:58.639847+0000","last_became_active":"2023-10-09T04:22:13.629473+0000","last_became_peered":"2023-10-09T04:22:13.629473+0000","last_unstale":"2023-10-09T04:24:58.639847+0000","last_undegraded":"2023-10-09T04:24:58.639847+0000","last_fullsized":"2023-10-09T04:24:58.639847+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:42.753957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":628,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T04:25:02.647288+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:25:02.647288+0000","last_peered":"2023-10-09T04:25:02.647288+0000","last_clean":"2023-10-09T04:25:02.647288+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:25:02.647288+0000","last_undegraded":"2023-10-09T04:25:02.647288+0000","last_fullsized":"2023-10-09T04:25:02.647288+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":23,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645297+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:24:57.645297+0000","last_peered":"2023-10-09T04:24:57.645297+0000","last_clean":"2023-10-09T04:24:57.645297+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:24:57.645297+0000","last_undegraded":"2023-10-09T04:24:57.645297+0000","last_fullsized":"2023-10-09T04:24:57.645297+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644399+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:24:57.644399+0000","last_peered":"2023-10-09T04:24:57.644399+0000","last_clean":"2023-10-09T04:24:57.644399+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:24:57.644399+0000","last_undegraded":"2023-10-09T04:24:57.644399+0000","last_fullsized":"2023-10-09T04:24:57.644399+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":466,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645641+0000","last_change":"2023-10-09T04:23:08.629976+0000","last_active":"2023-10-09T04:24:57.645641+0000","last_peered":"2023-10-09T04:24:57.645641+0000","last_clean":"2023-10-09T04:24:57.645641+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:24:57.645641+0000","last_undegraded":"2023-10-09T04:24:57.645641+0000","last_fullsized":"2023-10-09T04:24:57.645641+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.694404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.694404+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":473,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642527+0000","last_change":"2023-10-09T04:24:56.210458+0000","last_active":"2023-10-09T04:24:57.642527+0000","last_peered":"2023-10-09T04:24:57.642527+0000","last_clean":"2023-10-09T04:24:57.642527+0000","last_became_active":"2023-10-09T04:24:56.182765+0000","last_became_peered":"2023-10-09T04:24:56.182765+0000","last_unstale":"2023-10-09T04:24:57.642527+0000","last_undegraded":"2023-10-09T04:24:57.642527+0000","last_fullsized":"2023-10-09T04:24:57.642527+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:41.651696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":223,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.641742+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:24:57.641742+0000","last_peered":"2023-10-09T04:24:57.641742+0000","last_clean":"2023-10-09T04:24:57.641742+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:24:57.641742+0000","last_undegraded":"2023-10-09T04:24:57.641742+0000","last_fullsized":"2023-10-09T04:24:57.641742+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":612,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.648970+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:24:57.648970+0000","last_peered":"2023-10-09T04:24:57.648970+0000","last_clean":"2023-10-09T04:24:57.648970+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:24:57.648970+0000","last_undegraded":"2023-10-09T04:24:57.648970+0000","last_fullsized":"2023-10-09T04:24:57.648970+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645649+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:24:57.645649+0000","last_peered":"2023-10-09T04:24:57.645649+0000","last_clean":"2023-10-09T04:24:57.645649+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:24:57.645649+0000","last_undegraded":"2023-10-09T04:24:57.645649+0000","last_fullsized":"2023-10-09T04:24:57.645649+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":426,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649935+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:24:57.649935+0000","last_peered":"2023-10-09T04:24:57.649935+0000","last_clean":"2023-10-09T04:24:57.649935+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:24:57.649935+0000","last_undegraded":"2023-10-09T04:24:57.649935+0000","last_fullsized":"2023-10-09T04:24:57.649935+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"451'5521","reported_seq":8685,"reported_epoch":451,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:25:04.566881+0000","last_change":"2023-10-09T04:24:57.253694+0000","last_active":"2023-10-09T04:25:04.566881+0000","last_peered":"2023-10-09T04:25:04.566881+0000","last_clean":"2023-10-09T04:23:05.325743+0000","last_became_active":"2023-10-09T04:24:14.642895+0000","last_became_peered":"2023-10-09T04:24:14.642895+0000","last_unstale":"2023-10-09T04:25:04.566881+0000","last_undegraded":"2023-10-09T04:25:04.566881+0000","last_fullsized":"2023-10-09T04:25:04.566881+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106594304,"num_objects":1627,"num_object_clones":0,"num_object_copies":4881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1336,"num_objects_unfound":0,"num_objects_dirty":1627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2985,"num_write_kb":147552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":901,"num_bytes_recovered":59015168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1627}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":596,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.658053+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:25:03.658053+0000","last_peered":"2023-10-09T04:25:03.658053+0000","last_clean":"2023-10-09T04:25:03.658053+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:25:03.658053+0000","last_undegraded":"2023-10-09T04:25:03.658053+0000","last_fullsized":"2023-10-09T04:25:03.658053+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644868+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:24:57.644868+0000","last_peered":"2023-10-09T04:24:57.644868+0000","last_clean":"2023-10-09T04:24:57.644868+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:24:57.644868+0000","last_undegraded":"2023-10-09T04:24:57.644868+0000","last_fullsized":"2023-10-09T04:24:57.644868+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646043+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:24:57.646043+0000","last_peered":"2023-10-09T04:24:57.646043+0000","last_clean":"2023-10-09T04:24:57.646043+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:24:57.646043+0000","last_undegraded":"2023-10-09T04:24:57.646043+0000","last_fullsized":"2023-10-09T04:24:57.646043+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"446'5638","reported_seq":5724,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649531+0000","last_change":"2023-10-09T04:24:12.658965+0000","last_active":"2023-10-09T04:24:57.649531+0000","last_peered":"2023-10-09T04:24:57.649531+0000","last_clean":"2023-10-09T04:24:57.649531+0000","last_became_active":"2023-10-09T04:24:12.655745+0000","last_became_peered":"2023-10-09T04:24:12.655745+0000","last_unstale":"2023-10-09T04:24:57.649531+0000","last_undegraded":"2023-10-09T04:24:57.649531+0000","last_fullsized":"2023-10-09T04:24:57.649531+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3050,"num_write_kb":150688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":612,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649985+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:24:57.649985+0000","last_peered":"2023-10-09T04:24:57.649985+0000","last_clean":"2023-10-09T04:24:57.649985+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:24:57.649985+0000","last_undegraded":"2023-10-09T04:24:57.649985+0000","last_fullsized":"2023-10-09T04:24:57.649985+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":61,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642122+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:24:57.642122+0000","last_peered":"2023-10-09T04:24:57.642122+0000","last_clean":"2023-10-09T04:24:57.642122+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:24:57.642122+0000","last_undegraded":"2023-10-09T04:24:57.642122+0000","last_fullsized":"2023-10-09T04:24:57.642122+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":116,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.658407+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:25:03.658407+0000","last_peered":"2023-10-09T04:25:03.658407+0000","last_clean":"2023-10-09T04:25:03.658407+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:25:03.658407+0000","last_undegraded":"2023-10-09T04:25:03.658407+0000","last_fullsized":"2023-10-09T04:25:03.658407+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"443'5384","reported_seq":7802,"reported_epoch":447,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:57.650502+0000","last_change":"2023-10-09T04:24:57.179102+0000","last_active":"2023-10-09T04:24:57.650502+0000","last_peered":"2023-10-09T04:24:57.650502+0000","last_clean":"2023-10-09T04:23:05.250360+0000","last_became_active":"2023-10-09T04:24:57.172839+0000","last_became_peered":"2023-10-09T04:24:57.172839+0000","last_unstale":"2023-10-09T04:24:57.650502+0000","last_undegraded":"2023-10-09T04:24:57.650502+0000","last_fullsized":"2023-10-09T04:24:57.650502+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104923136,"num_objects":1601,"num_object_clones":0,"num_object_copies":4803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":1601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2913,"num_write_kb":144448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":567,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1601}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":876,"reported_epoch":447,"state":"activating+undersized+remapped","last_fresh":"2023-10-09T04:24:57.642757+0000","last_change":"2023-10-09T04:24:57.148202+0000","last_active":"2023-10-09T04:24:55.133349+0000","last_peered":"2023-10-09T04:24:50.133067+0000","last_clean":"2023-10-09T04:24:50.133067+0000","last_became_active":"2023-10-09T04:23:25.382932+0000","last_became_peered":"2023-10-09T04:23:25.382932+0000","last_unstale":"2023-10-09T04:24:57.642757+0000","last_undegraded":"2023-10-09T04:24:57.642757+0000","last_fullsized":"2023-10-09T04:24:57.147986+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:41.582619+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,1],"avail_no_missing":["7","1","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":492,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644107+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:24:57.644107+0000","last_peered":"2023-10-09T04:24:57.644107+0000","last_clean":"2023-10-09T04:24:57.644107+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:24:57.644107+0000","last_undegraded":"2023-10-09T04:24:57.644107+0000","last_fullsized":"2023-10-09T04:24:57.644107+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":409,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.648764+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:24:57.648764+0000","last_peered":"2023-10-09T04:24:57.648764+0000","last_clean":"2023-10-09T04:24:57.648764+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:24:57.648764+0000","last_undegraded":"2023-10-09T04:24:57.648764+0000","last_fullsized":"2023-10-09T04:24:57.648764+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"443'5468","reported_seq":5565,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650018+0000","last_change":"2023-10-09T04:24:12.681872+0000","last_active":"2023-10-09T04:24:57.650018+0000","last_peered":"2023-10-09T04:24:57.650018+0000","last_clean":"2023-10-09T04:24:57.650018+0000","last_became_active":"2023-10-09T04:24:12.676248+0000","last_became_peered":"2023-10-09T04:24:12.676248+0000","last_unstale":"2023-10-09T04:24:57.650018+0000","last_undegraded":"2023-10-09T04:24:57.650018+0000","last_fullsized":"2023-10-09T04:24:57.650018+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109084672,"num_objects":1664,"num_object_clones":0,"num_object_copies":4992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2942,"num_write_kb":147424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":684,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-10-09T04:25:04.666247+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:25:04.666247+0000","last_peered":"2023-10-09T04:25:04.666247+0000","last_clean":"2023-10-09T04:25:04.666247+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:25:04.666247+0000","last_undegraded":"2023-10-09T04:25:04.666247+0000","last_fullsized":"2023-10-09T04:25:04.666247+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":468,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.643564+0000","last_change":"2023-10-09T04:24:56.212579+0000","last_active":"2023-10-09T04:24:57.643564+0000","last_peered":"2023-10-09T04:24:57.643564+0000","last_clean":"2023-10-09T04:24:57.643564+0000","last_became_active":"2023-10-09T04:24:56.184403+0000","last_became_peered":"2023-10-09T04:24:56.184403+0000","last_unstale":"2023-10-09T04:24:57.643564+0000","last_undegraded":"2023-10-09T04:24:57.643564+0000","last_fullsized":"2023-10-09T04:24:57.643564+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:10.486526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645024+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:24:57.645024+0000","last_peered":"2023-10-09T04:24:57.645024+0000","last_clean":"2023-10-09T04:24:57.645024+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:24:57.645024+0000","last_undegraded":"2023-10-09T04:24:57.645024+0000","last_fullsized":"2023-10-09T04:24:57.645024+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"446'5503","reported_seq":10546,"reported_epoch":447,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:24:57.646754+0000","last_change":"2023-10-09T04:24:57.346110+0000","last_active":"2023-10-09T04:24:57.646754+0000","last_peered":"2023-10-09T04:24:57.646754+0000","last_clean":"2023-10-09T04:22:25.963778+0000","last_became_active":"2023-10-09T04:24:57.263787+0000","last_became_peered":"2023-10-09T04:24:57.263787+0000","last_unstale":"2023-10-09T04:24:57.646754+0000","last_undegraded":"2023-10-09T04:24:57.259969+0000","last_fullsized":"2023-10-09T04:24:57.148441+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107413504,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":5,"num_objects_missing":5,"num_objects_degraded":822,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2975,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1774,"num_bytes_recovered":116293632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,3],"avail_no_missing":["3"],"object_location_counts":[{"shards":"0,1,3","objects":159},{"shards":"0,3","objects":1475},{"shards":"1,3,4","objects":5}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":652,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.648512+0000","last_change":"2023-10-09T04:24:58.648512+0000","last_active":"2023-10-09T04:24:58.648512+0000","last_peered":"2023-10-09T04:24:58.648512+0000","last_clean":"2023-10-09T04:24:58.648512+0000","last_became_active":"2023-10-09T04:24:58.648143+0000","last_became_peered":"2023-10-09T04:24:58.648143+0000","last_unstale":"2023-10-09T04:24:58.648512+0000","last_undegraded":"2023-10-09T04:24:58.648512+0000","last_fullsized":"2023-10-09T04:24:58.648512+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:26.096956+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":419,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642876+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:24:57.642876+0000","last_peered":"2023-10-09T04:24:57.642876+0000","last_clean":"2023-10-09T04:24:57.642876+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:24:57.642876+0000","last_undegraded":"2023-10-09T04:24:57.642876+0000","last_fullsized":"2023-10-09T04:24:57.642876+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.639711+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:24:58.639711+0000","last_peered":"2023-10-09T04:24:58.639711+0000","last_clean":"2023-10-09T04:24:58.639711+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:24:58.639711+0000","last_undegraded":"2023-10-09T04:24:58.639711+0000","last_fullsized":"2023-10-09T04:24:58.639711+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"451'5608","reported_seq":8148,"reported_epoch":451,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:25:04.607063+0000","last_change":"2023-10-09T04:24:22.480653+0000","last_active":"2023-10-09T04:25:04.607063+0000","last_peered":"2023-10-09T04:25:04.607063+0000","last_clean":"2023-10-09T04:23:05.287910+0000","last_became_active":"2023-10-09T04:24:22.474880+0000","last_became_peered":"2023-10-09T04:24:22.474880+0000","last_unstale":"2023-10-09T04:25:04.607063+0000","last_undegraded":"2023-10-09T04:25:04.607063+0000","last_fullsized":"2023-10-09T04:25:04.607063+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111443968,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1700,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3018,"num_write_kb":151008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":603,"num_bytes_recovered":39550976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1700}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":694,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.649255+0000","last_change":"2023-10-09T04:24:58.649255+0000","last_active":"2023-10-09T04:24:58.649255+0000","last_peered":"2023-10-09T04:24:58.649255+0000","last_clean":"2023-10-09T04:24:58.649255+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:24:58.649255+0000","last_undegraded":"2023-10-09T04:24:58.649255+0000","last_fullsized":"2023-10-09T04:24:58.649255+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:26.800833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:26.800833+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":447,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650082+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:24:57.650082+0000","last_peered":"2023-10-09T04:24:57.650082+0000","last_clean":"2023-10-09T04:24:57.650082+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:24:57.650082+0000","last_undegraded":"2023-10-09T04:24:57.650082+0000","last_fullsized":"2023-10-09T04:24:57.650082+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":69,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646750+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:24:57.646750+0000","last_peered":"2023-10-09T04:24:57.646750+0000","last_clean":"2023-10-09T04:24:57.646750+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:24:57.646750+0000","last_undegraded":"2023-10-09T04:24:57.646750+0000","last_fullsized":"2023-10-09T04:24:57.646750+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"443'5537","reported_seq":8065,"reported_epoch":447,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:57.645803+0000","last_change":"2023-10-09T04:24:57.190956+0000","last_active":"2023-10-09T04:24:57.645803+0000","last_peered":"2023-10-09T04:24:57.645803+0000","last_clean":"2023-10-09T04:23:05.307068+0000","last_became_active":"2023-10-09T04:24:57.173550+0000","last_became_peered":"2023-10-09T04:24:57.173550+0000","last_unstale":"2023-10-09T04:24:57.645803+0000","last_undegraded":"2023-10-09T04:24:57.645803+0000","last_fullsized":"2023-10-09T04:24:57.645803+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5537,"log_dups_size":0,"ondisk_log_size":5537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1645,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2985,"num_write_kb":148160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":596,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1645}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":871,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642913+0000","last_change":"2023-10-09T04:19:15.469350+0000","last_active":"2023-10-09T04:24:57.642913+0000","last_peered":"2023-10-09T04:24:57.642913+0000","last_clean":"2023-10-09T04:24:57.642913+0000","last_became_active":"2023-10-09T04:18:18.486511+0000","last_became_peered":"2023-10-09T04:18:18.486511+0000","last_unstale":"2023-10-09T04:24:57.642913+0000","last_undegraded":"2023-10-09T04:24:57.642913+0000","last_fullsized":"2023-10-09T04:24:57.642913+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:29.618715+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":81,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646666+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:24:57.646666+0000","last_peered":"2023-10-09T04:24:57.646666+0000","last_clean":"2023-10-09T04:24:57.646666+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:24:57.646666+0000","last_undegraded":"2023-10-09T04:24:57.646666+0000","last_fullsized":"2023-10-09T04:24:57.646666+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645145+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:24:57.645145+0000","last_peered":"2023-10-09T04:24:57.645145+0000","last_clean":"2023-10-09T04:24:57.645145+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:24:57.645145+0000","last_undegraded":"2023-10-09T04:24:57.645145+0000","last_fullsized":"2023-10-09T04:24:57.645145+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"446'5468","reported_seq":5565,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650565+0000","last_change":"2023-10-09T04:24:22.451555+0000","last_active":"2023-10-09T04:24:57.650565+0000","last_peered":"2023-10-09T04:24:57.650565+0000","last_clean":"2023-10-09T04:24:57.650565+0000","last_became_active":"2023-10-09T04:24:22.446741+0000","last_became_peered":"2023-10-09T04:24:22.446741+0000","last_unstale":"2023-10-09T04:24:57.650565+0000","last_undegraded":"2023-10-09T04:24:57.650565+0000","last_fullsized":"2023-10-09T04:24:57.650565+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107282432,"num_objects":1637,"num_object_clones":0,"num_object_copies":4911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2949,"num_write_kb":146752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":617,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649109+0000","last_change":"2023-10-09T04:19:45.544226+0000","last_active":"2023-10-09T04:24:57.649109+0000","last_peered":"2023-10-09T04:24:57.649109+0000","last_clean":"2023-10-09T04:24:57.649109+0000","last_became_active":"2023-10-09T04:19:45.543844+0000","last_became_peered":"2023-10-09T04:19:45.543844+0000","last_unstale":"2023-10-09T04:24:57.649109+0000","last_undegraded":"2023-10-09T04:24:57.649109+0000","last_fullsized":"2023-10-09T04:24:57.649109+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:47.357931+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645437+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:24:57.645437+0000","last_peered":"2023-10-09T04:24:57.645437+0000","last_clean":"2023-10-09T04:24:57.645437+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:24:57.645437+0000","last_undegraded":"2023-10-09T04:24:57.645437+0000","last_fullsized":"2023-10-09T04:24:57.645437+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T04:24:56.187730+0000","last_change":"2023-10-09T04:24:56.183312+0000","last_active":"2023-10-09T04:24:56.187730+0000","last_peered":"2023-10-09T04:24:56.187730+0000","last_clean":"2023-10-09T04:24:56.187730+0000","last_became_active":"2023-10-09T04:24:56.181515+0000","last_became_peered":"2023-10-09T04:24:56.181515+0000","last_unstale":"2023-10-09T04:24:56.187730+0000","last_undegraded":"2023-10-09T04:24:56.187730+0000","last_fullsized":"2023-10-09T04:24:56.187730+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.570114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"451'5474","reported_seq":13864,"reported_epoch":451,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:25:03.708081+0000","last_change":"2023-10-09T04:24:57.253868+0000","last_active":"2023-10-09T04:25:03.708081+0000","last_peered":"2023-10-09T04:25:03.708081+0000","last_clean":"2023-10-09T04:23:21.324155+0000","last_became_active":"2023-10-09T04:24:57.247124+0000","last_became_peered":"2023-10-09T04:24:57.247124+0000","last_unstale":"2023-10-09T04:25:03.708081+0000","last_undegraded":"2023-10-09T04:24:57.163466+0000","last_fullsized":"2023-10-09T04:24:57.145795+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5474,"log_dups_size":0,"ondisk_log_size":5474,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108298240,"num_objects":1652,"num_object_clones":0,"num_object_copies":4956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":777,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2955,"num_write_kb":147456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2078,"num_bytes_recovered":136183808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1517},{"shards":"1,6,7","objects":135}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":552,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646836+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:24:57.646836+0000","last_peered":"2023-10-09T04:24:57.646836+0000","last_clean":"2023-10-09T04:24:57.646836+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:24:57.646836+0000","last_undegraded":"2023-10-09T04:24:57.646836+0000","last_fullsized":"2023-10-09T04:24:57.646836+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645659+0000","last_change":"2023-10-09T04:24:56.188476+0000","last_active":"2023-10-09T04:24:57.645659+0000","last_peered":"2023-10-09T04:24:57.645659+0000","last_clean":"2023-10-09T04:24:57.645659+0000","last_became_active":"2023-10-09T04:24:56.180264+0000","last_became_peered":"2023-10-09T04:24:56.180264+0000","last_unstale":"2023-10-09T04:24:57.645659+0000","last_undegraded":"2023-10-09T04:24:57.645659+0000","last_fullsized":"2023-10-09T04:24:57.645659+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:21.221377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646822+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:24:57.646822+0000","last_peered":"2023-10-09T04:24:57.646822+0000","last_clean":"2023-10-09T04:24:57.646822+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:24:57.646822+0000","last_undegraded":"2023-10-09T04:24:57.646822+0000","last_fullsized":"2023-10-09T04:24:57.646822+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"443'5761","reported_seq":10978,"reported_epoch":448,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:25:01.015651+0000","last_change":"2023-10-09T04:24:57.260387+0000","last_active":"2023-10-09T04:25:01.015651+0000","last_peered":"2023-10-09T04:25:01.015651+0000","last_clean":"2023-10-09T04:22:25.940554+0000","last_became_active":"2023-10-09T04:24:57.235976+0000","last_became_peered":"2023-10-09T04:24:57.235976+0000","last_unstale":"2023-10-09T04:25:01.015651+0000","last_undegraded":"2023-10-09T04:25:01.015651+0000","last_fullsized":"2023-10-09T04:24:57.147805+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5761,"log_dups_size":0,"ondisk_log_size":5761,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113180672,"num_objects":1727,"num_object_clones":0,"num_object_copies":5181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3100,"num_write_kb":154464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1959,"num_bytes_recovered":115408896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1727}],"blocked_by":[],"up_primary":4,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":612,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649399+0000","last_change":"2023-10-09T04:22:16.643512+0000","last_active":"2023-10-09T04:24:57.649399+0000","last_peered":"2023-10-09T04:24:57.649399+0000","last_clean":"2023-10-09T04:24:57.649399+0000","last_became_active":"2023-10-09T04:22:16.643150+0000","last_became_peered":"2023-10-09T04:22:16.643150+0000","last_unstale":"2023-10-09T04:24:57.649399+0000","last_undegraded":"2023-10-09T04:24:57.649399+0000","last_fullsized":"2023-10-09T04:24:57.649399+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:51.421460+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":116,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.657410+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:25:03.657410+0000","last_peered":"2023-10-09T04:25:03.657410+0000","last_clean":"2023-10-09T04:25:03.657410+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:25:03.657410+0000","last_undegraded":"2023-10-09T04:25:03.657410+0000","last_fullsized":"2023-10-09T04:25:03.657410+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645152+0000","last_change":"2023-10-09T04:24:56.179985+0000","last_active":"2023-10-09T04:24:57.645152+0000","last_peered":"2023-10-09T04:24:57.645152+0000","last_clean":"2023-10-09T04:24:57.645152+0000","last_became_active":"2023-10-09T04:24:56.178864+0000","last_became_peered":"2023-10-09T04:24:56.178864+0000","last_unstale":"2023-10-09T04:24:57.645152+0000","last_undegraded":"2023-10-09T04:24:57.645152+0000","last_fullsized":"2023-10-09T04:24:57.645152+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:07:41.304375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"446'5875","reported_seq":10817,"reported_epoch":447,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:57.641857+0000","last_change":"2023-10-09T04:24:57.280356+0000","last_active":"2023-10-09T04:24:57.641857+0000","last_peered":"2023-10-09T04:24:57.641857+0000","last_clean":"2023-10-09T04:23:05.321351+0000","last_became_active":"2023-10-09T04:24:57.263909+0000","last_became_peered":"2023-10-09T04:24:57.263909+0000","last_unstale":"2023-10-09T04:24:57.641857+0000","last_undegraded":"2023-10-09T04:24:57.641857+0000","last_fullsized":"2023-10-09T04:24:57.641857+0000","mapping_epoch":445,"log_start":"388'400","ondisk_log_start":"388'400","created":380,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5475,"log_dups_size":0,"ondisk_log_size":5475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:14.086008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215547904,"num_objects":3289,"num_object_clones":0,"num_object_copies":9867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2043,"num_objects_unfound":0,"num_objects_dirty":3289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5875,"num_write_kb":293248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2405,"num_bytes_recovered":157614080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,3],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"3,4,7","objects":3289}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":575,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.638711+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:24:58.638711+0000","last_peered":"2023-10-09T04:24:58.638711+0000","last_clean":"2023-10-09T04:24:58.638711+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:24:58.638711+0000","last_undegraded":"2023-10-09T04:24:58.638711+0000","last_fullsized":"2023-10-09T04:24:58.638711+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":49,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.647191+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:24:57.647191+0000","last_peered":"2023-10-09T04:24:57.647191+0000","last_clean":"2023-10-09T04:24:57.647191+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:24:57.647191+0000","last_undegraded":"2023-10-09T04:24:57.647191+0000","last_fullsized":"2023-10-09T04:24:57.647191+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":444,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649239+0000","last_change":"2023-10-09T04:18:15.744813+0000","last_active":"2023-10-09T04:24:57.649239+0000","last_peered":"2023-10-09T04:24:57.649239+0000","last_clean":"2023-10-09T04:24:57.649239+0000","last_became_active":"2023-10-09T04:18:15.744347+0000","last_became_peered":"2023-10-09T04:18:15.744347+0000","last_unstale":"2023-10-09T04:24:57.649239+0000","last_undegraded":"2023-10-09T04:24:57.649239+0000","last_fullsized":"2023-10-09T04:24:57.649239+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:24.106379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":317,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649899+0000","last_change":"2023-10-09T04:24:56.173092+0000","last_active":"2023-10-09T04:24:57.649899+0000","last_peered":"2023-10-09T04:24:57.649899+0000","last_clean":"2023-10-09T04:24:57.649899+0000","last_became_active":"2023-10-09T04:24:56.172767+0000","last_became_peered":"2023-10-09T04:24:56.172767+0000","last_unstale":"2023-10-09T04:24:57.649899+0000","last_undegraded":"2023-10-09T04:24:57.649899+0000","last_fullsized":"2023-10-09T04:24:57.649899+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:32.560836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"446'5938","reported_seq":15602,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645737+0000","last_change":"2023-10-09T04:24:50.135438+0000","last_active":"2023-10-09T04:24:57.645737+0000","last_peered":"2023-10-09T04:24:57.645737+0000","last_clean":"2023-10-09T04:24:57.645737+0000","last_became_active":"2023-10-09T04:24:50.121211+0000","last_became_peered":"2023-10-09T04:24:50.121211+0000","last_unstale":"2023-10-09T04:24:57.645737+0000","last_undegraded":"2023-10-09T04:24:57.645737+0000","last_fullsized":"2023-10-09T04:24:57.645737+0000","mapping_epoch":442,"log_start":"388'912","ondisk_log_start":"388'912","created":380,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:51.615182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210370560,"num_objects":3210,"num_object_clones":0,"num_object_copies":9630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5938,"num_write_kb":292736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4767,"num_bytes_recovered":312410112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":615,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649810+0000","last_change":"2023-10-09T04:23:08.631094+0000","last_active":"2023-10-09T04:24:57.649810+0000","last_peered":"2023-10-09T04:24:57.649810+0000","last_clean":"2023-10-09T04:24:57.649810+0000","last_became_active":"2023-10-09T04:23:08.630831+0000","last_became_peered":"2023-10-09T04:23:08.630831+0000","last_unstale":"2023-10-09T04:24:57.649810+0000","last_undegraded":"2023-10-09T04:24:57.649810+0000","last_fullsized":"2023-10-09T04:24:57.649810+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:17.921309+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":431,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649570+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:24:57.649570+0000","last_peered":"2023-10-09T04:24:57.649570+0000","last_clean":"2023-10-09T04:24:57.649570+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:24:57.649570+0000","last_undegraded":"2023-10-09T04:24:57.649570+0000","last_fullsized":"2023-10-09T04:24:57.649570+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644489+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:24:57.644489+0000","last_peered":"2023-10-09T04:24:57.644489+0000","last_clean":"2023-10-09T04:24:57.644489+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:24:57.644489+0000","last_undegraded":"2023-10-09T04:24:57.644489+0000","last_fullsized":"2023-10-09T04:24:57.644489+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":412,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T04:25:02.637781+0000","last_change":"2023-10-09T04:24:22.445259+0000","last_active":"2023-10-09T04:25:02.637781+0000","last_peered":"2023-10-09T04:25:02.637781+0000","last_clean":"2023-10-09T04:25:02.637781+0000","last_became_active":"2023-10-09T04:24:22.444960+0000","last_became_peered":"2023-10-09T04:24:22.444960+0000","last_unstale":"2023-10-09T04:25:02.637781+0000","last_undegraded":"2023-10-09T04:25:02.637781+0000","last_fullsized":"2023-10-09T04:25:02.637781+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:22.200720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"443'5578","reported_seq":5668,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649617+0000","last_change":"2023-10-09T04:24:55.835549+0000","last_active":"2023-10-09T04:24:57.649617+0000","last_peered":"2023-10-09T04:24:57.649617+0000","last_clean":"2023-10-09T04:24:57.649617+0000","last_became_active":"2023-10-09T04:24:12.633090+0000","last_became_peered":"2023-10-09T04:24:12.633090+0000","last_unstale":"2023-10-09T04:24:57.649617+0000","last_undegraded":"2023-10-09T04:24:57.649617+0000","last_fullsized":"2023-10-09T04:24:57.649617+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112656384,"num_objects":1719,"num_object_clones":0,"num_object_copies":5157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2990,"num_write_kb":150688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":603,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.735719+0000","last_change":"2023-10-09T04:24:58.735719+0000","last_active":"2023-10-09T04:24:58.735719+0000","last_peered":"2023-10-09T04:24:58.735719+0000","last_clean":"2023-10-09T04:24:58.735719+0000","last_became_active":"2023-10-09T04:24:58.735355+0000","last_became_peered":"2023-10-09T04:24:58.735355+0000","last_unstale":"2023-10-09T04:24:58.735719+0000","last_undegraded":"2023-10-09T04:24:58.735719+0000","last_fullsized":"2023-10-09T04:24:58.735719+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:36.727241+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":279,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645536+0000","last_change":"2023-10-09T04:24:22.449470+0000","last_active":"2023-10-09T04:24:57.645536+0000","last_peered":"2023-10-09T04:24:57.645536+0000","last_clean":"2023-10-09T04:24:57.645536+0000","last_became_active":"2023-10-09T04:24:22.448762+0000","last_became_peered":"2023-10-09T04:24:22.448762+0000","last_unstale":"2023-10-09T04:24:57.645536+0000","last_undegraded":"2023-10-09T04:24:57.645536+0000","last_fullsized":"2023-10-09T04:24:57.645536+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:13.213314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":387,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T04:25:02.636858+0000","last_change":"2023-10-09T04:24:22.444595+0000","last_active":"2023-10-09T04:25:02.636858+0000","last_peered":"2023-10-09T04:25:02.636858+0000","last_clean":"2023-10-09T04:25:02.636858+0000","last_became_active":"2023-10-09T04:24:22.444387+0000","last_became_peered":"2023-10-09T04:24:22.444387+0000","last_unstale":"2023-10-09T04:25:02.636858+0000","last_undegraded":"2023-10-09T04:25:02.636858+0000","last_fullsized":"2023-10-09T04:25:02.636858+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:37.195469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":153,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650711+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:24:57.650711+0000","last_peered":"2023-10-09T04:24:57.650711+0000","last_clean":"2023-10-09T04:24:57.650711+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:24:57.650711+0000","last_undegraded":"2023-10-09T04:24:57.650711+0000","last_fullsized":"2023-10-09T04:24:57.650711+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"446'6020","reported_seq":13760,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.639768+0000","last_change":"2023-10-09T04:24:27.462956+0000","last_active":"2023-10-09T04:24:58.639768+0000","last_peered":"2023-10-09T04:24:58.639768+0000","last_clean":"2023-10-09T04:24:58.639768+0000","last_became_active":"2023-10-09T04:24:27.454212+0000","last_became_peered":"2023-10-09T04:24:27.454212+0000","last_unstale":"2023-10-09T04:24:58.639768+0000","last_undegraded":"2023-10-09T04:24:58.639768+0000","last_fullsized":"2023-10-09T04:24:58.639768+0000","mapping_epoch":436,"log_start":"388'500","ondisk_log_start":"388'500","created":380,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:13.486516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219938816,"num_objects":3356,"num_object_clones":0,"num_object_copies":10068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6020,"num_write_kb":300032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3903,"num_bytes_recovered":255787008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":431,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650473+0000","last_change":"2023-10-09T04:10:38.325253+0000","last_active":"2023-10-09T04:24:57.650473+0000","last_peered":"2023-10-09T04:24:57.650473+0000","last_clean":"2023-10-09T04:24:57.650473+0000","last_became_active":"2023-10-09T04:10:38.324065+0000","last_became_peered":"2023-10-09T04:10:38.324065+0000","last_unstale":"2023-10-09T04:24:57.650473+0000","last_undegraded":"2023-10-09T04:24:57.650473+0000","last_fullsized":"2023-10-09T04:24:57.650473+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:44.713009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":902,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.641926+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:24:57.641926+0000","last_peered":"2023-10-09T04:24:57.641926+0000","last_clean":"2023-10-09T04:24:57.641926+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:24:57.641926+0000","last_undegraded":"2023-10-09T04:24:57.641926+0000","last_fullsized":"2023-10-09T04:24:57.641926+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":384,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649821+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:24:57.649821+0000","last_peered":"2023-10-09T04:24:57.649821+0000","last_clean":"2023-10-09T04:24:57.649821+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:24:57.649821+0000","last_undegraded":"2023-10-09T04:24:57.649821+0000","last_fullsized":"2023-10-09T04:24:57.649821+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-10-09T04:24:56.177740+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:24:56.177740+0000","last_peered":"2023-10-09T04:24:56.177740+0000","last_clean":"2023-10-09T04:24:56.177740+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:24:56.177740+0000","last_undegraded":"2023-10-09T04:24:56.177740+0000","last_fullsized":"2023-10-09T04:24:56.177740+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"451'5542","reported_seq":8794,"reported_epoch":451,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:25:04.552923+0000","last_change":"2023-10-09T04:24:56.193852+0000","last_active":"2023-10-09T04:25:04.552923+0000","last_peered":"2023-10-09T04:25:04.552923+0000","last_clean":"2023-10-09T04:23:05.325743+0000","last_became_active":"2023-10-09T04:24:12.649119+0000","last_became_peered":"2023-10-09T04:24:12.649119+0000","last_unstale":"2023-10-09T04:25:04.552923+0000","last_undegraded":"2023-10-09T04:25:04.552923+0000","last_fullsized":"2023-10-09T04:25:04.552923+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5542,"log_dups_size":0,"ondisk_log_size":5542,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105218048,"num_objects":1605,"num_object_clones":0,"num_object_copies":4815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1244,"num_objects_unfound":0,"num_objects_dirty":1605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3005,"num_write_kb":147552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":943,"num_bytes_recovered":61833216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1605}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":543,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.731960+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:24:58.731960+0000","last_peered":"2023-10-09T04:24:58.731960+0000","last_clean":"2023-10-09T04:24:58.731960+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:24:58.731960+0000","last_undegraded":"2023-10-09T04:24:58.731960+0000","last_fullsized":"2023-10-09T04:24:58.731960+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":448,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.641783+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:24:57.641783+0000","last_peered":"2023-10-09T04:24:57.641783+0000","last_clean":"2023-10-09T04:24:57.641783+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:24:57.641783+0000","last_undegraded":"2023-10-09T04:24:57.641783+0000","last_fullsized":"2023-10-09T04:24:57.641783+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645684+0000","last_change":"2023-10-09T04:24:56.170679+0000","last_active":"2023-10-09T04:24:57.645684+0000","last_peered":"2023-10-09T04:24:57.645684+0000","last_clean":"2023-10-09T04:24:57.645684+0000","last_became_active":"2023-10-09T04:24:56.170396+0000","last_became_peered":"2023-10-09T04:24:56.170396+0000","last_unstale":"2023-10-09T04:24:57.645684+0000","last_undegraded":"2023-10-09T04:24:57.645684+0000","last_fullsized":"2023-10-09T04:24:57.645684+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:44.041826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":51,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645967+0000","last_change":"2023-10-09T04:23:06.341197+0000","last_active":"2023-10-09T04:24:57.645967+0000","last_peered":"2023-10-09T04:24:57.645967+0000","last_clean":"2023-10-09T04:24:57.645967+0000","last_became_active":"2023-10-09T04:23:06.340773+0000","last_became_peered":"2023-10-09T04:23:06.340773+0000","last_unstale":"2023-10-09T04:24:57.645967+0000","last_undegraded":"2023-10-09T04:24:57.645967+0000","last_fullsized":"2023-10-09T04:24:57.645967+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:18:55.764215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"443'5770","reported_seq":6575,"reported_epoch":446,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T04:25:01.537125+0000","last_change":"2023-10-09T04:25:01.552409+0000","last_active":"2023-10-09T04:24:56.206348+0000","last_peered":"2023-10-09T04:24:55.057066+0000","last_clean":"2023-10-09T04:22:25.956502+0000","last_became_active":"2023-10-09T04:23:28.336672+0000","last_became_peered":"2023-10-09T04:23:28.336672+0000","last_unstale":"2023-10-09T04:25:01.537125+0000","last_undegraded":"2023-10-09T04:25:01.537125+0000","last_fullsized":"2023-10-09T04:24:57.147228+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:45.465306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214106136,"num_objects":3268,"num_object_clones":0,"num_object_copies":9804,"num_objects_missing_on_primary":1550,"num_objects_missing":1550,"num_objects_degraded":3772,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3268,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5770,"num_write_kb":289153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1252,"num_bytes_recovered":82051072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,0],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"0,4","objects":747},{"shards":"0,6","objects":299},{"shards":"4,6","objects":2222}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":567,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.639429+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:24:58.639429+0000","last_peered":"2023-10-09T04:24:58.639429+0000","last_clean":"2023-10-09T04:24:58.639429+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:24:58.639429+0000","last_undegraded":"2023-10-09T04:24:58.639429+0000","last_fullsized":"2023-10-09T04:24:58.639429+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":471,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642882+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:24:57.642882+0000","last_peered":"2023-10-09T04:24:57.642882+0000","last_clean":"2023-10-09T04:24:57.642882+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:24:57.642882+0000","last_undegraded":"2023-10-09T04:24:57.642882+0000","last_fullsized":"2023-10-09T04:24:57.642882+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":186,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650002+0000","last_change":"2023-10-09T04:24:22.447108+0000","last_active":"2023-10-09T04:24:57.650002+0000","last_peered":"2023-10-09T04:24:57.650002+0000","last_clean":"2023-10-09T04:24:57.650002+0000","last_became_active":"2023-10-09T04:24:22.446733+0000","last_became_peered":"2023-10-09T04:24:22.446733+0000","last_unstale":"2023-10-09T04:24:57.650002+0000","last_undegraded":"2023-10-09T04:24:57.650002+0000","last_fullsized":"2023-10-09T04:24:57.650002+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:33.132499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":167,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650399+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:24:57.650399+0000","last_peered":"2023-10-09T04:24:57.650399+0000","last_clean":"2023-10-09T04:24:57.650399+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:24:57.650399+0000","last_undegraded":"2023-10-09T04:24:57.650399+0000","last_fullsized":"2023-10-09T04:24:57.650399+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"451'5948","reported_seq":14649,"reported_epoch":451,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:25:04.626639+0000","last_change":"2023-10-09T04:24:57.279013+0000","last_active":"2023-10-09T04:25:04.626639+0000","last_peered":"2023-10-09T04:25:04.626639+0000","last_clean":"2023-10-09T04:24:55.103818+0000","last_became_active":"2023-10-09T04:24:57.266591+0000","last_became_peered":"2023-10-09T04:24:57.266591+0000","last_unstale":"2023-10-09T04:25:04.626639+0000","last_undegraded":"2023-10-09T04:25:04.626639+0000","last_fullsized":"2023-10-09T04:25:04.626639+0000","mapping_epoch":445,"log_start":"388'400","ondisk_log_start":"388'400","created":380,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:32.944822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208535552,"num_objects":3182,"num_object_clones":0,"num_object_copies":9546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3182,"num_objects_unfound":0,"num_objects_dirty":3182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5948,"num_write_kb":292160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4291,"num_bytes_recovered":281214976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5,6","objects":3182}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":448,"reported_epoch":447,"state":"activating+undersized+remapped","last_fresh":"2023-10-09T04:24:57.642718+0000","last_change":"2023-10-09T04:24:57.146205+0000","last_active":"2023-10-09T04:24:56.165284+0000","last_peered":"2023-10-09T04:24:16.085238+0000","last_clean":"2023-10-09T04:24:16.085238+0000","last_became_active":"2023-10-09T04:23:25.380615+0000","last_became_peered":"2023-10-09T04:23:25.380615+0000","last_unstale":"2023-10-09T04:24:57.642718+0000","last_undegraded":"2023-10-09T04:24:57.642718+0000","last_fullsized":"2023-10-09T04:24:57.146009+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.782283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.782283+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[2,6],"avail_no_missing":["2","4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":408,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644278+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:24:57.644278+0000","last_peered":"2023-10-09T04:24:57.644278+0000","last_clean":"2023-10-09T04:24:57.644278+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:24:57.644278+0000","last_undegraded":"2023-10-09T04:24:57.644278+0000","last_fullsized":"2023-10-09T04:24:57.644278+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.647037+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:24:57.647037+0000","last_peered":"2023-10-09T04:24:57.647037+0000","last_clean":"2023-10-09T04:24:57.647037+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:24:57.647037+0000","last_undegraded":"2023-10-09T04:24:57.647037+0000","last_fullsized":"2023-10-09T04:24:57.647037+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":346,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.642103+0000","last_change":"2023-10-09T04:24:22.449209+0000","last_active":"2023-10-09T04:24:57.642103+0000","last_peered":"2023-10-09T04:24:57.642103+0000","last_clean":"2023-10-09T04:24:57.642103+0000","last_became_active":"2023-10-09T04:24:22.448886+0000","last_became_peered":"2023-10-09T04:24:22.448886+0000","last_unstale":"2023-10-09T04:24:57.642103+0000","last_undegraded":"2023-10-09T04:24:57.642103+0000","last_fullsized":"2023-10-09T04:24:57.642103+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:03.309571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"443'5969","reported_seq":6044,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.638942+0000","last_change":"2023-10-09T04:22:13.630026+0000","last_active":"2023-10-09T04:24:58.638942+0000","last_peered":"2023-10-09T04:24:58.638942+0000","last_clean":"2023-10-09T04:24:58.638942+0000","last_became_active":"2023-10-09T04:22:13.629701+0000","last_became_peered":"2023-10-09T04:22:13.629701+0000","last_unstale":"2023-10-09T04:24:58.638942+0000","last_undegraded":"2023-10-09T04:24:58.638942+0000","last_fullsized":"2023-10-09T04:24:58.638942+0000","mapping_epoch":383,"log_start":"388'500","ondisk_log_start":"388'500","created":380,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:47.148119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5969,"num_write_kb":298304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":626,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.658132+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:25:03.658132+0000","last_peered":"2023-10-09T04:25:03.658132+0000","last_clean":"2023-10-09T04:25:03.658132+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:25:03.658132+0000","last_undegraded":"2023-10-09T04:25:03.658132+0000","last_fullsized":"2023-10-09T04:25:03.658132+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":244,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.638837+0000","last_change":"2023-10-09T04:24:22.444161+0000","last_active":"2023-10-09T04:24:58.638837+0000","last_peered":"2023-10-09T04:24:58.638837+0000","last_clean":"2023-10-09T04:24:58.638837+0000","last_became_active":"2023-10-09T04:24:22.443860+0000","last_became_peered":"2023-10-09T04:24:22.443860+0000","last_unstale":"2023-10-09T04:24:58.638837+0000","last_undegraded":"2023-10-09T04:24:58.638837+0000","last_fullsized":"2023-10-09T04:24:58.638837+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:35.445455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":81,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646188+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:24:57.646188+0000","last_peered":"2023-10-09T04:24:57.646188+0000","last_clean":"2023-10-09T04:24:57.646188+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:24:57.646188+0000","last_undegraded":"2023-10-09T04:24:57.646188+0000","last_fullsized":"2023-10-09T04:24:57.646188+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644945+0000","last_change":"2023-10-09T04:24:56.169203+0000","last_active":"2023-10-09T04:24:57.644945+0000","last_peered":"2023-10-09T04:24:57.644945+0000","last_clean":"2023-10-09T04:24:57.644945+0000","last_became_active":"2023-10-09T04:24:56.168737+0000","last_became_peered":"2023-10-09T04:24:56.168737+0000","last_unstale":"2023-10-09T04:24:57.644945+0000","last_undegraded":"2023-10-09T04:24:57.644945+0000","last_fullsized":"2023-10-09T04:24:57.644945+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:21.515886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"446'5512","reported_seq":8036,"reported_epoch":447,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:57.645565+0000","last_change":"2023-10-09T04:24:57.192401+0000","last_active":"2023-10-09T04:24:57.645565+0000","last_peered":"2023-10-09T04:24:57.645565+0000","last_clean":"2023-10-09T04:23:05.307068+0000","last_became_active":"2023-10-09T04:24:57.173537+0000","last_became_peered":"2023-10-09T04:24:57.173537+0000","last_unstale":"2023-10-09T04:24:57.645565+0000","last_undegraded":"2023-10-09T04:24:57.645565+0000","last_fullsized":"2023-10-09T04:24:57.645565+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109445120,"num_objects":1670,"num_object_clones":0,"num_object_copies":5010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1670,"num_objects_unfound":0,"num_objects_dirty":1670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2960,"num_write_kb":148160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":596,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":1670}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":477,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645598+0000","last_change":"2023-10-09T04:23:08.628990+0000","last_active":"2023-10-09T04:24:57.645598+0000","last_peered":"2023-10-09T04:24:57.645598+0000","last_clean":"2023-10-09T04:24:57.645598+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:24:57.645598+0000","last_undegraded":"2023-10-09T04:24:57.645598+0000","last_fullsized":"2023-10-09T04:24:57.645598+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:18.658338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:18.658338+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":69,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646981+0000","last_change":"2023-10-09T04:24:56.170052+0000","last_active":"2023-10-09T04:24:57.646981+0000","last_peered":"2023-10-09T04:24:57.646981+0000","last_clean":"2023-10-09T04:24:57.646981+0000","last_became_active":"2023-10-09T04:24:56.169607+0000","last_became_peered":"2023-10-09T04:24:56.169607+0000","last_unstale":"2023-10-09T04:24:57.646981+0000","last_undegraded":"2023-10-09T04:24:57.646981+0000","last_fullsized":"2023-10-09T04:24:57.646981+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.849923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":287,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T04:25:02.645279+0000","last_change":"2023-10-09T04:18:15.741078+0000","last_active":"2023-10-09T04:25:02.645279+0000","last_peered":"2023-10-09T04:25:02.645279+0000","last_clean":"2023-10-09T04:25:02.645279+0000","last_became_active":"2023-10-09T04:18:15.740750+0000","last_became_peered":"2023-10-09T04:18:15.740750+0000","last_unstale":"2023-10-09T04:25:02.645279+0000","last_undegraded":"2023-10-09T04:25:02.645279+0000","last_fullsized":"2023-10-09T04:25:02.645279+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:13.575294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"451'5651","reported_seq":8191,"reported_epoch":451,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:25:04.529074+0000","last_change":"2023-10-09T04:24:22.480882+0000","last_active":"2023-10-09T04:25:04.529074+0000","last_peered":"2023-10-09T04:25:04.529074+0000","last_clean":"2023-10-09T04:23:05.287910+0000","last_became_active":"2023-10-09T04:24:22.475675+0000","last_became_peered":"2023-10-09T04:24:22.475675+0000","last_unstale":"2023-10-09T04:25:04.529074+0000","last_undegraded":"2023-10-09T04:25:04.529074+0000","last_fullsized":"2023-10-09T04:25:04.529074+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108625920,"num_objects":1658,"num_object_clones":0,"num_object_copies":4974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1658,"num_objects_unfound":0,"num_objects_dirty":1658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3062,"num_write_kb":151008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":604,"num_bytes_recovered":39550976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1658}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":523,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-09T04:24:58.639385+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:24:58.639385+0000","last_peered":"2023-10-09T04:24:58.639385+0000","last_clean":"2023-10-09T04:24:58.639385+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:24:58.639385+0000","last_undegraded":"2023-10-09T04:24:58.639385+0000","last_fullsized":"2023-10-09T04:24:58.639385+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":465,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.650177+0000","last_change":"2023-10-09T04:24:56.181978+0000","last_active":"2023-10-09T04:24:57.650177+0000","last_peered":"2023-10-09T04:24:57.650177+0000","last_clean":"2023-10-09T04:24:57.650177+0000","last_became_active":"2023-10-09T04:24:56.164297+0000","last_became_peered":"2023-10-09T04:24:56.164297+0000","last_unstale":"2023-10-09T04:24:57.650177+0000","last_undegraded":"2023-10-09T04:24:57.650177+0000","last_fullsized":"2023-10-09T04:24:57.650177+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:15.703672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644327+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:24:57.644327+0000","last_peered":"2023-10-09T04:24:57.644327+0000","last_clean":"2023-10-09T04:24:57.644327+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:24:57.644327+0000","last_undegraded":"2023-10-09T04:24:57.644327+0000","last_fullsized":"2023-10-09T04:24:57.644327+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"446'5484","reported_seq":11307,"reported_epoch":447,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:57.645829+0000","last_change":"2023-10-09T04:24:57.297668+0000","last_active":"2023-10-09T04:24:57.645829+0000","last_peered":"2023-10-09T04:24:57.645829+0000","last_clean":"2023-10-09T04:24:54.998633+0000","last_became_active":"2023-10-09T04:24:57.261715+0000","last_became_peered":"2023-10-09T04:24:57.261715+0000","last_unstale":"2023-10-09T04:24:57.645829+0000","last_undegraded":"2023-10-09T04:24:57.645829+0000","last_fullsized":"2023-10-09T04:24:57.645829+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108658688,"num_objects":1658,"num_object_clones":0,"num_object_copies":4974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1658,"num_objects_unfound":0,"num_objects_dirty":1658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2956,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2160,"num_bytes_recovered":140345344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,3,1],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":1658}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":476,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649151+0000","last_change":"2023-10-09T04:24:56.202659+0000","last_active":"2023-10-09T04:24:57.649151+0000","last_peered":"2023-10-09T04:24:57.649151+0000","last_clean":"2023-10-09T04:24:57.649151+0000","last_became_active":"2023-10-09T04:24:56.166522+0000","last_became_peered":"2023-10-09T04:24:56.166522+0000","last_unstale":"2023-10-09T04:24:57.649151+0000","last_undegraded":"2023-10-09T04:24:57.649151+0000","last_fullsized":"2023-10-09T04:24:57.649151+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:15.858204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644120+0000","last_change":"2023-10-09T04:24:56.163870+0000","last_active":"2023-10-09T04:24:57.644120+0000","last_peered":"2023-10-09T04:24:57.644120+0000","last_clean":"2023-10-09T04:24:57.644120+0000","last_became_active":"2023-10-09T04:24:56.163181+0000","last_became_peered":"2023-10-09T04:24:56.163181+0000","last_unstale":"2023-10-09T04:24:57.644120+0000","last_undegraded":"2023-10-09T04:24:57.644120+0000","last_fullsized":"2023-10-09T04:24:57.644120+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:03.635879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"446'5457","reported_seq":5553,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.649928+0000","last_change":"2023-10-09T04:24:12.686316+0000","last_active":"2023-10-09T04:24:57.649928+0000","last_peered":"2023-10-09T04:24:57.649928+0000","last_clean":"2023-10-09T04:24:57.649928+0000","last_became_active":"2023-10-09T04:24:12.682272+0000","last_became_peered":"2023-10-09T04:24:12.682272+0000","last_unstale":"2023-10-09T04:24:57.649928+0000","last_undegraded":"2023-10-09T04:24:57.649928+0000","last_fullsized":"2023-10-09T04:24:57.649928+0000","mapping_epoch":428,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":429,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109805568,"num_objects":1676,"num_object_clones":0,"num_object_copies":5028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2932,"num_write_kb":147424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":799,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T04:25:02.637277+0000","last_change":"2023-10-09T04:22:16.629747+0000","last_active":"2023-10-09T04:25:02.637277+0000","last_peered":"2023-10-09T04:25:02.637277+0000","last_clean":"2023-10-09T04:25:02.637277+0000","last_became_active":"2023-10-09T04:22:16.629376+0000","last_became_peered":"2023-10-09T04:22:16.629376+0000","last_unstale":"2023-10-09T04:25:02.637277+0000","last_undegraded":"2023-10-09T04:25:02.637277+0000","last_fullsized":"2023-10-09T04:25:02.637277+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:21:55.959478+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:23.285011+0000","scrub_duration":0.0016555039999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":166,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T04:25:03.658583+0000","last_change":"2023-10-09T04:24:22.443922+0000","last_active":"2023-10-09T04:25:03.658583+0000","last_peered":"2023-10-09T04:25:03.658583+0000","last_clean":"2023-10-09T04:25:03.658583+0000","last_became_active":"2023-10-09T04:24:22.443562+0000","last_became_peered":"2023-10-09T04:24:22.443562+0000","last_unstale":"2023-10-09T04:25:03.658583+0000","last_undegraded":"2023-10-09T04:25:03.658583+0000","last_fullsized":"2023-10-09T04:25:03.658583+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:04.129668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":279,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.645505+0000","last_change":"2023-10-09T04:24:22.449672+0000","last_active":"2023-10-09T04:24:57.645505+0000","last_peered":"2023-10-09T04:24:57.645505+0000","last_clean":"2023-10-09T04:24:57.645505+0000","last_became_active":"2023-10-09T04:24:22.449279+0000","last_became_peered":"2023-10-09T04:24:22.449279+0000","last_unstale":"2023-10-09T04:24:57.645505+0000","last_undegraded":"2023-10-09T04:24:57.645505+0000","last_fullsized":"2023-10-09T04:24:57.645505+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:56.633978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.16","version":"443'5326","reported_seq":7744,"reported_epoch":448,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:24:58.642444+0000","last_change":"2023-10-09T04:24:57.186152+0000","last_active":"2023-10-09T04:24:58.642444+0000","last_peered":"2023-10-09T04:24:58.642444+0000","last_clean":"2023-10-09T04:23:05.250360+0000","last_became_active":"2023-10-09T04:24:57.175942+0000","last_became_peered":"2023-10-09T04:24:57.175942+0000","last_unstale":"2023-10-09T04:24:58.642444+0000","last_undegraded":"2023-10-09T04:24:58.642444+0000","last_fullsized":"2023-10-09T04:24:58.642444+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":428,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:22:09.577227+0000","last_clean_scrub_stamp":"2023-10-09T04:22:09.577227+0000","objects_scrubbed":0,"log_size":5326,"log_dups_size":0,"ondisk_log_size":5326,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1659,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2855,"num_write_kb":144448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":568,"num_bytes_recovered":37191680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1659}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":552,"reported_epoch":447,"state":"activating+undersized+remapped","last_fresh":"2023-10-09T04:24:57.640483+0000","last_change":"2023-10-09T04:24:57.147187+0000","last_active":"2023-10-09T04:24:56.144792+0000","last_peered":"2023-10-09T04:24:50.140672+0000","last_clean":"2023-10-09T04:24:50.140672+0000","last_became_active":"2023-10-09T04:24:22.444410+0000","last_became_peered":"2023-10-09T04:24:22.444410+0000","last_unstale":"2023-10-09T04:24:57.640483+0000","last_undegraded":"2023-10-09T04:24:57.640483+0000","last_fullsized":"2023-10-09T04:24:57.147036+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:31.724263+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[1,3],"avail_no_missing":["1","2","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":57,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.646370+0000","last_change":"2023-10-09T04:23:06.385526+0000","last_active":"2023-10-09T04:24:57.646370+0000","last_peered":"2023-10-09T04:24:57.646370+0000","last_clean":"2023-10-09T04:24:57.646370+0000","last_became_active":"2023-10-09T04:23:06.345850+0000","last_became_peered":"2023-10-09T04:23:06.345850+0000","last_unstale":"2023-10-09T04:24:57.646370+0000","last_undegraded":"2023-10-09T04:24:57.646370+0000","last_fullsized":"2023-10-09T04:24:57.646370+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:52:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-09T04:24:57.644747+0000","last_change":"2023-10-09T04:24:56.183973+0000","last_active":"2023-10-09T04:24:57.644747+0000","last_peered":"2023-10-09T04:24:57.644747+0000","last_clean":"2023-10-09T04:24:57.644747+0000","last_became_active":"2023-10-09T04:24:56.181453+0000","last_became_peered":"2023-10-09T04:24:56.181453+0000","last_unstale":"2023-10-09T04:24:57.644747+0000","last_undegraded":"2023-10-09T04:24:57.644747+0000","last_fullsized":"2023-10-09T04:24:57.644747+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:35.829983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":3462135832,"num_objects":52829,"num_object_clones":0,"num_object_copies":158487,"num_objects_missing_on_primary":1555,"num_objects_missing":1555,"num_objects_degraded":7748,"num_objects_misplaced":19396,"num_objects_unfound":0,"num_objects_dirty":52829,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":95199,"num_write_kb":4736833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33016,"num_bytes_recovered":2149580800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12093435904,"data_stored":12093423688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143465,"ondisk_log_size":143465,"up":78,"acting":72,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":11614,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":715597,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":104,"num_read_kb":195,"num_write":50,"num_write_kb":513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1686409,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":90,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201670,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2182848,"kb_used_data":1274744,"kb_used_omap":0,"kb_used_meta":908096,"kb_avail":91562304,"statfs":{"total":95995035648,"available":93759799296,"internally_reserved":0,"allocated":1305337856,"data_stored":1303995245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":929890304},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:45 2023","interfaces":[{"interface":"back","average":{"1min":4.7069999999999999,"5min":3.8980000000000001,"15min":3.6349999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.20799999999999999},"max":{"1min":70.614000000000004,"5min":148.65899999999999,"15min":148.65899999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":5.9100000000000001,"5min":4.2169999999999996,"15min":3.7320000000000002},"min":{"1min":0.33200000000000002,"5min":0.221,"15min":0.221},"max":{"1min":70.721999999999994,"5min":148.642,"15min":148.642},"last":0.34100000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:24:14 2023","interfaces":[{"interface":"back","average":{"1min":4.8010000000000002,"5min":10.42,"15min":6.851},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":74.933999999999997,"5min":261.935,"15min":261.935},"last":0.309},{"interface":"front","average":{"1min":4.8789999999999996,"5min":10.02,"15min":6.6669999999999998},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":69.355000000000004,"5min":262.07600000000002,"15min":262.07600000000002},"last":0.42499999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:24:45 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":4.5759999999999996,"15min":2.8149999999999999},"min":{"1min":0.313,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":66.328999999999994,"5min":253.41200000000001,"15min":253.41200000000001},"last":0.371},{"interface":"front","average":{"1min":5.4409999999999998,"5min":4.8550000000000004,"15min":3.2170000000000001},"min":{"1min":0.307,"5min":0.29699999999999999,"15min":0.246},"max":{"1min":66.314999999999998,"5min":253.39400000000001,"15min":253.39400000000001},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:24:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":4.2320000000000002,"15min":2.9100000000000001},"min":{"1min":0.42299999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.4859999999999998,"5min":255.97300000000001,"15min":255.97300000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.4119999999999999,"5min":4.5650000000000004,"15min":3.1560000000000001},"min":{"1min":0.42499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.5,"5min":256.06099999999998,"15min":256.06099999999998},"last":0.48599999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:24:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.81000000000000005,"15min":0.76700000000000002},"min":{"1min":0.27900000000000003,"5min":0.26000000000000001,"15min":0.23400000000000001},"max":{"1min":1.577,"5min":4.6150000000000002,"15min":16.195},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.004,"15min":1.095},"min":{"1min":0.29899999999999999,"5min":0.251,"15min":0.251},"max":{"1min":1.6299999999999999,"5min":7.2549999999999999,"15min":12.566000000000001},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:24:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.6140000000000001,"15min":1.5920000000000001},"min":{"1min":0.30099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":17.882999999999999,"5min":17.882999999999999,"15min":120.066},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.7010000000000001,"5min":1.4039999999999999,"15min":1.4430000000000001},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.26600000000000001},"max":{"1min":17.568999999999999,"5min":17.568999999999999,"15min":120.09699999999999},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 04:24:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.286,"15min":1.1299999999999999},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.307},"max":{"1min":1.9890000000000001,"5min":7.3310000000000004,"15min":8.4550000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.032,"15min":0.88400000000000001},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":1.6639999999999999,"5min":5.968,"15min":8.9009999999999998},"last":0.46899999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463306,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1792316,"kb_used_data":1478644,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":91952836,"statfs":{"total":95995035648,"available":94159704064,"internally_reserved":0,"allocated":1514131456,"data_stored":1512800793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:42 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":2.9729999999999999,"15min":3.7570000000000001},"min":{"1min":0.222,"5min":0.19800000000000001,"15min":0.193},"max":{"1min":3.2280000000000002,"5min":133.78999999999999,"15min":133.78999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.871,"5min":2.9350000000000001,"15min":3.9129999999999998},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":3.9990000000000001,"5min":133.69300000000001,"15min":133.69300000000001},"last":0.36099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":5.5119999999999996,"5min":9.2569999999999997,"15min":5.3200000000000003},"min":{"1min":0.30099999999999999,"5min":0.24299999999999999,"15min":0.22800000000000001},"max":{"1min":64.738,"5min":315.73599999999999,"15min":315.73599999999999},"last":0.315},{"interface":"front","average":{"1min":5.2629999999999999,"5min":9.6010000000000009,"15min":5.4169999999999998},"min":{"1min":0.26900000000000002,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":64.789000000000001,"5min":316.91500000000002,"15min":316.91500000000002},"last":0.442}]},{"osd":2,"last update":"Mon Oct 9 04:24:42 2023","interfaces":[{"interface":"back","average":{"1min":5.016,"5min":1.9710000000000001,"15min":2.9260000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.23899999999999999},"max":{"1min":59.460999999999999,"5min":59.460999999999999,"15min":139.06899999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":5.0599999999999996,"5min":2.0150000000000001,"15min":2.968},"min":{"1min":0.34499999999999997,"5min":0.33700000000000002,"15min":0.28599999999999998},"max":{"1min":59.491,"5min":59.491,"15min":139.173},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":2.0920000000000001,"15min":2.2719999999999998},"min":{"1min":0.308,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.0759999999999996,"5min":43.475999999999999,"15min":53.497999999999998},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.8480000000000001,"15min":2.0960000000000001},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.6059999999999999,"5min":43.369,"15min":53.460999999999999},"last":0.48699999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":2.5259999999999998,"15min":1.3560000000000001},"min":{"1min":0.31900000000000001,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":3.161,"5min":161.27500000000001,"15min":161.27500000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":2.6739999999999999,"15min":1.472},"min":{"1min":0.34799999999999998,"5min":0.27100000000000002,"15min":0.22600000000000001},"max":{"1min":0.92500000000000004,"5min":160.32599999999999,"15min":160.32599999999999},"last":0.42299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.95899999999999996,"15min":1.03},"min":{"1min":0.36899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.6309999999999998,"5min":7.032,"15min":22.879000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95999999999999996,"15min":1.0720000000000001},"min":{"1min":0.34799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.369,"5min":6.4109999999999996,"15min":23.006},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:24:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":2.585,"15min":1.5780000000000001},"min":{"1min":0.40500000000000003,"5min":0.32600000000000001,"15min":0.313},"max":{"1min":3.1930000000000001,"5min":161.39400000000001,"15min":161.39400000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.4889999999999999,"15min":1.5940000000000001},"min":{"1min":0.34499999999999997,"5min":0.33900000000000002,"15min":0.27800000000000002},"max":{"1min":0.80900000000000005,"5min":161.41200000000001,"15min":161.41200000000001},"last":0.47399999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089616,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1916556,"kb_used_data":1211460,"kb_used_omap":9,"kb_used_meta":705078,"kb_avail":91828596,"statfs":{"total":95995035648,"available":94032482304,"internally_reserved":0,"allocated":1240535040,"data_stored":1238507851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":722000873},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":5.9279999999999999,"15min":5.1459999999999999},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17199999999999999},"max":{"1min":8.2149999999999999,"5min":259.26799999999997,"15min":259.26799999999997},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.3340000000000001,"5min":5.8769999999999998,"15min":5.867},"min":{"1min":0.23899999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":7.8929999999999998,"5min":258.46899999999999,"15min":258.46899999999999},"last":0.252}]},{"osd":2,"last update":"Mon Oct 9 04:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":5.0030000000000001,"15min":3.3479999999999999},"min":{"1min":0.33900000000000002,"5min":0.20699999999999999,"15min":0.191},"max":{"1min":8.0890000000000004,"5min":259.315,"15min":259.315},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.9059999999999999,"5min":5.6050000000000004,"15min":3.7690000000000001},"min":{"1min":0.249,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":11.029999999999999,"5min":259.733,"15min":259.733},"last":0.22}]},{"osd":3,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":2.246,"15min":3.48},"min":{"1min":0.30499999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.401999999999999,"5min":19.640000000000001,"15min":138.87299999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.169,"15min":3.944},"min":{"1min":0.29799999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.2629999999999999,"5min":12.353999999999999,"15min":138.49799999999999},"last":0.28499999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":5.0069999999999997,"15min":3.028},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.1910000000000007,"5min":257.83300000000003,"15min":257.83300000000003},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.121,"5min":4.4930000000000003,"15min":2.7869999999999999},"min":{"1min":0.35699999999999998,"5min":0.30499999999999999,"15min":0.25900000000000001},"max":{"1min":4.4809999999999999,"5min":257.863,"15min":257.863},"last":0.29199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":2.1429999999999998,"15min":2.3170000000000002},"min":{"1min":0.313,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":8.2300000000000004,"5min":25.696999999999999,"15min":38.848999999999997},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.0469999999999999,"5min":2.0910000000000002,"15min":2.3380000000000001},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":2.8849999999999998,"5min":21.893999999999998,"15min":39.593000000000004},"last":0.32400000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.6100000000000001,"15min":1.952},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.30299999999999999},"max":{"1min":3.0169999999999999,"5min":8.3529999999999998,"15min":25.931000000000001},"last":0.439},{"interface":"front","average":{"1min":1.593,"5min":1.633,"15min":1.6659999999999999},"min":{"1min":0.373,"5min":0.29999999999999999,"15min":0.29799999999999999},"max":{"1min":8.1630000000000003,"5min":19.440000000000001,"15min":19.440000000000001},"last":1.3120000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:24:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":6.8390000000000004,"15min":3.8420000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.9390000000000001,"5min":259.08499999999998,"15min":259.08499999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.8009999999999999,"5min":6.6509999999999998,"15min":3.5110000000000001},"min":{"1min":0.40699999999999997,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":8.1899999999999995,"5min":259.13299999999998,"15min":259.13299999999998},"last":0.46600000000000003}]}]},{"osd":0,"up_from":84,"seq":360777253141,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963860,"kb_used_data":1244620,"kb_used_omap":28,"kb_used_meta":719203,"kb_avail":91781292,"statfs":{"total":95995035648,"available":93984043008,"internally_reserved":0,"allocated":1274490880,"data_stored":1272380980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":736464720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:24:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.8650000000000002,"15min":4.7210000000000001},"min":{"1min":0.224,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":5.9139999999999997,"5min":96.305000000000007,"15min":209.215},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.2130000000000001,"5min":4.4500000000000002,"15min":4.8159999999999998},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.383,"5min":95.861000000000004,"15min":120.934},"last":0.33400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:24:51 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":3.8149999999999999,"15min":2.8839999999999999},"min":{"1min":0.29899999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.3449999999999998,"5min":88.088999999999999,"15min":88.088999999999999},"last":0.434},{"interface":"front","average":{"1min":1.992,"5min":3.9399999999999999,"15min":3.177},"min":{"1min":0.29099999999999998,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":7.3620000000000001,"5min":88.004999999999995,"15min":88.004999999999995},"last":0.36799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:24:20 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":3.306,"15min":3.7719999999999998},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.192},"max":{"1min":5.8710000000000004,"5min":87.891999999999996,"15min":156.488},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.278,"5min":3.3130000000000002,"15min":3.863},"min":{"1min":0.39300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":7.1379999999999999,"5min":88.147999999999996,"15min":156.50700000000001},"last":0.38300000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":2.399,"15min":2.4140000000000001},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":4.7530000000000001,"5min":88.058000000000007,"15min":88.058000000000007},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.23,"5min":1.8680000000000001,"15min":2.2719999999999998},"min":{"1min":0.378,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.5920000000000001,"5min":26.734000000000002,"15min":49.607999999999997},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":2.2989999999999999,"15min":3.1509999999999998},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":3.5569999999999999,"5min":49.526000000000003,"15min":156.96100000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.149,"5min":2.0150000000000001,"15min":2.7130000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":4.2060000000000004,"5min":49.356999999999999,"15min":156.97300000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.5820000000000001,"15min":2.4500000000000002},"min":{"1min":0.44,"5min":0.37,"15min":0.30599999999999999},"max":{"1min":4.2629999999999999,"5min":7.8700000000000001,"15min":137.36099999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.387,"5min":2.4470000000000001,"15min":2.8860000000000001},"min":{"1min":0.46100000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.9470000000000001,"5min":88.180000000000007,"15min":136.88999999999999},"last":0.46100000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.742,"15min":2.3370000000000002},"min":{"1min":0.41199999999999998,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":4.2350000000000003,"5min":11.298999999999999,"15min":121.324},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.722,"15min":2.532},"min":{"1min":0.438,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.8780000000000001,"5min":88.120000000000005,"15min":88.120000000000005},"last":0.51600000000000001}]}]},{"osd":2,"up_from":94,"seq":403726926090,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2168992,"kb_used_data":1460568,"kb_used_omap":81,"kb_used_meta":708334,"kb_avail":91576160,"statfs":{"total":95995035648,"available":93773987840,"internally_reserved":0,"allocated":1495621632,"data_stored":1493581694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":725334234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":1.8660000000000001,"15min":3.7759999999999998},"min":{"1min":0.216,"5min":0.17699999999999999,"15min":0.17399999999999999},"max":{"1min":9.9440000000000008,"5min":21.792000000000002,"15min":81.385999999999996},"last":0.28399999999999997},{"interface":"front","average":{"1min":2.5680000000000001,"5min":2.1869999999999998,"15min":3.5880000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.20000000000000001},"max":{"1min":9.8569999999999993,"5min":21.603000000000002,"15min":79.453999999999994},"last":0.22600000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":6.8920000000000003,"5min":4.7720000000000002,"15min":4.641},"min":{"1min":0.27100000000000002,"5min":0.19700000000000001,"15min":0.191},"max":{"1min":115.184,"5min":115.184,"15min":158.13399999999999},"last":1.8100000000000001},{"interface":"front","average":{"1min":6.0350000000000001,"5min":4.548,"15min":4.6859999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20799999999999999},"max":{"1min":113.94,"5min":113.94,"15min":161.12},"last":1.056}]},{"osd":3,"last update":"Mon Oct 9 04:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.877,"15min":2.5720000000000001},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.20599999999999999},"max":{"1min":4.093,"5min":28.611999999999998,"15min":58.997},"last":1.077},{"interface":"front","average":{"1min":2.4940000000000002,"5min":2.54,"15min":2.7610000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23499999999999999},"max":{"1min":8.0549999999999997,"5min":27.733000000000001,"15min":66.135000000000005},"last":0.32400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.9359999999999999,"15min":1.8899999999999999},"min":{"1min":0.32900000000000001,"5min":0.28499999999999998,"15min":0.25900000000000001},"max":{"1min":7.1429999999999998,"5min":24.814,"15min":58.798000000000002},"last":1.23},{"interface":"front","average":{"1min":0.82499999999999996,"5min":1.224,"15min":1.8879999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.27300000000000002},"max":{"1min":4.133,"5min":24.021000000000001,"15min":56.616},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":3.056,"15min":2.5649999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.26400000000000001},"max":{"1min":7.609,"5min":82.030000000000001,"15min":82.030000000000001},"last":0.374},{"interface":"front","average":{"1min":1.274,"5min":2.8620000000000001,"15min":2.4870000000000001},"min":{"1min":0.34300000000000003,"5min":0.34100000000000003,"15min":0.28100000000000003},"max":{"1min":7.1109999999999998,"5min":82.016000000000005,"15min":82.016000000000005},"last":1.0940000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.105,"15min":2.3719999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.308},"max":{"1min":4.1689999999999996,"5min":7.3230000000000004,"15min":114.14},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.0219999999999998,"5min":2.0139999999999998,"15min":2.484},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.26800000000000002},"max":{"1min":7.5759999999999996,"5min":15.456,"15min":114.123},"last":0.51400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":2.2989999999999999,"15min":2.8559999999999999},"min":{"1min":0.42199999999999999,"5min":0.38,"15min":0.33700000000000002},"max":{"1min":4.1959999999999997,"5min":101.675,"15min":117.32599999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.0630000000000002,"5min":3.25,"15min":2.9950000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":7.6390000000000002,"5min":101.72499999999999,"15min":117.28700000000001},"last":0.55700000000000005}]}]},{"osd":3,"up_from":99,"seq":425201762564,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2521472,"kb_used_data":1641528,"kb_used_omap":7,"kb_used_meta":879928,"kb_avail":91223680,"statfs":{"total":95995035648,"available":93413048320,"internally_reserved":0,"allocated":1680924672,"data_stored":1678865641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":901046888},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.0379999999999998,"15min":4.1760000000000002},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.16900000000000001},"max":{"1min":9.3049999999999997,"5min":10.672000000000001,"15min":125.765},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.738,"15min":4.7190000000000003},"min":{"1min":0.25700000000000001,"5min":0.185,"15min":0.185},"max":{"1min":4.7679999999999998,"5min":22.446999999999999,"15min":125.46899999999999},"last":0.307}]},{"osd":1,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":4.117,"15min":5.2699999999999996},"min":{"1min":0.22,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.7889999999999997,"5min":170.196,"15min":186.21299999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.1819999999999999,"5min":3.9830000000000001,"15min":5.1689999999999996},"min":{"1min":0.23000000000000001,"5min":0.221,"15min":0.193},"max":{"1min":14.042,"5min":166.953,"15min":191.78},"last":0.53400000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.4910000000000001,"15min":2.8759999999999999},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.187},"max":{"1min":9.2720000000000002,"5min":9.2720000000000002,"15min":127.599},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.1190000000000002,"5min":2.3130000000000002,"15min":4.101},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.214},"max":{"1min":12.032999999999999,"5min":12.032999999999999,"15min":129.98599999999999},"last":0.34000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":3.6699999999999999,"15min":2.5579999999999998},"min":{"1min":0.34200000000000003,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":4.8280000000000003,"5min":211.84700000000001,"15min":211.84700000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.8939999999999999,"5min":3.8879999999999999,"15min":2.673},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":9.7149999999999999,"5min":211.17500000000001,"15min":211.17500000000001},"last":0.38}]},{"osd":5,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":2.036,"15min":2.161},"min":{"1min":0.35599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.6709999999999994,"5min":11.772,"15min":17.901},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.419,"15min":1.6919999999999999},"min":{"1min":0.35799999999999998,"5min":0.29699999999999999,"15min":0.248},"max":{"1min":7.7530000000000001,"5min":8.4670000000000005,"15min":18.068000000000001},"last":1.1439999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.218,"15min":1.5880000000000001},"min":{"1min":0.376,"5min":0.33500000000000002,"15min":0.26500000000000001},"max":{"1min":7.7050000000000001,"5min":7.7050000000000001,"15min":14.976000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.288,"5min":1.6799999999999999,"15min":1.988},"min":{"1min":0.371,"5min":0.371,"15min":0.29999999999999999},"max":{"1min":4.7960000000000003,"5min":10.071,"15min":14.529},"last":0.55000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:24:15 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":4.1429999999999998,"15min":2.7759999999999998},"min":{"1min":0.38,"5min":0.35399999999999998,"15min":0.34399999999999997},"max":{"1min":4.7290000000000001,"5min":212.62700000000001,"15min":212.62700000000001},"last":1.2529999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":4.3620000000000001,"15min":2.956},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.7579999999999991,"5min":212.04599999999999,"15min":212.04599999999999},"last":0.45700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986579,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2689524,"kb_used_data":2099756,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":91055628,"statfs":{"total":95995035648,"available":93240963072,"internally_reserved":0,"allocated":2150150144,"data_stored":2148818942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":2.5819999999999999,"15min":5.6040000000000001},"min":{"1min":0.22500000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":12.194000000000001,"5min":72.759,"15min":154.62100000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.48,"5min":2.6360000000000001,"15min":5.7199999999999998},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23400000000000001},"max":{"1min":17.827000000000002,"5min":72.704999999999998,"15min":154.37299999999999},"last":0.38500000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:24:12 2023","interfaces":[{"interface":"back","average":{"1min":6.7350000000000003,"5min":3.9039999999999999,"15min":5.1520000000000001},"min":{"1min":0.24399999999999999,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":90.900000000000006,"5min":90.900000000000006,"15min":135.25399999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":8.0039999999999996,"5min":3.996,"15min":5.0730000000000004},"min":{"1min":0.28499999999999998,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":90.787000000000006,"5min":90.787000000000006,"15min":134.44300000000001},"last":0.43099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":5.0099999999999998,"15min":3.093},"min":{"1min":0.26900000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.6489999999999991,"5min":170.536,"15min":170.536},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.639,"5min":5.141,"15min":3.0870000000000002},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.30599999999999999},"max":{"1min":8.6310000000000002,"5min":170.55500000000001,"15min":170.55500000000001},"last":0.47099999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:24:12 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":3.4849999999999999,"15min":2.7480000000000002},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27100000000000002},"max":{"1min":30.789999999999999,"5min":154.13200000000001,"15min":154.13200000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.546,"5min":3.3730000000000002,"15min":2.883},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":9.2710000000000008,"5min":154.077,"15min":154.077},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:24:06 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.0740000000000001,"15min":1.6000000000000001},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":2.4329999999999998,"5min":13.599,"15min":111.809},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.556,"15min":1.827},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.25},"max":{"1min":5.0679999999999996,"5min":9.1530000000000005,"15min":112.002},"last":0.33500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":2.0219999999999998,"15min":1.5940000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":4.976,"5min":47.801000000000002,"15min":76.307000000000002},"last":0.376},{"interface":"front","average":{"1min":0.52700000000000002,"5min":1.738,"15min":1.444},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":1.2310000000000001,"5min":47.753,"15min":78.421999999999997},"last":0.505}]},{"osd":7,"last update":"Mon Oct 9 04:24:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.26,"15min":1.048},"min":{"1min":0.309,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":1.5449999999999999,"5min":8.7789999999999999,"15min":34.039000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":1.4770000000000001,"15min":1.194},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.29199999999999998},"max":{"1min":1.304,"5min":7.5430000000000001,"15min":33.433999999999997},"last":0.40899999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724943,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2364008,"kb_used_data":1471456,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":91381144,"statfs":{"total":95995035648,"available":93574291456,"internally_reserved":0,"allocated":1506770944,"data_stored":1505448260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:24:51 2023","interfaces":[{"interface":"back","average":{"1min":14.598000000000001,"5min":4.8479999999999999,"15min":5.9169999999999998},"min":{"1min":0.26900000000000002,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":139.86799999999999,"5min":139.86799999999999,"15min":151.23599999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":14.539,"5min":5.0110000000000001,"15min":6.2709999999999999},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":139.84899999999999,"5min":139.84899999999999,"15min":151.566},"last":0.70699999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:24:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":8.0739999999999998,"15min":5.4889999999999999},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":22.722999999999999,"5min":296.23099999999999,"15min":296.23099999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.5510000000000002,"5min":8.3330000000000002,"15min":5.7380000000000004},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":22.437000000000001,"5min":295.065,"15min":295.065},"last":0.68200000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:24:51 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":5.165,"15min":3.77},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.24399999999999999},"max":{"1min":3.2559999999999998,"5min":168.779,"15min":168.779},"last":0.64800000000000002},{"interface":"front","average":{"1min":3.3959999999999999,"5min":5.4950000000000001,"15min":3.6779999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.25600000000000001},"max":{"1min":41.485999999999997,"5min":168.03100000000001,"15min":168.03100000000001},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:24:07 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":2.274,"15min":2.093},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":8.9350000000000005,"5min":41,"15min":41},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.3280000000000001,"5min":2.258,"15min":2.1840000000000002},"min":{"1min":0.38,"5min":0.317,"15min":0.23499999999999999},"max":{"1min":8.9169999999999998,"5min":40.988,"15min":40.988},"last":0.56399999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:24:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.03,"15min":0.95099999999999996},"min":{"1min":0.26800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.1989999999999998,"5min":6.0899999999999999,"15min":10.162000000000001},"last":0.63},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.026,"15min":0.91300000000000003},"min":{"1min":0.27900000000000003,"5min":0.27500000000000002,"15min":0.26500000000000001},"max":{"1min":0.58899999999999997,"5min":7.5890000000000004,"15min":11.297000000000001},"last":0.35199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":1.0229999999999999,"15min":1.341},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":0.91200000000000003,"5min":9.4090000000000007,"15min":123.239},"last":0.377},{"interface":"front","average":{"1min":0.627,"5min":0.99299999999999999,"15min":1.4399999999999999},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.29799999999999999},"max":{"1min":2.0489999999999999,"5min":6.1689999999999996,"15min":123.014},"last":0.66500000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:24:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.133,"15min":0.98099999999999998},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":1.075,"5min":9.3800000000000008,"15min":12.096},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.68600000000000005,"5min":1.141,"15min":1.077},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.307},"max":{"1min":3.238,"5min":6.1459999999999999,"15min":9.9760000000000009},"last":0.60399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1268518912,"data_stored":1268514840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1234239488,"data_stored":1234239488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1481965568,"data_stored":1481965568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1672679424,"data_stored":1672675352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2124939264,"data_stored":2124939264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1501757440,"data_stored":1501757440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1509036032,"data_stored":1509031960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1300299776,"data_stored":1300299776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:25:09.593 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:25:09.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:09 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:09 smithi181 ceph-mon[196979]: failed to encode map e453 with expected crc 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:09 smithi181 ceph-mon[196979]: failed to encode map e453 with expected crc 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1958324256' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:09 smithi181 ceph-mon[196979]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T04:25:09.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:09.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[221067]: failed to encode map e453 with expected crc 2023-10-09T04:25:09.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[221067]: failed to encode map e453 with expected crc 2023-10-09T04:25:09.911 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1958324256' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:25:09.912 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[221067]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T04:25:09.912 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:09.912 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[213841]: failed to encode map e453 with expected crc 2023-10-09T04:25:09.912 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[213841]: failed to encode map e453 with expected crc 2023-10-09T04:25:09.913 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1958324256' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:25:09.913 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:09 smithi163 ceph-mon[213841]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T04:25:09.920 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:09.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:10.026 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:10.069 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:10.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:10.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:10.233 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:10.268 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:10.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:10.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:10.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:10.674 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:10.722 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:10.782 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:10.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:10.800 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:10.831 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:10 smithi163 ceph-mon[213841]: pgmap v1196: 129 pgs: 1 active+recovering+undersized+remapped, 114 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 5310/155814 objects degraded (3.408%); 19369/155814 objects misplaced (12.431%); 0 B/s, 61 objects/s recovering 2023-10-09T04:25:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:25:10] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.43.0" 2023-10-09T04:25:10.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:10 smithi163 ceph-mon[221067]: pgmap v1196: 129 pgs: 1 active+recovering+undersized+remapped, 114 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 5310/155814 objects degraded (3.408%); 19369/155814 objects misplaced (12.431%); 0 B/s, 61 objects/s recovering 2023-10-09T04:25:10.889 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:10.922 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:10.938 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:10.938 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:25:11.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:10 smithi181 ceph-mon[196979]: pgmap v1196: 129 pgs: 1 active+recovering+undersized+remapped, 114 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 5310/155814 objects degraded (3.408%); 19369/155814 objects misplaced (12.431%); 0 B/s, 61 objects/s recovering 2023-10-09T04:25:11.176 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:11.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:11.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:11.288 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:11.315 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[221067]: failed to encode map e454 with expected crc 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[221067]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[221067]: failed to encode map e454 with expected crc 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[221067]: failed to encode map e454 with expected crc 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[221067]: 10.7 deep-scrub ok 2023-10-09T04:25:11.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[213841]: failed to encode map e454 with expected crc 2023-10-09T04:25:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[213841]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T04:25:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[213841]: failed to encode map e454 with expected crc 2023-10-09T04:25:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[213841]: failed to encode map e454 with expected crc 2023-10-09T04:25:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:11.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:11 smithi163 ceph-mon[213841]: 10.7 deep-scrub ok 2023-10-09T04:25:11.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:25:12.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:11 smithi181 ceph-mon[196979]: failed to encode map e454 with expected crc 2023-10-09T04:25:12.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:11 smithi181 ceph-mon[196979]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T04:25:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:11 smithi181 ceph-mon[196979]: failed to encode map e454 with expected crc 2023-10-09T04:25:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:11 smithi181 ceph-mon[196979]: failed to encode map e454 with expected crc 2023-10-09T04:25:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:11 smithi181 ceph-mon[196979]: 10.7 deep-scrub ok 2023-10-09T04:25:12.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:25:12.324 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:12.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:12.427 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:12.454 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:12.611 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:12.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:25:12.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T04:25:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: pgmap v1199: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 1583/153417 objects degraded (1.032%); 21190/153417 objects misplaced (13.812%); 3.5 MiB/s, 251 objects/s recovering 2023-10-09T04:25:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:12 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 5310/155814 objects degraded (3.408%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:12.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T04:25:12.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.773 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: pgmap v1199: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 1583/153417 objects degraded (1.032%); 21190/153417 objects misplaced (13.812%); 3.5 MiB/s, 251 objects/s recovering 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 5310/155814 objects degraded (3.408%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: failed to encode map e455 with expected crc 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: pgmap v1199: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 1583/153417 objects degraded (1.032%); 21190/153417 objects misplaced (13.812%); 3.5 MiB/s, 251 objects/s recovering 2023-10-09T04:25:12.774 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:12 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 5310/155814 objects degraded (3.408%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:12.839 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:12.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:12.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:12.947 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:12.975 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:13.028 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:13.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:13.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:13.132 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:13.174 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:13.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-10-09T04:25:13.683 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:13.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:13.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/571036481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T04:25:13.696 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:13 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T04:25:13.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:13.819 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:14.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/571036481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T04:25:14.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:13 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T04:25:14.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/571036481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T04:25:14.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:13 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-10-09T04:25:14.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:14 smithi163 ceph-mon[221067]: pgmap v1200: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 1580/152685 objects degraded (1.035%); 21190/152685 objects misplaced (13.878%); 4.3 MiB/s, 187 objects/s recovering 2023-10-09T04:25:14.822 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:14 smithi163 ceph-mon[213841]: pgmap v1200: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 1580/152685 objects degraded (1.035%); 21190/152685 objects misplaced (13.878%); 4.3 MiB/s, 187 objects/s recovering 2023-10-09T04:25:14.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:14.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:14.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:14.954 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:15.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:14 smithi181 ceph-mon[196979]: pgmap v1200: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 1580/152685 objects degraded (1.035%); 21190/152685 objects misplaced (13.878%); 4.3 MiB/s, 187 objects/s recovering 2023-10-09T04:25:15.305 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:15.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:15.364 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:25:15.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:15.461 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:15.490 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:15.551 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:15.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:15.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:15.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:15.691 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:15.894 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:15.894 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:25:15.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:15.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:16.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:16.038 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:16.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:16.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:16.194 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:16.232 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:16.296 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:16.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:16.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:16.430 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:16.473 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:16.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:16.577 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:16.612 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:17.211 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:25:17.233 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[221067]: pgmap v1201: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 1580/151758 objects degraded (1.041%); 20901/151758 objects misplaced (13.773%); 3.6 MiB/s, 154 objects/s recovering 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1440553340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[213841]: pgmap v1201: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 1580/151758 objects degraded (1.041%); 20901/151758 objects misplaced (13.773%); 3.6 MiB/s, 154 objects/s recovering 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1440553340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:17.234 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:17 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:17.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:17 smithi181 ceph-mon[196979]: pgmap v1201: 129 pgs: 1 peering, 1 active+recovering+undersized+remapped, 113 active+clean, 1 active+clean+scrubbing+deep, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 232 op/s; 1580/151758 objects degraded (1.041%); 20901/151758 objects misplaced (13.773%); 3.6 MiB/s, 154 objects/s recovering 2023-10-09T04:25:17.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:17.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:17.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1440553340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:17.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:17 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:17.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:17.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:17.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:17.550 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:17.603 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:17.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:17.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:17.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:17.737 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:17.789 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:17.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:17.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:17.924 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.014 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:18 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/151758 objects degraded (1.041%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.015 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.016 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/151758 objects degraded (1.041%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:18.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:25:18.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:18.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:18.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:18.276 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:18.304 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:18.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:18.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:18.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T04:25:18.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.314 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: failed to encode map e456 with expected crc 2023-10-09T04:25:18.315 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:18 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/151758 objects degraded (1.041%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:18.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:18.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:18.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:18.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:18.485 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:18.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:18.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:18.567 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:18.596 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:18.622 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:18.648 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:18.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:18.670 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:18.684 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:18.697 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:18.728 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:18.754 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:18.756 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.756 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.756 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.756 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.756 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.757 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.757 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.757 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:25:18.806 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:18.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:18.899 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:18.913 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:18.938 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:18.985 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:18.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:19.014 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:19 smithi163 ceph-mon[213841]: failed to encode map e456 with expected crc 2023-10-09T04:25:19.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:19 smithi163 ceph-mon[221067]: failed to encode map e456 with expected crc 2023-10-09T04:25:19.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:19 smithi163 ceph-mon[221067]: pgmap v1203: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/148401 objects degraded (1.065%); 20508/148401 objects misplaced (13.819%); 4.2 MiB/s, 165 objects/s recovering 2023-10-09T04:25:19.015 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1574914280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:19.091 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:19.130 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:19.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:19.192 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:19 smithi181 ceph-mon[196979]: failed to encode map e456 with expected crc 2023-10-09T04:25:19.192 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:19 smithi181 ceph-mon[196979]: pgmap v1203: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/148401 objects degraded (1.065%); 20508/148401 objects misplaced (13.819%); 4.2 MiB/s, 165 objects/s recovering 2023-10-09T04:25:19.193 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1574914280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:19.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:19.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:19.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:19 smithi163 ceph-mon[213841]: pgmap v1203: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/148401 objects degraded (1.065%); 20508/148401 objects misplaced (13.819%); 4.2 MiB/s, 165 objects/s recovering 2023-10-09T04:25:19.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1574914280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:19.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:19.327 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:19.376 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:19.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:19.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:19.511 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:19.570 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:19.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:19.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:19.709 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:19.775 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:19.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:19.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:19.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:19.921 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:20.068 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:25:20.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:20.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:20.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:20.366 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:25:20.366 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:25:20.366 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:25:20.366 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:25:20.367 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:25:20.367 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:25:20.377 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[221067]: pgmap v1204: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/147456 objects degraded (1.072%); 20275/147456 objects misplaced (13.750%); 619 KiB/s, 9 objects/s recovering 2023-10-09T04:25:20.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:20.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:20 smithi163 ceph-mon[213841]: pgmap v1204: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/147456 objects degraded (1.072%); 20275/147456 objects misplaced (13.750%); 619 KiB/s, 9 objects/s recovering 2023-10-09T04:25:20.710 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:20.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:20.764 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:25:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:20 smithi181 ceph-mon[196979]: pgmap v1204: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/147456 objects degraded (1.072%); 20275/147456 objects misplaced (13.750%); 619 KiB/s, 9 objects/s recovering 2023-10-09T04:25:20.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:20.842 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:20.891 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:20.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:20.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:21.007 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:21.051 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:25:20] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2023-10-09T04:25:21.337 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:25:21.337 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:25:21.337 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:25:21.337 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:25:21.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:21.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:21.355 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:21.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:21.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:21.355 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:21.367 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:21.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:21.474 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:21.505 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:21.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:21.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:22.066 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:25:22.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:25:22.209 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:22.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:22.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:22.342 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:22.371 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:22 smithi163 ceph-mon[213841]: pgmap v1205: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/144708 objects degraded (1.092%); 19904/144708 objects misplaced (13.755%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:22.371 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:22 smithi163 ceph-mon[221067]: pgmap v1205: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/144708 objects degraded (1.092%); 19904/144708 objects misplaced (13.755%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:22.730 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:22.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:22.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:22 smithi181 ceph-mon[196979]: pgmap v1205: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/144708 objects degraded (1.092%); 19904/144708 objects misplaced (13.755%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:22.835 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:22.875 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:22.984 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:23.011 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:23.045 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:23.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:23.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:23.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:23.186 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:23.234 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:23.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:23.339 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:23.367 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:23.413 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:23.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:23.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:23.470 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:25:23.470 INFO:teuthology.orchestra.run.smithi181.stdout: "success": "" 2023-10-09T04:25:23.470 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:25:23.495 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:23 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/144708 objects degraded (1.092%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:23.521 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:23.561 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:23.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:23 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/144708 objects degraded (1.092%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:23.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:23 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/144708 objects degraded (1.092%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:24.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:24.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:24.129 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:24.157 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:24.344 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:25:24.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/745189254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:24.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:24 smithi163 ceph-mon[221067]: pgmap v1206: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/143910 objects degraded (1.098%); 19904/143910 objects misplaced (13.831%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T04:25:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/745189254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:24.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:24 smithi163 ceph-mon[213841]: pgmap v1206: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/143910 objects degraded (1.098%); 19904/143910 objects misplaced (13.831%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T04:25:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/745189254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:24.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:24 smithi181 ceph-mon[196979]: pgmap v1206: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1580/143910 objects degraded (1.098%); 19904/143910 objects misplaced (13.831%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T04:25:24.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:24.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:24.919 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:24.948 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:25.315 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:25.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:25.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:25.452 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:25.497 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:25.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:25.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:25.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.567 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/745189254' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:25:25.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T04:25:25.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:25:25.568 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/745189254' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.569 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:25 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:25:25.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/745189254' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: failed to encode map e457 with expected crc 2023-10-09T04:25:25.636 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:25 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:25:25.639 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:25.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:25.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:25.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:25.819 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:25.819 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:25.841 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:25.862 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:25.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:25.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:25.883 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:25.902 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:25.911 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:25.911 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:25:25.921 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:25.940 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:25.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:25.959 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.960 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:25:25.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:26.005 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:26.066 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:26.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:26.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:26.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:26.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:26.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:26.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:26.353 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:26.408 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:26.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1655116817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:26.664 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:26 smithi163 ceph-mon[221067]: pgmap v1208: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 1580/143130 objects degraded (1.104%); 19675/143130 objects misplaced (13.746%); 672 KiB/s, 10 objects/s recovering 2023-10-09T04:25:26.664 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1655116817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:26.664 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:26 smithi163 ceph-mon[213841]: pgmap v1208: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 1580/143130 objects degraded (1.104%); 19675/143130 objects misplaced (13.746%); 672 KiB/s, 10 objects/s recovering 2023-10-09T04:25:26.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:25:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1655116817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:26.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:26 smithi181 ceph-mon[196979]: pgmap v1208: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 1580/143130 objects degraded (1.104%); 19675/143130 objects misplaced (13.746%); 672 KiB/s, 10 objects/s recovering 2023-10-09T04:25:26.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:26.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:26.888 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:26.914 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:26.961 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:26.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:27.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:27.097 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:27.212 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:27.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:27.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:27.345 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:27 smithi163 ceph-mon[213841]: 10.19 scrub starts 2023-10-09T04:25:27.345 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:27 smithi163 ceph-mon[221067]: 10.19 scrub starts 2023-10-09T04:25:27.346 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:27.351 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:27.364 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:25:27.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:27.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:27.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:27.527 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:27.629 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:27.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:27.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:27.767 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:27 smithi181 ceph-mon[196979]: 10.19 scrub starts 2023-10-09T04:25:27.769 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:27.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:27.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:27.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:27.929 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:27.972 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:28.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:28.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:28.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:28.151 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:28.195 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:28.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:28.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:28.299 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:28.325 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:28.526 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:28.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:28.556 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:28.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:28 smithi163 ceph-mon[221067]: 10.17 scrub starts 2023-10-09T04:25:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:28 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/143130 objects degraded (1.104%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:28 smithi163 ceph-mon[221067]: pgmap v1209: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 1580/140037 objects degraded (1.128%); 19332/140037 objects misplaced (13.805%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:28 smithi163 ceph-mon[213841]: 10.17 scrub starts 2023-10-09T04:25:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:28 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/143130 objects degraded (1.104%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:28.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:28 smithi163 ceph-mon[213841]: pgmap v1209: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 1580/140037 objects degraded (1.128%); 19332/140037 objects misplaced (13.805%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:28.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:28.655 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:28 smithi181 ceph-mon[196979]: 10.17 scrub starts 2023-10-09T04:25:28.655 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:28 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/143130 objects degraded (1.104%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:28.655 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:28 smithi181 ceph-mon[196979]: pgmap v1209: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 1580/140037 objects degraded (1.128%); 19332/140037 objects misplaced (13.805%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:28.658 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:28.722 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:28.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:28.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:28.862 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:28.915 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:28.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:28.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:29.020 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:29.056 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:29.152 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:29.153 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:25:29.271 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:25:29.324 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:25:29.325 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:25:29.325 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:25:29.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T04:25:29.361 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:29 smithi163 ceph-mon[221067]: 10.13 scrub starts 2023-10-09T04:25:29.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:29 smithi163 ceph-mon[213841]: 10.13 scrub starts 2023-10-09T04:25:29.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:29 smithi181 ceph-mon[196979]: 10.13 scrub starts 2023-10-09T04:25:30.021 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:30.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:30.124 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:30.152 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:30.215 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:30.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:30.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:30.349 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:30.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:30.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:30.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:30.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:25:30.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:30 smithi181 ceph-mon[196979]: 10.13 scrub ok 2023-10-09T04:25:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:30 smithi181 ceph-mon[196979]: 10.3 scrub starts 2023-10-09T04:25:30.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:30 smithi181 ceph-mon[196979]: pgmap v1210: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 1580/139071 objects degraded (1.136%); 19102/139071 objects misplaced (13.735%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:30.525 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:30.583 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:30.585 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:30.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:30.621 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:30.640 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:30 smithi163 ceph-mon[221067]: 10.13 scrub ok 2023-10-09T04:25:30.640 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:30 smithi163 ceph-mon[221067]: 10.3 scrub starts 2023-10-09T04:25:30.640 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:30 smithi163 ceph-mon[221067]: pgmap v1210: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 1580/139071 objects degraded (1.136%); 19102/139071 objects misplaced (13.735%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:30.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:30 smithi163 ceph-mon[213841]: 10.13 scrub ok 2023-10-09T04:25:30.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:30 smithi163 ceph-mon[213841]: 10.3 scrub starts 2023-10-09T04:25:30.641 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:30 smithi163 ceph-mon[213841]: pgmap v1210: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 1580/139071 objects degraded (1.136%); 19102/139071 objects misplaced (13.735%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:25:30.641 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:25:30] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2023-10-09T04:25:30.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:30.729 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:30.864 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:30.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:30.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:30.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:30.999 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:31.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:31.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:31.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:31.383 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:31.434 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:31.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:31.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:31.568 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:31 smithi163 ceph-mon[221067]: 10.3 scrub ok 2023-10-09T04:25:31.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:31 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:25:31.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:31 smithi163 ceph-mon[213841]: 10.3 scrub ok 2023-10-09T04:25:31.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:31 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:25:31.611 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:31.615 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:31 smithi181 ceph-mon[196979]: 10.3 scrub ok 2023-10-09T04:25:31.615 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:31 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:25:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:31.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:31.717 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:31.743 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:31.788 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:31.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:31.894 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:31.925 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:32.102 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:32.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:32.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:32.207 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:32.244 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:32.365 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:25:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:32 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3049192752' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:32 smithi163 ceph-mon[221067]: pgmap v1211: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 1580/136299 objects degraded (1.159%); 18731/136299 objects misplaced (13.743%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:25:32.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:32 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3049192752' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:32 smithi163 ceph-mon[213841]: pgmap v1211: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 1580/136299 objects degraded (1.159%); 18731/136299 objects misplaced (13.743%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:25:32.691 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:32.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:32 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3049192752' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:32.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:32 smithi181 ceph-mon[196979]: pgmap v1211: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 1580/136299 objects degraded (1.159%); 18731/136299 objects misplaced (13.743%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:25:32.797 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:32.874 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:33.269 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:33.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:33.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:33.365 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:33.376 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:33.406 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:33.497 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: 10.19 scrub starts 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: failed to encode map e458 with expected crc 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: failed to encode map e458 with expected crc 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: 10.17 deep-scrub starts 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/136299 objects degraded (1.159%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: 10.19 scrub starts 2023-10-09T04:25:33.498 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:25:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: failed to encode map e458 with expected crc 2023-10-09T04:25:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: failed to encode map e458 with expected crc 2023-10-09T04:25:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T04:25:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: 10.17 deep-scrub starts 2023-10-09T04:25:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:33 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/136299 objects degraded (1.159%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:33.754 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:33.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:33.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: 10.19 scrub starts 2023-10-09T04:25:33.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:25:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: failed to encode map e458 with expected crc 2023-10-09T04:25:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: failed to encode map e458 with expected crc 2023-10-09T04:25:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T04:25:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: 10.17 deep-scrub starts 2023-10-09T04:25:33.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:33 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/136299 objects degraded (1.159%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:33.804 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:33.827 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:33.846 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:33.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:33.866 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:33.886 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:33.889 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:33.915 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:33.939 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:33.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:33.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:33.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:33.967 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:33.969 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.969 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.969 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.969 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:25:33.970 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:25:34.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:34.065 INFO:teuthology.orchestra.run.smithi163.stdout:moved 0 / 387 (0%) 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:avg 48.375 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 5.02338 -> 5.02338 (expected baseline 6.50601) 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 41 -> 41 pgs (0.847545 -> 0.847545 * mean) 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.4 with 57 -> 57 pgs (1.17829 -> 1.17829 * mean) 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:oload 1000 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 0.05 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:25:34.066 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0217 2023-10-09T04:25:34.067 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.2174 2023-10-09T04:25:34.071 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:25:34.101 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:34.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:34.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:34.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:34.258 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:34.289 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:34.341 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:34.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:34.360 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3327728136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:34.360 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:34 smithi181 ceph-mon[196979]: pgmap v1213: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 1580/135447 objects degraded (1.167%); 18731/135447 objects misplaced (13.829%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:25:34.360 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:34 smithi181 ceph-mon[196979]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:34.360 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:34.360 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:34.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3327728136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:34.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[213841]: pgmap v1213: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 1580/135447 objects degraded (1.167%); 18731/135447 objects misplaced (13.829%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:25:34.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[213841]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:34.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:34.361 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:34.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:34.476 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:34.527 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:34.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:34.634 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:34.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3327728136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:34.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[221067]: pgmap v1213: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 1580/135447 objects degraded (1.167%); 18731/135447 objects misplaced (13.829%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:25:34.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[221067]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:34.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:34.660 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:34.662 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:35.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:25:35.329 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:35.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:35.433 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:35.461 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:35.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:35 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:25:35.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:35 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:25:35.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:35 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:25:35.867 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:35.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:35.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:35.997 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:36.052 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:36.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:36.117 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T04:25:36.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T04:25:36.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:36.190 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:36.243 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:36.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:36.250 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:36.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:36.397 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:36 smithi163 ceph-mon[221067]: pgmap v1214: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 1580/134544 objects degraded (1.174%); 18450/134544 objects misplaced (13.713%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:25:36.399 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:36 smithi163 ceph-mon[213841]: pgmap v1214: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 1580/134544 objects degraded (1.174%); 18450/134544 objects misplaced (13.713%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:25:36.402 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:36 smithi181 ceph-mon[196979]: pgmap v1214: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 1580/134544 objects degraded (1.174%); 18450/134544 objects misplaced (13.713%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:25:36.675 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:36.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:36.717 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:25:36.718 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:25:36.718 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:25:36.718 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:25:36.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:36.830 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:36.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:36.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:37.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:37.113 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:37.438 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:25:37.462 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:37.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:37.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:37.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:37.595 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:37.646 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:37.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:37.752 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:37.777 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/134544 objects degraded (1.174%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:37.777 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/134544 objects degraded (1.174%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:37.779 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:38.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/134544 objects degraded (1.174%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:38.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:38.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:38.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:38.193 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:38.223 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:38.224 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:25:38.405 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:38.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:38.510 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:38.559 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:38.741 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:38.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:38.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:38.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:38.962 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:39.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:38 smithi181 ceph-mon[196979]: pgmap v1215: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 1580/130659 objects degraded (1.209%); 18005/130659 objects misplaced (13.780%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:25:39.024 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:38 smithi163 ceph-mon[221067]: pgmap v1215: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 1580/130659 objects degraded (1.209%); 18005/130659 objects misplaced (13.780%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:25:39.024 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:38 smithi163 ceph-mon[213841]: pgmap v1215: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 1580/130659 objects degraded (1.209%); 18005/130659 objects misplaced (13.780%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:25:39.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:39.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:39.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:39.164 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:39.190 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:39.236 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:39.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:39.342 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:39.368 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:39.412 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:39.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:39.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:39.543 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:39.734 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:25:40.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1385256205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:25:40.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1385256205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:25:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1385256205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:25:40.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:40.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:40.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:40.234 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:40.285 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:40.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:40.395 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:40.424 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:40.717 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:40.717 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:25:40] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.43.0" 2023-10-09T04:25:40.978 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:40.979 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:40.979 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[221067]: failed to encode map e459 with expected crc 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1385256205' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[221067]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[221067]: failed to encode map e459 with expected crc 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[221067]: pgmap v1217: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 1580/129366 objects degraded (1.221%); 17724/129366 objects misplaced (13.701%); 792 KiB/s, 12 objects/s recovering 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[213841]: failed to encode map e459 with expected crc 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1385256205' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[213841]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T04:25:40.980 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[213841]: failed to encode map e459 with expected crc 2023-10-09T04:25:40.981 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:40 smithi163 ceph-mon[213841]: pgmap v1217: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 1580/129366 objects degraded (1.221%); 17724/129366 objects misplaced (13.701%); 792 KiB/s, 12 objects/s recovering 2023-10-09T04:25:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:40.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:40 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:40 smithi181 ceph-mon[196979]: failed to encode map e459 with expected crc 2023-10-09T04:25:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1385256205' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:25:41.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:40 smithi181 ceph-mon[196979]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T04:25:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:40 smithi181 ceph-mon[196979]: failed to encode map e459 with expected crc 2023-10-09T04:25:41.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:40 smithi181 ceph-mon[196979]: pgmap v1217: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 288 op/s; 1580/129366 objects degraded (1.221%); 17724/129366 objects misplaced (13.701%); 792 KiB/s, 12 objects/s recovering 2023-10-09T04:25:41.054 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:41.083 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:41.112 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:41.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:41.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:41.283 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:41.314 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:41.365 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:41.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:41.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:41.471 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:41.504 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:41.547 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:41.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:41.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:41.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:41.683 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:41.733 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:41.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:41.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:41.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:41.892 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:41.914 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:41.940 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:41.959 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:41.983 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:41.985 INFO:teuthology.orchestra.run.smithi163.stdout:moved 0 / 387 (0%) 2023-10-09T04:25:41.985 INFO:teuthology.orchestra.run.smithi163.stdout:avg 48.375 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 5.02338 -> 5.02338 (expected baseline 6.50601) 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 41 -> 41 pgs (0.847545 -> 0.847545 * mean) 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.4 with 57 -> 57 pgs (1.17829 -> 1.17829 * mean) 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:oload 1000 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 0.05 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0210 2023-10-09T04:25:41.986 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.2101 2023-10-09T04:25:41.990 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:25:42.007 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:42.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1149832737' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:42.028 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:42.052 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:42.075 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.077 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.078 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:25:42.101 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1149832737' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:42.101 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1149832737' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:42.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:25:42.955 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:42.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:42.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:43.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:43.086 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:43.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:43.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:43.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:42 smithi163 ceph-mon[221067]: pgmap v1218: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 1580/126621 objects degraded (1.248%); 17347/126621 objects misplaced (13.700%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:25:43.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:42 smithi163 ceph-mon[221067]: from='client.36414 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:43.198 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:42 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/129366 objects degraded (1.221%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:43.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:42 smithi163 ceph-mon[213841]: pgmap v1218: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 1580/126621 objects degraded (1.248%); 17347/126621 objects misplaced (13.700%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:25:43.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:42 smithi163 ceph-mon[213841]: from='client.36414 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:43.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:42 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/129366 objects degraded (1.221%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:43.247 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:43.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:42 smithi181 ceph-mon[196979]: pgmap v1218: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 1580/126621 objects degraded (1.248%); 17347/126621 objects misplaced (13.700%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:25:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:42 smithi181 ceph-mon[196979]: from='client.36414 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:42 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/129366 objects degraded (1.221%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:43.286 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:43.447 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:25:43.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T04:25:43.501 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:43.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:43.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:43.634 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:43.696 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:43.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:43.802 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:43.842 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:43.863 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:43.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:43.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:43.967 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:43.967 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:25:44.000 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:44.048 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:44.257 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:44.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:44.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:44.388 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:44.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:44.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:44.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:44.590 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:45.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:45.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:45.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:45.105 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:45.136 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:45.186 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:45.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:45.198 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:45.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:25:45.249 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:45 smithi163 ceph-mon[221067]: pgmap v1219: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 321 op/s; 1580/125796 objects degraded (1.256%); 17347/125796 objects misplaced (13.790%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:25:45.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:45 smithi163 ceph-mon[213841]: pgmap v1219: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 321 op/s; 1580/125796 objects degraded (1.256%); 17347/125796 objects misplaced (13.790%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:25:45.291 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:45.319 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:45 smithi181 ceph-mon[196979]: pgmap v1219: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 321 op/s; 1580/125796 objects degraded (1.256%); 17347/125796 objects misplaced (13.790%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:25:45.321 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:45.364 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:45.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:45.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:45.504 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:45.537 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:25:45.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:45.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:45.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:45.759 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:46.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:46.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:46.177 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:46.209 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:46.260 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:46.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:46.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:46.364 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:46.394 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:46.447 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:46.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:46.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:46.584 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:46.640 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:46.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:46.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:46.718 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:46.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:46.757 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:46.780 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:46.837 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:46.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:46.943 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:46.980 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:47.034 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:25:47.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:47 smithi181 ceph-mon[196979]: pgmap v1220: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 1580/124821 objects degraded (1.266%); 17045/124821 objects misplaced (13.656%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:25:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2926225463' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:47 smithi163 ceph-mon[221067]: pgmap v1220: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 1580/124821 objects degraded (1.266%); 17045/124821 objects misplaced (13.656%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:25:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2926225463' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:47.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:47 smithi163 ceph-mon[213841]: pgmap v1220: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 1580/124821 objects degraded (1.266%); 17045/124821 objects misplaced (13.656%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:25:47.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2926225463' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:47.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:moved 0 / 387 (0%) 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:avg 48.375 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 5.02338 -> 5.02338 (expected baseline 6.50601) 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 41 -> 41 pgs (0.847545 -> 0.847545 * mean) 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.4 with 57 -> 57 pgs (1.17829 -> 1.17829 * mean) 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:oload 1000 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 1 2023-10-09T04:25:47.416 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:25:47.417 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0206 2023-10-09T04:25:47.417 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.2059 2023-10-09T04:25:47.419 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:25:47.570 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:47.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:47.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:47.707 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:47.751 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:47.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:47.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:47.856 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:47.885 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:48.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:48.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:48.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[213841]: failed to encode map e460 with expected crc 2023-10-09T04:25:48.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[213841]: failed to encode map e460 with expected crc 2023-10-09T04:25:48.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[213841]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/124821 objects degraded (1.266%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[221067]: failed to encode map e460 with expected crc 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[221067]: failed to encode map e460 with expected crc 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[221067]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T04:25:48.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:48 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/124821 objects degraded (1.266%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:48.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:48.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:48.276 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:48 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:48 smithi181 ceph-mon[196979]: failed to encode map e460 with expected crc 2023-10-09T04:25:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:48 smithi181 ceph-mon[196979]: failed to encode map e460 with expected crc 2023-10-09T04:25:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:25:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:48 smithi181 ceph-mon[196979]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T04:25:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:48 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/124821 objects degraded (1.266%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:49.024 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:49 smithi163 ceph-mon[213841]: from='client.36429 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:49.024 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:49 smithi163 ceph-mon[213841]: pgmap v1222: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 1580/121170 objects degraded (1.304%); 16618/121170 objects misplaced (13.715%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:25:49.244 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:49.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:49.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:49 smithi163 ceph-mon[221067]: from='client.36429 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:49.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:49 smithi163 ceph-mon[221067]: pgmap v1222: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 1580/121170 objects degraded (1.304%); 16618/121170 objects misplaced (13.715%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:25:49.316 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T04:25:49.316 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-10-09T04:25:49.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:49.371 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:49 smithi181 ceph-mon[196979]: from='client.36429 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:49.371 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:49 smithi181 ceph-mon[196979]: pgmap v1222: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 1580/121170 objects degraded (1.304%); 16618/121170 objects misplaced (13.715%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T04:25:49.374 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:49.422 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:49.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:49.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:49.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:49.559 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:49.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:49.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:49.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:49.739 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:50.001 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:50.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:50.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:50.153 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:50.376 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:50.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:50.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:50.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:50.504 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:50.504 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:25:50.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:50.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:50 smithi163 ceph-mon[213841]: pgmap v1223: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 1580/120057 objects degraded (1.316%); 16351/120057 objects misplaced (13.619%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:25:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:50.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:50 smithi163 ceph-mon[221067]: pgmap v1223: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 1580/120057 objects degraded (1.316%); 16351/120057 objects misplaced (13.619%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:25:50.620 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:50.739 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:50.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:50.747 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:25:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:50 smithi181 ceph-mon[196979]: pgmap v1223: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 1580/120057 objects degraded (1.316%); 16351/120057 objects misplaced (13.619%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:25:50.770 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:50.793 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:50.820 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:50.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:50.847 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:50.871 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:50.874 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:50.878 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:25:50] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2023-10-09T04:25:50.894 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:50.915 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:50.919 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.919 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.919 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.919 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.919 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.919 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.920 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.920 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:25:50.923 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:50.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:50.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:51.030 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:51.060 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:51.520 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:25:51.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2398518002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2398518002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:51.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2398518002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:52.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:25:52.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:52 smithi181 ceph-mon[196979]: pgmap v1224: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 1580/116379 objects degraded (1.358%); 15835/116379 objects misplaced (13.606%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:25:52.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:52 smithi163 ceph-mon[221067]: pgmap v1224: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 1580/116379 objects degraded (1.358%); 15835/116379 objects misplaced (13.606%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:25:52.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:52 smithi163 ceph-mon[213841]: pgmap v1224: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 1580/116379 objects degraded (1.358%); 15835/116379 objects misplaced (13.606%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:25:52.610 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:25:52.671 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:52.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:52.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:52.820 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:52.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:52.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:52.996 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:53.045 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:53.139 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:53.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:53.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:53.265 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:53.298 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:53.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:53.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:53.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:53.426 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:53 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/116379 objects degraded (1.358%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:53.428 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:53 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/116379 objects degraded (1.358%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:53.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:53.484 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:53 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/116379 objects degraded (1.358%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:53.489 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:53.551 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:53.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:53.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:53.690 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:53.792 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:53.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:53.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:53.896 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:53.926 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:53.985 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:53.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:54.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:54.118 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:54.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:54.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:54.273 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:54.300 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:54.336 INFO:teuthology.orchestra.run.smithi163.stdout:moved 2 / 387 (0.516796%) 2023-10-09T04:25:54.336 INFO:teuthology.orchestra.run.smithi163.stdout:avg 48.375 2023-10-09T04:25:54.336 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 5.02338 -> 4.27017 (expected baseline 6.50601) 2023-10-09T04:25:54.336 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 41 -> 42 pgs (0.847545 -> 0.868217 * mean) 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.4 with 57 -> 55 pgs (1.17829 -> 1.13695 * mean) 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:oload 110 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 0.05 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0201 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.0221 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 weight 1.0000 -> 0.9500 2023-10-09T04:25:54.337 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 weight 1.0000 -> 0.9500 2023-10-09T04:25:54.367 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:54.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:54.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:54.473 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:54.500 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:54.556 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:54.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:54.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:54 smithi163 ceph-mon[221067]: from='client.36435 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'62260','4':'62260'}"}]: dispatch 2023-10-09T04:25:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:54 smithi163 ceph-mon[221067]: pgmap v1225: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 1580/115371 objects degraded (1.369%); 15835/115371 objects misplaced (13.725%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:25:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:54 smithi163 ceph-mon[213841]: from='client.36435 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'62260','4':'62260'}"}]: dispatch 2023-10-09T04:25:54.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:54 smithi163 ceph-mon[213841]: pgmap v1225: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 1580/115371 objects degraded (1.369%); 15835/115371 objects misplaced (13.725%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:25:54.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:25:54.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:54.700 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:54 smithi181 ceph-mon[196979]: from='client.36435 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:54.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'3':'62260','4':'62260'}"}]: dispatch 2023-10-09T04:25:54.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:54 smithi181 ceph-mon[196979]: pgmap v1225: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 1580/115371 objects degraded (1.369%); 15835/115371 objects misplaced (13.725%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:25:55.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:55.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:55.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:55.134 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:55.150 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:25:55.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T04:25:55.189 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:55.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:55.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:55.323 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:55.379 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:55.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:55.466 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:55.484 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:55.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:55 smithi181 ceph-mon[196979]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:55 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'62260','4':'62260'}"}]': finished 2023-10-09T04:25:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:55 smithi181 ceph-mon[196979]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T04:25:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:55 smithi181 ceph-mon[196979]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:55 smithi181 ceph-mon[196979]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:55 smithi181 ceph-mon[196979]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[213841]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'62260','4':'62260'}"}]': finished 2023-10-09T04:25:55.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[213841]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[213841]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[213841]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[213841]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[221067]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'62260','4':'62260'}"}]': finished 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[221067]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[221067]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[221067]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:55 smithi163 ceph-mon[221067]: failed to encode map e461 with expected crc 2023-10-09T04:25:55.531 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:55.627 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:55.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:55.647 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:25:55.732 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:55.765 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:55.866 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:55.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:55.972 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:56.006 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:56.090 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:56.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:56.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:25:56.226 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:56.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:25:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:56.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:25:56.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:56.396 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:25:56.415 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:25:56.415 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:25:56.415 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.415 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111327 10.d 10.1e3fa31d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:56.044093+0000", 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.37049103799999999, 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.37051325200000002, 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111327 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.416 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.044093+0000", 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.044093+0000", 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.417 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.044092+0000", 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.044097+0000", 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9910000000000002e-06 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.418 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.044110+0000", 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3118999999999999e-05 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.044117+0000", 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4479999999999999e-06 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.419 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111267 10.d 10:b8f4e800:::benchmark_data_smithi163_523108_object37251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:55.881038+0000", 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.53354585499999996, 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.53361134399999999, 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for degraded object", 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111267 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.420 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.881038+0000", 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.881038+0000", 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.421 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.881038+0000", 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.881041+0000", 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9079999999999999e-06 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.881050+0000", 2023-10-09T04:25:56.422 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.713e-06 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.881055+0000", 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5179999999999998e-06 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.404728+0000", 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.52367295300000005 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.423 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.412823+0000", 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080956550000000002 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111373 10.d 10.7732cfbd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:56.129727+0000", 2023-10-09T04:25:56.424 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.28485722000000002, 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.28496480200000002, 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111373 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.425 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.129727+0000", 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.129727+0000", 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.129726+0000", 2023-10-09T04:25:56.426 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.129731+0000", 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7010000000000004e-06 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.129742+0000", 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1539000000000001e-05 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.427 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.129749+0000", 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3239999999999999e-06 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111420 10.d 10.42925f1d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e463)", 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:56.401510+0000", 2023-10-09T04:25:56.428 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.013074532, 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013220851, 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111420 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.429 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.401510+0000", 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.401510+0000", 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.430 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.401508+0000", 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.401516+0000", 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9130000000000003e-06 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.431 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.401529+0000", 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3191e-05 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.401536+0000", 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3289999999999996e-06 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.432 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111304 10.d 10.b0871bd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:55.998901+0000", 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.41568306300000002, 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.41586144000000003, 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.433 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111304 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.998901+0000", 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.998901+0000", 2023-10-09T04:25:56.434 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.998899+0000", 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.998906+0000", 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1210000000000001e-06 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.435 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.998918+0000", 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1321999999999999e-05 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.998922+0000", 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5870000000000003e-06 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.436 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111275 10.d 10:b685a8d1:::benchmark_data_smithi163_523108_object37259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:55.914470+0000", 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.50011416799999997, 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.50032463400000005, 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.437 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111275 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.914470+0000", 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.914470+0000", 2023-10-09T04:25:56.438 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.914469+0000", 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.914475+0000", 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7400000000000001e-06 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.439 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.914505+0000", 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0645000000000002e-05 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.914510+0000", 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9470000000000002e-06 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.440 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.413116+0000", 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.49860588099999997 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111407 10.d 10.cea3cb8d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:56.254982+0000", 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.15960240000000001, 2023-10-09T04:25:56.441 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15984593499999999, 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111407 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.254982+0000", 2023-10-09T04:25:56.442 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.254982+0000", 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.254980+0000", 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:25:56.443 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.254987+0000", 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7089999999999996e-06 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.255000+0000", 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3118999999999999e-05 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.444 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.255007+0000", 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7950000000000002e-06 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111248 10.d 10:b3cc8fb6:::benchmark_data_smithi163_523108_object37232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:55.846185+0000", 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.56839947999999996, 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.56867488700000002, 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.445 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for degraded object", 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111248 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.846185+0000", 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.446 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.846185+0000", 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.846184+0000", 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.447 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.846189+0000", 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.925e-06 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.846201+0000", 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1652e-05 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.846206+0000", 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8160000000000001e-06 2023-10-09T04:25:56.448 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.396345+0000", 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.55013868499999996 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.404336+0000", 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0079915820000000005 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.449 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111384 10.d 10.3d6528bd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:56.147097+0000", 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.267486697, 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.267800865, 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.450 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111384 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.147097+0000", 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.451 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.147097+0000", 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.147096+0000", 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.147102+0000", 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0140000000000004e-06 2023-10-09T04:25:56.452 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.147117+0000", 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4966e-05 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.147123+0000", 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5480000000000001e-06 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.453 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111414 10.d 10.f282b94d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:56.263731+0000", 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.150852604, 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15119653, 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.454 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111414 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.263731+0000", 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.263731+0000", 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.455 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.263731+0000", 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.263735+0000", 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6349999999999997e-06 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.456 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.263748+0000", 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2592000000000001e-05 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:56.263754+0000", 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.117e-06 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.457 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:111283 10.d 10.ee59adbd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:25:55.960186+0000", 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.454398107, 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.454772177, 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "queued for pg", 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 111283 2023-10-09T04:25:56.458 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.960186+0000", 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.960186+0000", 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.459 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.960185+0000", 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.960190+0000", 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4190000000000001e-06 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.460 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.960204+0000", 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3674e-05 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:25:55.960209+0000", 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1950000000000002e-06 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:25:56.461 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:25:56.462 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 11 2023-10-09T04:25:56.462 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:25:56.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T04:25:56.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.463 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2747818147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:56.463 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:56.463 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:56 smithi181 ceph-mon[196979]: pgmap v1228: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 1580/114321 objects degraded (1.382%); 15509/114321 objects misplaced (13.566%); 928 KiB/s, 14 objects/s recovering 2023-10-09T04:25:56.463 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:56.483 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:56.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:56.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:56.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:56.624 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2747818147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:56.672 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[213841]: pgmap v1228: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 1580/114321 objects degraded (1.382%); 15509/114321 objects misplaced (13.566%); 928 KiB/s, 14 objects/s recovering 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: failed to encode map e462 with expected crc 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2747818147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:25:56.673 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:56 smithi163 ceph-mon[221067]: pgmap v1228: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 285 op/s; 1580/114321 objects degraded (1.382%); 15509/114321 objects misplaced (13.566%); 928 KiB/s, 14 objects/s recovering 2023-10-09T04:25:56.687 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:56.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:56.792 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:25:56.834 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:56.906 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:25:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:56.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:25:57.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:25:57.035 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:57.339 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:25:57.340 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:25:57.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:57.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:25:57.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:57.487 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:57.713 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[221067]: failed to encode map e463 with expected crc 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[221067]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[221067]: failed to encode map e463 with expected crc 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:57.715 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[213841]: failed to encode map e463 with expected crc 2023-10-09T04:25:57.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:25:57.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[213841]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T04:25:57.716 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:57 smithi163 ceph-mon[213841]: failed to encode map e463 with expected crc 2023-10-09T04:25:57.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:57.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:57 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:25:57.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:57 smithi181 ceph-mon[196979]: failed to encode map e463 with expected crc 2023-10-09T04:25:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:57 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:25:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:57 smithi181 ceph-mon[196979]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T04:25:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:57 smithi181 ceph-mon[196979]: failed to encode map e463 with expected crc 2023-10-09T04:25:57.817 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:25:57.819 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:25:57.844 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:58.273 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:25:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:58.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:25:58.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:58.432 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:58.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:25:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:58.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:25:58.546 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:25:58.565 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:25:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:58 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/114321 objects degraded (1.382%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:58.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:58 smithi163 ceph-mon[221067]: pgmap v1230: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1580/110271 objects degraded (1.433%); 15058/110271 objects misplaced (13.655%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:25:58.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:58 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/114321 objects degraded (1.382%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:58.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:58 smithi163 ceph-mon[213841]: pgmap v1230: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1580/110271 objects degraded (1.433%); 15058/110271 objects misplaced (13.655%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:25:58.584 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:25:58.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:25:58.603 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:25:58.621 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:58.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:25:58.681 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:25:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:58.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:25:58.713 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:25:58.746 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:25:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:58 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/114321 objects degraded (1.382%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:58.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:58 smithi181 ceph-mon[196979]: pgmap v1230: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 1580/110271 objects degraded (1.433%); 15058/110271 objects misplaced (13.655%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:25:58.773 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.775 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:25:58.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:25:58.824 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:59.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:25:59.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:59.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout:moved 23 / 387 (5.94315%) 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout:avg 48.375 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout:stddev 4.27017 -> 4.09077 (expected baseline 6.50601) 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout:min osd.2 with 42 -> 45 pgs (0.868217 -> 0.930233 * mean) 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout:max osd.4 with 55 -> 46 pgs (1.13695 -> 0.950904 * mean) 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:25:59.141 INFO:teuthology.orchestra.run.smithi163.stdout:oload 110 2023-10-09T04:25:59.142 INFO:teuthology.orchestra.run.smithi163.stdout:max_change 1 2023-10-09T04:25:59.142 INFO:teuthology.orchestra.run.smithi163.stdout:max_change_osds 4 2023-10-09T04:25:59.142 INFO:teuthology.orchestra.run.smithi163.stdout:average_utilization 0.0195 2023-10-09T04:25:59.142 INFO:teuthology.orchestra.run.smithi163.stdout:overload_utilization 0.0215 2023-10-09T04:25:59.142 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4 weight 0.9500 -> 0.7359 2023-10-09T04:25:59.142 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3 weight 0.9500 -> 0.8324 2023-10-09T04:25:59.143 INFO:teuthology.orchestra.run.smithi163.stderr:no change 2023-10-09T04:25:59.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:25:59.217 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:25:59 smithi163 conmon[200567]: 2023-10-09T04:25:59.139+0000 7f40871b1700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T04:25:59.221 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1321982821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:59.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:25:59 smithi181 ceph-mon[196979]: from='client.46388 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:59.476 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:25:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:25:59.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:25:59.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1321982821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:59.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:25:59 smithi163 ceph-mon[221067]: from='client.46388 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:59.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1321982821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:25:59.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:25:59 smithi163 ceph-mon[213841]: from='client.46388 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:25:59.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:25:59.619 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:25:59.650 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:25:59.650 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:00.036 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:26:00.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:00.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:00.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:00.225 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:00.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:00.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:00.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:00.412 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:00.445 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:00.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:00.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:00.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:00 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:26:00.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:00 smithi181 ceph-mon[196979]: pgmap v1231: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 1580/108807 objects degraded (1.452%); 14713/108807 objects misplaced (13.522%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:26:00.564 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:00.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:26:00.618 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:00.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:00 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:26:00.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:00 smithi163 ceph-mon[221067]: pgmap v1231: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 1580/108807 objects degraded (1.452%); 14713/108807 objects misplaced (13.522%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:26:00.638 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:00 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:26:00.638 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:00 smithi163 ceph-mon[213841]: pgmap v1231: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 1580/108807 objects degraded (1.452%); 14713/108807 objects misplaced (13.522%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:26:00.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:00.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:00.810 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:00.845 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:00.915 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:00.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:01.019 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:26:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:26:00] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2023-10-09T04:26:01.024 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:01.066 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:01.131 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:01.134 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:01.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:01.239 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:01.271 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:01.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:01.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:01.494 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:01.527 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:01.778 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:01.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:01.837 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:26:01.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:01.910 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:01.953 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:01.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:02.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:02.084 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:02.240 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:02.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:02.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:02.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:02.525 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:02.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:02.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:02.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:02.841 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:02.992 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:02.993 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:03.035 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:26:03.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[221067]: 10.19 scrub starts 2023-10-09T04:26:03.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[221067]: pgmap v1232: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 1580/105945 objects degraded (1.491%); 14385/105945 objects misplaced (13.578%); 955 KiB/s, 14 objects/s recovering 2023-10-09T04:26:03.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2278645237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:03.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/105945 objects degraded (1.491%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:03.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[213841]: 10.19 scrub starts 2023-10-09T04:26:03.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[213841]: pgmap v1232: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 1580/105945 objects degraded (1.491%); 14385/105945 objects misplaced (13.578%); 955 KiB/s, 14 objects/s recovering 2023-10-09T04:26:03.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2278645237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:03.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:03 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/105945 objects degraded (1.491%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:03.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:03.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:03.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:03.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:03.268 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:03.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:03 smithi181 ceph-mon[196979]: 10.19 scrub starts 2023-10-09T04:26:03.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:03 smithi181 ceph-mon[196979]: pgmap v1232: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 1580/105945 objects degraded (1.491%); 14385/105945 objects misplaced (13.578%); 955 KiB/s, 14 objects/s recovering 2023-10-09T04:26:03.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:03 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2278645237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:03.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:03 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/105945 objects degraded (1.491%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:03.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:03.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:03.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:03.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:03.685 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:03.846 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:26:03.856 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:03.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:03.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:04.003 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:04.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:04.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:04.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:04.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:26:04.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[221067]: failed to encode map e464 with expected crc 2023-10-09T04:26:04.193 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[221067]: failed to encode map e464 with expected crc 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2278645237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[221067]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[213841]: failed to encode map e464 with expected crc 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[213841]: failed to encode map e464 with expected crc 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2278645237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:04.194 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:04 smithi163 ceph-mon[213841]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T04:26:04.196 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:04.247 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:04.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:04.256 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:04.351 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:04.379 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:04 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:26:04.379 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:04 smithi181 ceph-mon[196979]: failed to encode map e464 with expected crc 2023-10-09T04:26:04.379 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:04 smithi181 ceph-mon[196979]: failed to encode map e464 with expected crc 2023-10-09T04:26:04.379 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2278645237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:04.380 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:04 smithi181 ceph-mon[196979]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T04:26:04.382 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:04.428 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:04.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:04.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:04.496 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:04.521 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:04.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:04.546 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:04.569 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:04.576 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:04.595 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:04.623 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:04.649 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:04.672 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.674 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:26:04.743 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:04.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:04.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:04.895 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:05.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:05.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:05.089 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 5, 7, 3, 6, 0, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:26:05.089 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:26:05.089 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 5, 7, 3, 6, 0, 4] 2023-10-09T04:26:05.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T04:26:05.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:05.161 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:05.246 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:05.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:05.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:05.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:05 smithi181 ceph-mon[196979]: pgmap v1234: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 1580/103584 objects degraded (1.525%); 14136/103584 objects misplaced (13.647%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:26:05.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2028759240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[221067]: pgmap v1234: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 1580/103584 objects degraded (1.525%); 14136/103584 objects misplaced (13.647%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:26:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2028759240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[213841]: pgmap v1234: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 1580/103584 objects degraded (1.525%); 14136/103584 objects misplaced (13.647%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T04:26:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2028759240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:05.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:05.378 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:05.548 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:05.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:05.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:05.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:05.666 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:05.706 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:05.761 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:05.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:05.865 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:05.894 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:05.947 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:05.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:05.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:05.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:26:06.052 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:06.090 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:06.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:06.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:06.152 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:26:06.152 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:26:06.152 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:26:06.152 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:26:06.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:06.280 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:06.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:06.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:06.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:06.478 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:06.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:06.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:06.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:06.656 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:06.894 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:06.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:07.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:07.018 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:07.048 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:07.083 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:26:07.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:07.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:07.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:07.241 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:07.271 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:07.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:07 smithi163 ceph-mon[221067]: 10.17 deep-scrub starts 2023-10-09T04:26:07.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:07 smithi163 ceph-mon[221067]: pgmap v1235: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 1580/102474 objects degraded (1.542%); 13792/102474 objects misplaced (13.459%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:26:07.312 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:07 smithi163 ceph-mon[213841]: 10.17 deep-scrub starts 2023-10-09T04:26:07.313 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:07 smithi163 ceph-mon[213841]: pgmap v1235: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 1580/102474 objects degraded (1.542%); 13792/102474 objects misplaced (13.459%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:26:07.415 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:07.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:07.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:07 smithi181 ceph-mon[196979]: 10.17 deep-scrub starts 2023-10-09T04:26:07.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:07 smithi181 ceph-mon[196979]: pgmap v1235: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 1580/102474 objects degraded (1.542%); 13792/102474 objects misplaced (13.459%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T04:26:07.523 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:07.551 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:26:07.560 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:07.585 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:07.609 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:07.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:07.715 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:07.752 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:08.182 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:08 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/102474 objects degraded (1.542%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:08.183 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/260273244' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T04:26:08.183 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:08 smithi181 ceph-mon[196979]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:08 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/102474 objects degraded (1.542%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/260273244' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:08 smithi163 ceph-mon[221067]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:07 smithi163 conmon[213817]: 2023-10-09T04:26:07.877+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e464 definitely_dead 0 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:08 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/102474 objects degraded (1.542%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/260273244' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T04:26:08.237 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:08 smithi163 ceph-mon[213841]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T04:26:08.487 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:08.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:08.591 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:08.605 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:08.653 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:08.705 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.2. 2023-10-09T04:26:09.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:09.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:09.047 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:09 smithi163 ceph-mon[213841]: pgmap v1236: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 1580/97947 objects degraded (1.613%); 13288/97947 objects misplaced (13.567%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:26:09.084 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:09 smithi181 ceph-mon[196979]: pgmap v1236: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 1580/97947 objects degraded (1.613%); 13288/97947 objects misplaced (13.567%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:26:09.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/260273244' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T04:26:09.085 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:09 smithi181 ceph-mon[196979]: osdmap e465: 8 total, 8 up, 7 in 2023-10-09T04:26:09.133 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:09.163 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:09.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:09.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:09.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:09.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/260273244' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T04:26:09.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:09 smithi163 ceph-mon[213841]: osdmap e465: 8 total, 8 up, 7 in 2023-10-09T04:26:09.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:09 smithi163 ceph-mon[221067]: pgmap v1236: 129 pgs: 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 1580/97947 objects degraded (1.613%); 13288/97947 objects misplaced (13.567%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:26:09.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/260273244' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T04:26:09.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:09 smithi163 ceph-mon[221067]: osdmap e465: 8 total, 8 up, 7 in 2023-10-09T04:26:09.341 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:26:09.341 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:26:09.341 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:26:09.341 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:26:09.342 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:26:09.342 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:09.695 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:26:09.716 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:09.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:09.822 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:09.861 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:10.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: failed to encode map e465 with expected crc 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: failed to encode map e465 with expected crc 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: 10.17 scrub starts 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3213303205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:10.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: osdmap e466: 8 total, 8 up, 7 in 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: 10.18 continuing backfill to osd.0 from (0'0,464'6164] 10:196ecca4:::benchmark_data_smithi163_523108_object41170:head to 464'6164 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: 10.8 continuing backfill to osd.0 from (0'0,464'6183] 10:11acbf38:::benchmark_data_smithi163_523108_object27981:head to 464'6183 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: 10.15 continuing backfill to osd.0 from (0'0,464'6324] MIN to 464'6324 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: 10.e continuing backfill to osd.4 from (388'400,464'7201] MIN to 464'7201 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[221067]: 10.5 continuing backfill to osd.0 from (0'0,464'6283] MIN to 464'6283 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: failed to encode map e465 with expected crc 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: failed to encode map e465 with expected crc 2023-10-09T04:26:10.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: 10.17 scrub starts 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3213303205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: osdmap e466: 8 total, 8 up, 7 in 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: 10.18 continuing backfill to osd.0 from (0'0,464'6164] 10:196ecca4:::benchmark_data_smithi163_523108_object41170:head to 464'6164 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: 10.8 continuing backfill to osd.0 from (0'0,464'6183] 10:11acbf38:::benchmark_data_smithi163_523108_object27981:head to 464'6183 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: 10.15 continuing backfill to osd.0 from (0'0,464'6324] MIN to 464'6324 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: 10.e continuing backfill to osd.4 from (388'400,464'7201] MIN to 464'7201 2023-10-09T04:26:10.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:10 smithi163 ceph-mon[213841]: 10.5 continuing backfill to osd.0 from (0'0,464'6283] MIN to 464'6283 2023-10-09T04:26:10.413 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:10.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:10.476 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:10.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: failed to encode map e465 with expected crc 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: failed to encode map e465 with expected crc 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: 10.17 scrub starts 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3213303205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: osdmap e466: 8 total, 8 up, 7 in 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: 10.18 continuing backfill to osd.0 from (0'0,464'6164] 10:196ecca4:::benchmark_data_smithi163_523108_object41170:head to 464'6164 2023-10-09T04:26:10.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: 10.8 continuing backfill to osd.0 from (0'0,464'6183] 10:11acbf38:::benchmark_data_smithi163_523108_object27981:head to 464'6183 2023-10-09T04:26:10.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: 10.15 continuing backfill to osd.0 from (0'0,464'6324] MIN to 464'6324 2023-10-09T04:26:10.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: 10.e continuing backfill to osd.4 from (388'400,464'7201] MIN to 464'7201 2023-10-09T04:26:10.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:10 smithi181 ceph-mon[196979]: 10.5 continuing backfill to osd.0 from (0'0,464'6283] MIN to 464'6283 2023-10-09T04:26:10.550 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:26:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:26:10] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.43.0" 2023-10-09T04:26:10.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:10.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:10.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:11.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:11.119 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: pgmap v1239: 129 pgs: 5 activating+remapped, 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 1580/96696 objects degraded (1.634%); 13027/96696 objects misplaced (13.472%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: osdmap e467: 8 total, 8 up, 7 in 2023-10-09T04:26:11.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:11 smithi181 ceph-mon[196979]: failed to encode map e467 with expected crc 2023-10-09T04:26:11.271 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:11.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:11.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:11.305 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: pgmap v1239: 129 pgs: 5 activating+remapped, 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 1580/96696 objects degraded (1.634%); 13027/96696 objects misplaced (13.472%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: osdmap e467: 8 total, 8 up, 7 in 2023-10-09T04:26:11.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[221067]: failed to encode map e467 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e466 with expected crc 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: pgmap v1239: 129 pgs: 5 activating+remapped, 1 active+recovering+undersized+remapped, 115 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 350 op/s; 1580/96696 objects degraded (1.634%); 13027/96696 objects misplaced (13.472%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T04:26:11.307 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: osdmap e467: 8 total, 8 up, 7 in 2023-10-09T04:26:11.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:11 smithi163 ceph-mon[213841]: failed to encode map e467 with expected crc 2023-10-09T04:26:11.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:11.410 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:11.468 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:11.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:11.544 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:11.562 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:11.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:11.580 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:11.599 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:11.600 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:11.644 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:11.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:11.706 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:11.737 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:11.748 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:11.760 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:11.787 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:11.788 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.788 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.788 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.788 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.788 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.788 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.789 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.789 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:26:11.797 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:12.034 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:12.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 464'7258 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/766627197' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: osdmap e468: 8 total, 8 up, 7 in 2023-10-09T04:26:12.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[213841]: failed to encode map e468 with expected crc 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 464'7258 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/766627197' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: osdmap e468: 8 total, 8 up, 7 in 2023-10-09T04:26:12.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:12 smithi163 ceph-mon[221067]: failed to encode map e468 with expected crc 2023-10-09T04:26:12.145 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:12.174 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:12.234 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:12.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:12.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: failed to encode map e467 with expected crc 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 464'7258 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/766627197' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: osdmap e468: 8 total, 8 up, 7 in 2023-10-09T04:26:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:12 smithi181 ceph-mon[196979]: failed to encode map e468 with expected crc 2023-10-09T04:26:12.338 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:12.384 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:12.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:12.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:12.618 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:12.618 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:12.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:12.685 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:12.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:12.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:12.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:26:12.850 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:12.921 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:12.991 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:12.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:13.099 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:13.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.134 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: pgmap v1242: 129 pgs: 2 active+recovery_wait+degraded, 5 activating+remapped, 1 active+recovering+undersized+remapped, 103 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 10878/95154 objects degraded (11.432%); 12879/95154 objects misplaced (13.535%); 64 KiB/s, 0 objects/s recovering 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/96696 objects degraded (1.634%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: osdmap e469: 8 total, 8 up, 7 in 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:13.135 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: pgmap v1242: 129 pgs: 2 active+recovery_wait+degraded, 5 activating+remapped, 1 active+recovering+undersized+remapped, 103 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 10878/95154 objects degraded (11.432%); 12879/95154 objects misplaced (13.535%); 64 KiB/s, 0 objects/s recovering 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/96696 objects degraded (1.634%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: osdmap e469: 8 total, 8 up, 7 in 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:13.136 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:13 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:13.137 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:13.187 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:13.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:13.265 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:13.295 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e468 with expected crc 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: pgmap v1242: 129 pgs: 2 active+recovery_wait+degraded, 5 activating+remapped, 1 active+recovering+undersized+remapped, 103 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 10878/95154 objects degraded (11.432%); 12879/95154 objects misplaced (13.535%); 64 KiB/s, 0 objects/s recovering 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/96696 objects degraded (1.634%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: osdmap e469: 8 total, 8 up, 7 in 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:13.324 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:13 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:13.326 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:13.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:13.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:13.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:26:13.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:13.572 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:13.647 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:13.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:13.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:13.795 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:13.852 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:13.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:13.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:13.965 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:13.988 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:14.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:14.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:14.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:14.163 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:14.250 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:14 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.250 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:14 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:14 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:14 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:14 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.251 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:14 smithi181 ceph-mon[196979]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[221067]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:14 smithi163 ceph-mon[213841]: failed to encode map e469 with expected crc 2023-10-09T04:26:14.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:14.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:14.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:14.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:14.510 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:14.640 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:26:14.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:14.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:14.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:14.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:14.802 INFO:tasks.thrashosds.thrasher:in_osds: [1, 5, 7, 3, 6, 0, 4] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:26:14.802 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:26:14.802 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T04:26:14.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T04:26:14.812 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:14.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:14.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:15.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:15.057 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:15.130 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:15.170 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:15.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:15.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:15.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:15.303 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:15 smithi163 ceph-mon[221067]: pgmap v1244: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 5 activating+remapped, 101 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 10878/93591 objects degraded (11.623%); 14612/93591 objects misplaced (15.613%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:15.303 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:15 smithi163 ceph-mon[213841]: pgmap v1244: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 5 activating+remapped, 101 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 10878/93591 objects degraded (11.623%); 14612/93591 objects misplaced (15.613%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:15.305 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:15.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:15.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:15.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:15.503 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:15 smithi181 ceph-mon[196979]: pgmap v1244: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 5 activating+remapped, 101 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 10878/93591 objects degraded (11.623%); 14612/93591 objects misplaced (15.613%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:15.627 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:15.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:15.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:15.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:15.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:15.829 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:15.829 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:16.080 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:26:16.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:16.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:16.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:16.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1069767756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:16.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1069767756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:16.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1069767756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:16.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:16.385 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:16.661 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:16.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:16.733 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:26:16.765 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:16.820 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:16.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:16.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:16.975 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:17.009 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:17.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[213841]: pgmap v1245: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 101 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 10878/92589 objects degraded (11.749%); 14300/92589 objects misplaced (15.445%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:26:17.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:17.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[213841]: failed to encode map e470 with expected crc 2023-10-09T04:26:17.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1069767756' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:17.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[213841]: osdmap e470: 8 total, 8 up, 7 in 2023-10-09T04:26:17.080 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[221067]: pgmap v1245: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 101 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 10878/92589 objects degraded (11.749%); 14300/92589 objects misplaced (15.445%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:26:17.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:17.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[221067]: failed to encode map e470 with expected crc 2023-10-09T04:26:17.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1069767756' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:17.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:17 smithi163 ceph-mon[221067]: osdmap e470: 8 total, 8 up, 7 in 2023-10-09T04:26:17.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:17.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:17.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:17.244 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:17.251 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:17.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:17.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:17.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:17.464 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:17 smithi181 ceph-mon[196979]: pgmap v1245: 129 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 101 active+clean, 6 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 10878/92589 objects degraded (11.749%); 14300/92589 objects misplaced (15.445%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T04:26:17.464 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:17 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:17.464 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:17 smithi181 ceph-mon[196979]: failed to encode map e470 with expected crc 2023-10-09T04:26:17.464 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1069767756' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:17.464 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:17 smithi181 ceph-mon[196979]: osdmap e470: 8 total, 8 up, 7 in 2023-10-09T04:26:17.470 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:17.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:17.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:17.645 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:17.680 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:17.924 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:17.924 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:17.930 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:17.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:17.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:17.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:17.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:17.991 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:18.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:18.054 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:18.057 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:18.088 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:18.094 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.094 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.094 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.095 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.095 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.095 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.095 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.095 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:26:18.109 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:18.154 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:18.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:18.242 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:18 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 10878/92589 objects degraded (11.749%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:18.242 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/366746199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:18.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:18 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 10878/92589 objects degraded (11.749%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:18.242 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/366746199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:18.259 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:18.284 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:18.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:18.338 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:18 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 10878/92589 objects degraded (11.749%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:18.338 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/366746199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:18.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:18.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:18.482 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:18.665 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:18.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:18.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:18.772 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:18.808 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:18.922 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:18.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:19.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:19.066 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:19.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:26:19.138 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:19.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:19.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:19.262 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:26:19.263 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:26:19.263 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:26:19.263 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:26:19.263 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:26:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:19 smithi181 ceph-mon[196979]: pgmap v1247: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.2 KiB/s rd, 5.9 KiB/s wr, 391 op/s; 10878/87810 objects degraded (12.388%); 13627/87810 objects misplaced (15.519%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:26:19.287 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:19.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:19 smithi163 ceph-mon[213841]: pgmap v1247: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.2 KiB/s rd, 5.9 KiB/s wr, 391 op/s; 10878/87810 objects degraded (12.388%); 13627/87810 objects misplaced (15.519%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:26:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:19 smithi163 ceph-mon[221067]: pgmap v1247: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.2 KiB/s rd, 5.9 KiB/s wr, 391 op/s; 10878/87810 objects degraded (12.388%); 13627/87810 objects misplaced (15.519%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:26:19.360 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:19.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:19.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:19.507 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:19.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:19.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:19.566 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:19.657 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:19.681 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:19 smithi163 conmon[213817]: 2023-10-09T04:26:19.461+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e470 definitely_dead 0 2023-10-09T04:26:19.684 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:19.836 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:19.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:19.941 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:19.981 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:20.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:20.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:20.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:26:20.310 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:20.342 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.2. 2023-10-09T04:26:20.382 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:20.427 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:20.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/519044578' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:26:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:26:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:20 smithi181 ceph-mon[196979]: pgmap v1248: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 374 op/s; 10878/86205 objects degraded (12.619%); 13244/86205 objects misplaced (15.363%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:20.535 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:20.566 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:20.581 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/519044578' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[213841]: pgmap v1248: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 374 op/s; 10878/86205 objects degraded (12.619%); 13244/86205 objects misplaced (15.363%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/519044578' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:26:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:20 smithi163 ceph-mon[221067]: pgmap v1248: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 12 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1.7 KiB/s rd, 4.6 KiB/s wr, 374 op/s; 10878/86205 objects degraded (12.619%); 13244/86205 objects misplaced (15.363%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:20.623 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:20.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:20.674 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:20.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:20.764 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:20.842 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:26:20.928 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:20.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:20.958 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:26:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:26:20] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-10-09T04:26:21.037 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:21.064 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:21.267 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:21.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:21.371 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:21.465 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:21.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:21.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:21.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:21 smithi181 ceph-mon[196979]: failed to encode map e471 with expected crc 2023-10-09T04:26:21.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:21 smithi181 ceph-mon[196979]: failed to encode map e471 with expected crc 2023-10-09T04:26:21.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:21 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T04:26:21.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:21 smithi181 ceph-mon[196979]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T04:26:21.572 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:21 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[221067]: failed to encode map e471 with expected crc 2023-10-09T04:26:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[221067]: failed to encode map e471 with expected crc 2023-10-09T04:26:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T04:26:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[221067]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T04:26:21.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:21.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[213841]: failed to encode map e471 with expected crc 2023-10-09T04:26:21.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[213841]: failed to encode map e471 with expected crc 2023-10-09T04:26:21.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T04:26:21.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[213841]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T04:26:21.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:21 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:21.670 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:21.720 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:21.863 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T04:26:21.917 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:21.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:21.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:22.021 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:22.092 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:22.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:22.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:22.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:22.249 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:22.347 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:22.403 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:22.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:22.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T04:26:22.596 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: 10.18 continuing backfill to osd.0 from (0'0,470'6303] 10:196ecca4:::benchmark_data_smithi163_523108_object41170:head to 470'6303 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: 10.e continuing backfill to osd.4 from (388'400,470'7472] MIN to 470'7472 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: 10.8 continuing backfill to osd.0 from (0'0,470'6318] 10:11acbf38:::benchmark_data_smithi163_523108_object27981:head to 470'6318 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: 10.15 continuing backfill to osd.0 from (0'0,470'6438] MIN to 470'6438 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: 10.5 continuing backfill to osd.0 from (0'0,470'6437] MIN to 470'6437 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.597 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[221067]: pgmap v1251: 129 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 11 peering, 1 active+recovering+undersized+remapped, 99 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.2 KiB/s wr, 495 op/s; 1580/83697 objects degraded (1.888%); 12588/83697 objects misplaced (15.040%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.598 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: 10.18 continuing backfill to osd.0 from (0'0,470'6303] 10:196ecca4:::benchmark_data_smithi163_523108_object41170:head to 470'6303 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: 10.e continuing backfill to osd.4 from (388'400,470'7472] MIN to 470'7472 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: 10.8 continuing backfill to osd.0 from (0'0,470'6318] 10:11acbf38:::benchmark_data_smithi163_523108_object27981:head to 470'6318 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: 10.15 continuing backfill to osd.0 from (0'0,470'6438] MIN to 470'6438 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: 10.5 continuing backfill to osd.0 from (0'0,470'6437] MIN to 470'6437 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.599 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:22 smithi163 ceph-mon[213841]: pgmap v1251: 129 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 11 peering, 1 active+recovering+undersized+remapped, 99 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.2 KiB/s wr, 495 op/s; 1580/83697 objects degraded (1.888%); 12588/83697 objects misplaced (15.040%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:26:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:22.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:22.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: 10.18 continuing backfill to osd.0 from (0'0,470'6303] 10:196ecca4:::benchmark_data_smithi163_523108_object41170:head to 470'6303 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.748 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: 10.e continuing backfill to osd.4 from (388'400,470'7472] MIN to 470'7472 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: 10.8 continuing backfill to osd.0 from (0'0,470'6318] 10:11acbf38:::benchmark_data_smithi163_523108_object27981:head to 470'6318 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: 10.15 continuing backfill to osd.0 from (0'0,470'6438] MIN to 470'6438 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: 10.5 continuing backfill to osd.0 from (0'0,470'6437] MIN to 470'6437 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: failed to encode map e472 with expected crc 2023-10-09T04:26:22.749 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:22 smithi181 ceph-mon[196979]: pgmap v1251: 129 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 11 peering, 1 active+recovering+undersized+remapped, 99 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.2 KiB/s wr, 495 op/s; 1580/83697 objects degraded (1.888%); 12588/83697 objects misplaced (15.040%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:26:22.769 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:22.918 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:22.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:23.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:23.075 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69232 10.e 10:75e66242:::benchmark_data_smithi163_523108_object69231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.591183+0000", 2023-10-09T04:26:23.207 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.60996852100001, 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0726746409999999, 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69232 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591183+0000", 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591183+0000", 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591181+0000", 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591240+0000", 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.872e-05 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591257+0000", 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6988999999999999e-05 2023-10-09T04:26:23.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591264+0000", 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5089999999999999e-06 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.645952+0000", 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0546883129999998 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.646113+0000", 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016113 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.646349+0000", 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000236264 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.650803+0000", 2023-10-09T04:26:23.212 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044541820000000001 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.650932+0000", 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012804900000000001 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.662657+0000", 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011725549 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.662702+0000", 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4826000000000001e-05 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663858+0000", 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001155643 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69248 10.e 10:75e713a4:::benchmark_data_smithi163_523108_object69247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.836531+0000", 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.364620651, 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8380690340000001, 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69248 2023-10-09T04:26:23.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836531+0000", 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836531+0000", 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836529+0000", 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836574+0000", 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4994999999999999e-05 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.217 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836593+0000", 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9465000000000001e-05 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836602+0000", 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3480000000000004e-06 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.648625+0000", 2023-10-09T04:26:23.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.812023709 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.648776+0000", 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015048399999999999 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.648960+0000", 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000184227 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656320+0000", 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007359516 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656337+0000", 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6969999999999998e-05 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.672446+0000", 2023-10-09T04:26:23.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016109893 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.672524+0000", 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7594000000000003e-05 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674600+0000", 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020758339999999999 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.221 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69252 10.e 10:7c6b28f5:::benchmark_data_smithi163_523108_object69251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.841254+0000", 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.359897921, 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8353127420000002, 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.222 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69252 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841254+0000", 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841254+0000", 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841253+0000", 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841321+0000", 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7558000000000006e-05 2023-10-09T04:26:23.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841330+0000", 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7389999999999994e-06 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841342+0000", 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1891e-05 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.650979+0000", 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8096368279999999 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.651091+0000", 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111653 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.651205+0000", 2023-10-09T04:26:23.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114021 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656348+0000", 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051431910000000001 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656408+0000", 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0182000000000001e-05 2023-10-09T04:26:23.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674675+0000", 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018266871 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674726+0000", 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0769999999999997e-05 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.676566+0000", 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00184059 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69257 10.e 10:79c28b6b:::benchmark_data_smithi163_523108_object69256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.872079+0000", 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.32907281199999, 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.809892643, 2023-10-09T04:26:23.229 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69257 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872079+0000", 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872079+0000", 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872077+0000", 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872133+0000", 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5439000000000002e-05 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872148+0000", 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5488000000000001e-05 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872155+0000", 2023-10-09T04:26:23.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8549999999999996e-06 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.652867+0000", 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7807122660000001 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.652971+0000", 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010373400000000001 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.653086+0000", 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115046 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656440+0000", 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033538729999999998 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656448+0000", 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8509999999999999e-06 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679138+0000", 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022690393999999999 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679211+0000", 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2861999999999995e-05 2023-10-09T04:26:23.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.681971+0000", 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027603689999999999 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69264 10.e 10:74615863:::benchmark_data_smithi163_523108_object69263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.906946+0000", 2023-10-09T04:26:23.236 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.294205879, 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7769406810000001, 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69264 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.906946+0000", 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.906946+0000", 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.906945+0000", 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.907005+0000", 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9942000000000003e-05 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.907014+0000", 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6739999999999992e-06 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.907018+0000", 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9060000000000004e-06 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.654647+0000", 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7476289889999999 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.654799+0000", 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015171999999999999 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.654916+0000", 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117608 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660310+0000", 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053939180000000001 2023-10-09T04:26:23.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660361+0000", 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0278999999999998e-05 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.682017+0000", 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021656412999999999 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.682060+0000", 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3090000000000002e-05 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.683886+0000", 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018261359999999999 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69282 10.e 10:7bf871c6:::benchmark_data_smithi163_523108_object69281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.015311+0000", 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.18584027399999, 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6712104050000001, 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69282 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.244 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015311+0000", 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015311+0000", 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015310+0000", 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015365+0000", 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5050999999999999e-05 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015379+0000", 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4032000000000001e-05 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015385+0000", 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0580000000000003e-06 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656478+0000", 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6410930989999999 2023-10-09T04:26:23.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656596+0000", 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118272 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656740+0000", 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000144048 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660410+0000", 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036698579999999998 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660419+0000", 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1630000000000002e-06 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.684011+0000", 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023592014000000001 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.684054+0000", 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3112000000000003e-05 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686522+0000", 2023-10-09T04:26:23.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00246714 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69285 10.e 10:7ce41a17:::benchmark_data_smithi163_523108_object69284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.071025+0000", 2023-10-09T04:26:23.251 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.130126023, 2023-10-09T04:26:23.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6243140670000003, 2023-10-09T04:26:23.252 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.252 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.252 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69285 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071025+0000", 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071025+0000", 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071024+0000", 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071048+0000", 2023-10-09T04:26:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3989e-05 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071062+0000", 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4272e-05 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071068+0000", 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7080000000000002e-06 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.658261+0000", 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.587193251 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.658420+0000", 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000158627 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.658622+0000", 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202456 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663871+0000", 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052484669999999997 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663911+0000", 2023-10-09T04:26:23.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0809000000000002e-05 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.693365+0000", 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029453842000000001 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.693406+0000", 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0679999999999997e-05 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.695339+0000", 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019334759999999999 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69287 10.e 10:7fcc9332:::benchmark_data_smithi163_523108_object69286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.083976+0000", 2023-10-09T04:26:23.259 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.11717547000001, 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6132521139999998, 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69287 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083976+0000", 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083976+0000", 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083975+0000", 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083998+0000", 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3048e-05 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.084012+0000", 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3766e-05 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.084017+0000", 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3340000000000001e-06 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663942+0000", 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.579925121 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.664062+0000", 2023-10-09T04:26:23.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011936500000000001 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.664171+0000", 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109647 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674622+0000", 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010451026 2023-10-09T04:26:23.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679015+0000", 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043928140000000001 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.695384+0000", 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016368627 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.695421+0000", 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7499999999999997e-05 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697228+0000", 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018069659999999999 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69303 10.e 10:75ecb351:::benchmark_data_smithi163_523108_object69302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.130126+0000", 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.07102535300001, 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5815739410000003, 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69303 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.267 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130126+0000", 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130126+0000", 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.268 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130125+0000", 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130177+0000", 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1683e-05 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130189+0000", 2023-10-09T04:26:23.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2656e-05 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130195+0000", 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6319999999999996e-06 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.666273+0000", 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.536077922 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.270 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.666380+0000", 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.666512+0000", 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000131939 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.271 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674633+0000", 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081214129999999992 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679074+0000", 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044414240000000002 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.709593+0000", 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030518841000000001 2023-10-09T04:26:23.272 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.709640+0000", 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7169000000000002e-05 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.711700+0000", 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020595779999999998 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.273 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69310 10.e 10:7b904be6:::benchmark_data_smithi163_523108_object69309:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.185883+0000", 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 174.01526875799999, 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5277444510000002, 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.274 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69310 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185883+0000", 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.275 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185883+0000", 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185881+0000", 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185908+0000", 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7138999999999999e-05 2023-10-09T04:26:23.276 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185924+0000", 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5197999999999999e-05 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185932+0000", 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9859999999999994e-06 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.277 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.668476+0000", 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.482544866 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.668602+0000", 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125854 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.278 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.668778+0000", 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000175584 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.678946+0000", 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010168513000000001 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.683948+0000", 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050016749999999997 2023-10-09T04:26:23.279 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.711780+0000", 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027831794 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.711844+0000", 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4277999999999997e-05 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.280 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.713627+0000", 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017830280000000001 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69329 10.e 10:703d185a:::benchmark_data_smithi163_523108_object69328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.291333+0000", 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 173.90981799900001, 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4245665820000002, 2023-10-09T04:26:23.281 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69329 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291333+0000", 2023-10-09T04:26:23.282 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291333+0000", 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291332+0000", 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.283 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291376+0000", 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.392e-05 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291387+0000", 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0954999999999999e-05 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.284 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291392+0000", 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0150000000000003e-06 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.670409+0000", 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3790162119999998 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.670574+0000", 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016513000000000001 2023-10-09T04:26:23.285 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.670688+0000", 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011466700000000001 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.678958+0000", 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082692180000000001 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.286 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.683974+0000", 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005016408 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.713704+0000", 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029730143000000001 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.713777+0000", 2023-10-09T04:26:23.287 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2943999999999998e-05 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.715900+0000", 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021229460000000001 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69426 10.e 10:7098e19e:::benchmark_data_smithi163_523108_object69425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.288 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.928499+0000", 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 173.27265289100001, 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7898835379999998, 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69426 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.289 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928499+0000", 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928499+0000", 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928497+0000", 2023-10-09T04:26:23.290 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928551+0000", 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4336000000000003e-05 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928567+0000", 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5727999999999999e-05 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.291 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928574+0000", 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9240000000000001e-06 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686587+0000", 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7580135079999999 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.292 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686760+0000", 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000172788 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686949+0000", 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000188333 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697250+0000", 2023-10-09T04:26:23.293 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010301273 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697305+0000", 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5287000000000001e-05 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.715978+0000", 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018672993999999998 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.294 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.716045+0000", 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7008999999999995e-05 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.718382+0000", 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023370040000000002 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.295 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69489 10.e 10:7a3246dd:::benchmark_data_smithi163_523108_object69488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:30.498807+0000", 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 172.70234469499999, 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.224276691, 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.297 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69489 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498807+0000", 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.298 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498807+0000", 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498805+0000", 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498863+0000", 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7686000000000003e-05 2023-10-09T04:26:23.299 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498876+0000", 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3013999999999999e-05 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498882+0000", 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8100000000000003e-06 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.300 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.689318+0000", 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.190435871 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.689466+0000", 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000148209 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:26:23.301 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.689604+0000", 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138465 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697261+0000", 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007656688 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697367+0000", 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010572400000000001 2023-10-09T04:26:23.302 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.720882+0000", 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023515794999999999 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.720954+0000", 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1761999999999995e-05 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.303 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.723083+0000", 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021292540000000001 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116398 10.f 10:f6de70b0:::benchmark_data_smithi163_523108_object42382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:08.726639+0000", 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 14.474512484, 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1461028689999999, 2023-10-09T04:26:23.304 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116398 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726639+0000", 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.305 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726639+0000", 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726638+0000", 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.306 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726643+0000", 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8149999999999996e-06 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726656+0000", 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3525e-05 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.307 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726661+0000", 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0749999999999997e-06 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726708+0000", 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6900000000000002e-05 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726715+0000", 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4400000000000002e-06 2023-10-09T04:26:23.308 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.739515+0000", 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0128005419999999 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.739527+0000", 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1764e-05 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.309 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.840092+0000", 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.100565035 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.840229+0000", 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000136718 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.840762+0000", 2023-10-09T04:26:23.310 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00053350200000000004 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865462+0000", 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024699523000000001 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865510+0000", 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7876e-05 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.311 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865616+0000", 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106165 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872563+0000", 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069471330000000003 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872623+0000", 2023-10-09T04:26:23.312 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0189000000000003e-05 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872742+0000", 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118701 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.313 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116433 10.f 10:f66d42fa:::benchmark_data_smithi163_523108_object42417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.800654+0000", 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.40049716, 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.082506435, 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116433 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.314 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800654+0000", 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800654+0000", 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.315 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800653+0000", 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800659+0000", 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0159999999999997e-06 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800672+0000", 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3023000000000001e-05 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.316 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800678+0000", 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9689999999999999e-06 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800729+0000", 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1001999999999997e-05 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.317 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800738+0000", 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.816e-06 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.847848+0000", 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0471098089999999 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.848001+0000", 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152948 2023-10-09T04:26:23.318 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.848139+0000", 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013824200000000001 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865546+0000", 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017406883000000001 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.319 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865579+0000", 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3408999999999998e-05 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872754+0000", 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0071743529999999996 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.882996+0000", 2023-10-09T04:26:23.320 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010242816 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883047+0000", 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0534000000000001e-05 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883161+0000", 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113752 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.321 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116446 10.f 10:fd542c17:::benchmark_data_smithi163_523108_object42430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.830509+0000", 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.370642423, 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.053200197, 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.322 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116446 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830509+0000", 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830509+0000", 2023-10-09T04:26:23.323 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830507+0000", 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830515+0000", 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9189999999999999e-06 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.324 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830529+0000", 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3729e-05 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830536+0000", 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3479999999999998e-06 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.325 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.832864+0000", 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023279910000000002 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.832876+0000", 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1646e-05 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.851302+0000", 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.018425677 2023-10-09T04:26:23.326 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.851473+0000", 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017120500000000001 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.851653+0000", 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000179727 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.327 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865654+0000", 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014001632 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865698+0000", 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3856999999999997e-05 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872760+0000", 2023-10-09T04:26:23.328 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0070616699999999999 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883560+0000", 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010800362000000001 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883605+0000", 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4913999999999997e-05 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.329 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883709+0000", 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010422 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116453 10.f 10:fc5b6f4e:::benchmark_data_smithi163_523108_object42437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.840683+0000", 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.360468502, 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0432569810000001, 2023-10-09T04:26:23.330 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116453 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840683+0000", 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.331 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840683+0000", 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840682+0000", 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.332 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840688+0000", 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7629999999999997e-06 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840700+0000", 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2116999999999999e-05 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840705+0000", 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9799999999999998e-06 2023-10-09T04:26:23.333 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840750+0000", 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5630000000000002e-05 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840759+0000", 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5629999999999992e-06 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.334 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.854741+0000", 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0139816880000001 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.854875+0000", 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013423499999999999 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.855026+0000", 2023-10-09T04:26:23.335 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015111899999999999 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867814+0000", 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012787587 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867914+0000", 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9958999999999996e-05 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.336 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872766+0000", 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048521149999999997 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883765+0000", 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010999581 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883820+0000", 2023-10-09T04:26:23.337 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5129e-05 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883940+0000", 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119577 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116493 10.f 10:fed8895e:::benchmark_data_smithi163_523108_object42477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:26:23.338 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:10.132543+0000", 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.068608714, 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75157952400000005, 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116493 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.339 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132543+0000", 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132543+0000", 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132541+0000", 2023-10-09T04:26:23.340 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132547+0000", 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9499999999999998e-06 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132559+0000", 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2272e-05 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.341 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132565+0000", 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3859999999999998e-06 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132615+0000", 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0396999999999999e-05 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.342 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132622+0000", 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0040000000000002e-06 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.858203+0000", 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.72558060099999999 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.858358+0000", 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015483900000000001 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.343 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.858531+0000", 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000173752 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867848+0000", 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093166810000000003 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.344 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867945+0000", 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6560999999999999e-05 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872772+0000", 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048271959999999997 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883980+0000", 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011207943999999999 2023-10-09T04:26:23.345 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884025+0000", 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5442000000000003e-05 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884122+0000", 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6967999999999997e-05 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.346 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116495 10.f 10:f09f63e6:::benchmark_data_smithi163_523108_object42479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:10.138865+0000", 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 13.062286813, 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.74547842600000003, 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.347 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116495 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138865+0000", 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138865+0000", 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.348 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138864+0000", 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138869+0000", 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8509999999999998e-06 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.349 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138878+0000", 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4409999999999999e-06 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138883+0000", 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.135e-06 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138931+0000", 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7457000000000003e-05 2023-10-09T04:26:23.350 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138938+0000", 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.622e-06 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.861740+0000", 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.72280164000000002 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.351 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.861906+0000", 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016563600000000001 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.862090+0000", 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018444199999999999 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867881+0000", 2023-10-09T04:26:23.352 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057914000000000004 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872553+0000", 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046718059999999997 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872778+0000", 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000224937 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.353 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884175+0000", 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011396509000000001 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884224+0000", 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9230000000000001e-05 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.354 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884343+0000", 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119158 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:120724 10.f 10:fc37408d:::benchmark_data_smithi163_523108_object46708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:20.337568+0000", 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 2.8635834039999999, 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.067171087, 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:23.355 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120724 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337568+0000", 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.356 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337568+0000", 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337567+0000", 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:23.357 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337574+0000", 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1060000000000002e-06 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337589+0000", 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5855999999999999e-05 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337597+0000", 2023-10-09T04:26:23.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1369999999999996e-06 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.338389+0000", 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00079237399999999999 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.338572+0000", 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000182972 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.359 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.338730+0000", 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000158066 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.341931+0000", 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032007619999999998 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:23.360 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.342216+0000", 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000285586 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.355426+0000", 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013209449999999999 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.355436+0000", 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0579e-05 2023-10-09T04:26:23.361 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.404678+0000", 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.049242062 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.404739+0000", 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0647999999999999e-05 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.362 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.363 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:23.363 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:23.363 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:26:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:23.364 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:23.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:23.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:23.496 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.496 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/83697 objects degraded (1.888%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2560078206' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:26:23.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:26:23.501 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:23.671 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:26:23.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T04:26:23.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.691 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/83697 objects degraded (1.888%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2560078206' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:26:23.692 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e473 with expected crc 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/83697 objects degraded (1.888%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.693 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: failed to encode map e474 with expected crc 2023-10-09T04:26:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2560078206' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:26:23.694 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:26:24.325 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:24.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:24.427 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:24.439 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:26:24.454 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:24.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:24.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.771 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.772 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:24 smithi181 ceph-mon[196979]: pgmap v1255: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 5 remapped+peering, 11 peering, 99 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 1580/81603 objects degraded (1.936%); 10425/81603 objects misplaced (12.775%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T04:26:24.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.802 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[221067]: pgmap v1255: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 5 remapped+peering, 11 peering, 99 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 1580/81603 objects degraded (1.936%); 10425/81603 objects misplaced (12.775%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.803 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: failed to encode map e475 with expected crc 2023-10-09T04:26:24.804 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:24 smithi163 ceph-mon[213841]: pgmap v1255: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 5 remapped+peering, 11 peering, 99 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 1580/81603 objects degraded (1.936%); 10425/81603 objects misplaced (12.775%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T04:26:24.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:24.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:24.906 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:24.966 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:25.059 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:25.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:25.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:25.184 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:25.219 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:25.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:25.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:25.416 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:25.466 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:25.513 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:25.541 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:25.550 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:25.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:25.573 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:25.604 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:25.635 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:25.656 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:25.665 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:25.697 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:25.703 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:25.735 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:25.748 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.748 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.749 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:26:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: 1.1c scrub starts 2023-10-09T04:26:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T04:26:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/505114960' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:25.784 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:25.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:25.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: 1.1c scrub starts 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/505114960' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: 1.1c scrub starts 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: failed to encode map e476 with expected crc 2023-10-09T04:26:25.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/505114960' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:25.891 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:26.003 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:26.529 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:26.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:26.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:26.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:26.683 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:26.864 INFO:tasks.thrashosds.thrasher:in_osds: [1, 5, 7, 3, 6, 0, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:26:26.864 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:26:26.865 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:26.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:26.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:26 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:26:26.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:26 smithi181 ceph-mon[196979]: 1.14 scrub starts 2023-10-09T04:26:26.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:26 smithi181 ceph-mon[196979]: pgmap v1257: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 5 remapped+peering, 10 peering, 100 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 252 op/s; 1580/80664 objects degraded (1.959%); 10130/80664 objects misplaced (12.558%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:26:26.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:26:26.973 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:26.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:26 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:26:26.991 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:26 smithi163 ceph-mon[213841]: 1.14 scrub starts 2023-10-09T04:26:26.992 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:26 smithi163 ceph-mon[213841]: pgmap v1257: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 5 remapped+peering, 10 peering, 100 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 252 op/s; 1580/80664 objects degraded (1.959%); 10130/80664 objects misplaced (12.558%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:26:26.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:26 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:26:26.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:26 smithi163 ceph-mon[221067]: 1.14 scrub starts 2023-10-09T04:26:26.992 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:26 smithi163 ceph-mon[221067]: pgmap v1257: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 5 remapped+peering, 10 peering, 100 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 252 op/s; 1580/80664 objects degraded (1.959%); 10130/80664 objects misplaced (12.558%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:26:27.065 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:27.276 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:27.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:27.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:27.459 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:27.553 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:27.553 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:27.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:27.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:27.624 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:27.665 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:27.668 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:27.750 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:27.773 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:27 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/80664 objects degraded (1.959%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:27.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:27.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:27.941 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:27 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/80664 objects degraded (1.959%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:27.942 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:27 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/80664 objects degraded (1.959%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:27.970 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:28.018 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:28.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:28.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:28.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:26:28.677 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:28.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:28 smithi163 ceph-mon[213841]: 10.19 scrub starts 2023-10-09T04:26:28.694 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:28 smithi163 ceph-mon[213841]: pgmap v1258: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 1580/75099 objects degraded (2.104%); 9038/75099 objects misplaced (12.035%); 3.4 MiB/s, 148 objects/s recovering 2023-10-09T04:26:28.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:28 smithi163 ceph-mon[221067]: 10.19 scrub starts 2023-10-09T04:26:28.695 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:28 smithi163 ceph-mon[221067]: pgmap v1258: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 1580/75099 objects degraded (2.104%); 9038/75099 objects misplaced (12.035%); 3.4 MiB/s, 148 objects/s recovering 2023-10-09T04:26:28.769 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:28.791 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:26:29.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:28 smithi181 ceph-mon[196979]: 10.19 scrub starts 2023-10-09T04:26:29.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:28 smithi181 ceph-mon[196979]: pgmap v1258: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 1580/75099 objects degraded (2.104%); 9038/75099 objects misplaced (12.035%); 3.4 MiB/s, 148 objects/s recovering 2023-10-09T04:26:29.429 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:29.658 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:29.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:29.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:29.763 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:29.835 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:29.969 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:29.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:30.077 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:30.146 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:30.363 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:30.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:30.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:30.381 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:30.476 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:30.518 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:30.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:30.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:30.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: 10.17 scrub starts 2023-10-09T04:26:30.635 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: 1.1c scrub starts 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[221067]: pgmap v1259: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 1580/72588 objects degraded (2.177%); 8410/72588 objects misplaced (11.586%); 2.6 MiB/s, 114 objects/s recovering 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: 10.17 scrub starts 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: 1.1c scrub starts 2023-10-09T04:26:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:30 smithi163 ceph-mon[213841]: pgmap v1259: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 1580/72588 objects degraded (2.177%); 8410/72588 objects misplaced (11.586%); 2.6 MiB/s, 114 objects/s recovering 2023-10-09T04:26:30.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:30.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: 10.17 scrub starts 2023-10-09T04:26:30.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.724 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: 1.1c scrub starts 2023-10-09T04:26:30.725 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.725 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:30.725 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:30 smithi181 ceph-mon[196979]: pgmap v1259: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 1580/72588 objects degraded (2.177%); 8410/72588 objects misplaced (11.586%); 2.6 MiB/s, 114 objects/s recovering 2023-10-09T04:26:30.728 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:30.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:30.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:30.973 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:26:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:26:30] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-10-09T04:26:31.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:31.132 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:31.335 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:31.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:31.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:31.470 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:31.514 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:31.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:31.592 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:31.592 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:31.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:31.620 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:31.620 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:31.620 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:31.620 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: 1.14 scrub starts 2023-10-09T04:26:31.620 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2255896525' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:31.621 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:31.621 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4051754645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:26:31.621 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:26:31.690 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:31.758 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:26:31.775 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: 1.14 scrub starts 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2255896525' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4051754645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:26:31.793 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:26:31.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:31.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:31.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:31.795 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: 1.14 scrub starts 2023-10-09T04:26:31.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2255896525' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:31.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:31.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4051754645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:26:31.796 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:26:31.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:31.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:31.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:32.051 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:32.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:26:32.681 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:26:32.681 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.681 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:32.681 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:26:32.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T04:26:32.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.682 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:32 smithi181 ceph-mon[196979]: pgmap v1261: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 1580/68169 objects degraded (2.318%); 8072/68169 objects misplaced (11.841%); 2.4 MiB/s, 104 objects/s recovering 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[213841]: pgmap v1261: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 1580/68169 objects degraded (2.318%); 8072/68169 objects misplaced (11.841%); 2.4 MiB/s, 104 objects/s recovering 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T04:26:32.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: failed to encode map e477 with expected crc 2023-10-09T04:26:32.834 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:32 smithi163 ceph-mon[221067]: pgmap v1261: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 1580/68169 objects degraded (2.318%); 8072/68169 objects misplaced (11.841%); 2.4 MiB/s, 104 objects/s recovering 2023-10-09T04:26:32.967 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:32.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:33.074 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:33.154 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:33.282 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:33.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:33.317 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:26:33.367 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:26:33.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:33.431 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:33 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/72588 objects degraded (2.177%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:33.465 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:33 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/72588 objects degraded (2.177%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:33 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/72588 objects degraded (2.177%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:33.617 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:33.623 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:33.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:33.651 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:33.687 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:33.724 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:33.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:33.760 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:33.792 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:33.799 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:33.843 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:33.872 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:33.881 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.881 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.881 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.881 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.881 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.881 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.882 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:26:33.882 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:26:34.024 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:34.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:34.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:34.138 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:34.189 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:34.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:34.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:34.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:34.542 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/184441983' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[213841]: 1.1c scrub starts 2023-10-09T04:26:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[213841]: pgmap v1262: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1580/64221 objects degraded (2.460%); 7609/64221 objects misplaced (11.848%); 3.7 MiB/s, 123 objects/s recovering 2023-10-09T04:26:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:34.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:34.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/184441983' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[221067]: 1.1c scrub starts 2023-10-09T04:26:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[221067]: pgmap v1262: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1580/64221 objects degraded (2.460%); 7609/64221 objects misplaced (11.848%); 3.7 MiB/s, 123 objects/s recovering 2023-10-09T04:26:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:34.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:34.606 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:34.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:34.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:34.711 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/184441983' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:34 smithi181 ceph-mon[196979]: 1.1c scrub starts 2023-10-09T04:26:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:34 smithi181 ceph-mon[196979]: pgmap v1262: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1580/64221 objects degraded (2.460%); 7609/64221 objects misplaced (11.848%); 3.7 MiB/s, 123 objects/s recovering 2023-10-09T04:26:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:34.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:34.778 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:34.849 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:34.891 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:34.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:35.001 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:35.024 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:26:35.043 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:35.172 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:35.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:35.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:35.281 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:35.331 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:35.537 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:35.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:35.623 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:26:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:26:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:26:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:26:35.623 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28475 9.1 9:8009e758:::benchmark_data_smithi163_464650_object28474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 487.81820708200001, 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.096956874, 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28475 2023-10-09T04:26:35.624 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801182+0000", 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.625 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801180+0000", 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801209+0000", 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8646e-05 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.626 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801226+0000", 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.715e-05 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801234+0000", 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3459999999999996e-06 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.627 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801291+0000", 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7438999999999997e-05 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801409+0000", 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011781399999999999 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.801670+0000", 2023-10-09T04:26:35.628 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026062299999999997 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809884+0000", 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082140670000000002 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.809890+0000", 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6460000000000004e-06 2023-10-09T04:26:35.629 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812682+0000", 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.002791615 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812694+0000", 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.171e-05 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.630 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.884929+0000", 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0722356550000001 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for degraded object", 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.885962+0000", 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010329860000000001 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898005+0000", 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012042361999999999 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898138+0000", 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013387100000000001 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.632 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28492 9.1 9:80d8c0c4:::benchmark_data_smithi163_464650_object28491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 487.70721206600001, 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9871532940000001, 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28492 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.633 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912177+0000", 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.634 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912175+0000", 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912211+0000", 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6192000000000002e-05 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.635 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912224+0000", 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2965999999999999e-05 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912230+0000", 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6540000000000004e-06 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.636 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912283+0000", 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3115999999999998e-05 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.912289+0000", 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2700000000000001e-06 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812717+0000", 2023-10-09T04:26:35.637 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90042778800000001 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812726+0000", 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5189999999999992e-06 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886022+0000", 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.073296604 2023-10-09T04:26:35.638 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886114+0000", 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1397999999999995e-05 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.886285+0000", 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000171807 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.639 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891737+0000", 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005451661 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898177+0000", 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064403500000000001 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898550+0000", 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00037258199999999999 2023-10-09T04:26:35.640 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898598+0000", 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8214000000000003e-05 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899330+0000", 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000731593 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.641 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:28496 9.1 9:834685fc:::benchmark_data_smithi163_464650_object28495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 487.69956130899999, 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9805368130000001, 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.642 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 28496 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:26:35.643 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919827+0000", 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919880+0000", 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3053000000000002e-05 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.644 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919893+0000", 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3439e-05 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919899+0000", 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.161e-06 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.645 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919957+0000", 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8118999999999997e-05 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:27.919969+0000", 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2065000000000001e-05 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812748+0000", 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.89277856099999997 2023-10-09T04:26:35.646 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:28.812756+0000", 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6079999999999998e-06 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887444+0000", 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.074688672 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.647 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887596+0000", 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000151533 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.887736+0000", 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013991200000000001 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.891747+0000", 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040110479999999997 2023-10-09T04:26:35.648 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.898230+0000", 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0064832809999999996 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899391+0000", 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001160863 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.649 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.899443+0000", 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1969000000000003e-05 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:18:29.900364+0000", 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00092131999999999999 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.650 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87402 9.d 9:b58a4af0:::benchmark_data_smithi163_464650_object87401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.72278399300001, 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9327528470000002, 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87402 2023-10-09T04:26:35.651 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.652 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896603+0000", 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896662+0000", 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8721999999999998e-05 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.653 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896672+0000", 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0342e-05 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896678+0000", 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5890000000000002e-06 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.654 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897746+0000", 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001067901 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897884+0000", 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013837699999999999 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.898027+0000", 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014309899999999999 2023-10-09T04:26:35.655 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.903780+0000", 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057525900000000001 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913854+0000", 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010074411 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.656 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913865+0000", 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0667e-05 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904284+0000", 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99041863600000002 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904295+0000", 2023-10-09T04:26:35.657 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1622e-05 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829288+0000", 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9249930480000002 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829357+0000", 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9139000000000003e-05 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.658 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87427 9.d 9:b85b518c:::benchmark_data_smithi163_464650_object87426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.68964498299999, 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9083351629999998, 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.659 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87427 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.660 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929742+0000", 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929812+0000", 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9973000000000002e-05 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929833+0000", 2023-10-09T04:26:35.661 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0372000000000001e-05 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929840+0000", 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0380000000000002e-06 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929955+0000", 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115658 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.662 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929964+0000", 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.02e-06 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904451+0000", 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97448704799999997 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.663 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904459+0000", 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829570+0000", 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9251109199999998 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829657+0000", 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6693000000000005e-05 2023-10-09T04:26:35.664 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829772+0000", 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115109 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837839+0000", 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080666330000000001 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.665 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837847+0000", 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5720000000000003e-06 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837892+0000", 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4297e-05 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837929+0000", 2023-10-09T04:26:35.666 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7622e-05 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837980+0000", 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0816000000000003e-05 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838079+0000", 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8758000000000006e-05 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.667 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87460 9.d 9:be691015:::benchmark_data_smithi163_464650_object87459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.64871736800001, 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8737964319999998, 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87460 2023-10-09T04:26:35.668 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.669 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970670+0000", 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970704+0000", 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3708999999999999e-05 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.670 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970712+0000", 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2910000000000007e-06 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970718+0000", 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2909999999999998e-06 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971579+0000", 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086131599999999999 2023-10-09T04:26:35.671 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971589+0000", 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0108e-05 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904489+0000", 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93289962699999995 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.672 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904496+0000", 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3939999999999998e-06 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832602+0000", 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9281065480000001 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832830+0000", 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022731300000000001 2023-10-09T04:26:35.673 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.833032+0000", 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202256 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837853+0000", 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048209000000000004 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.674 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838118+0000", 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000265083 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838174+0000", 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5485999999999997e-05 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.675 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844377+0000", 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062031949999999999 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844410+0000", 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3266999999999998e-05 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844468+0000", 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7698000000000002e-05 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.676 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87469 9.d 9:b0d8b7f7:::benchmark_data_smithi163_464650_object87468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.64550871400002, 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8824357369999998, 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.677 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87469 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.678 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973879+0000", 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973909+0000", 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0063999999999999e-05 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.679 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973921+0000", 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.204e-05 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973927+0000", 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2049999999999999e-06 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974432+0000", 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050552799999999999 2023-10-09T04:26:35.680 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974441+0000", 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904518+0000", 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93007710099999996 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.681 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904525+0000", 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9280000000000004e-06 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835034+0000", 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9305090869999999 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835189+0000", 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015497800000000001 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.682 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835319+0000", 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012983499999999999 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838122+0000", 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028034420000000002 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838149+0000", 2023-10-09T04:26:35.683 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6636999999999999e-05 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838193+0000", 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4202999999999997e-05 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856143+0000", 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017949975 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.684 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856198+0000", 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4968e-05 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856316+0000", 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117589 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.685 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87510 9.d 9:b1e25812:::benchmark_data_smithi163_464650_object87509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.59615130100002, 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8391414909999999, 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87510 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.686 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.687 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023236+0000", 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023264+0000", 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7639000000000001e-05 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023280+0000", 2023-10-09T04:26:35.688 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6189999999999999e-05 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023285+0000", 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4380000000000003e-06 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023338+0000", 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2713999999999997e-05 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.689 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023350+0000", 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1912999999999999e-05 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904548+0000", 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88119805299999998 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.690 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904555+0000", 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1690000000000003e-06 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838206+0000", 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9336506519999999 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838292+0000", 2023-10-09T04:26:35.691 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6051999999999995e-05 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838426+0000", 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013363900000000001 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844098+0000", 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056724699999999998 2023-10-09T04:26:35.692 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844218+0000", 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012015000000000001 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844519+0000", 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000300842 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.693 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862218+0000", 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017698935999999998 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862272+0000", 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4085999999999997e-05 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.694 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862379+0000", 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87520 9.d 9:b10313ad:::benchmark_data_smithi163_464650_object87519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.587523476, 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.830771683, 2023-10-09T04:26:35.695 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87520 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.696 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031864+0000", 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.697 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031940+0000", 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6348999999999997e-05 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031956+0000", 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5733e-05 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031962+0000", 2023-10-09T04:26:35.698 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1330000000000004e-06 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032020+0000", 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8168999999999998e-05 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032031+0000", 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0546e-05 2023-10-09T04:26:35.699 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904577+0000", 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87254678500000005 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904584+0000", 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8279999999999997e-06 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.700 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841092+0000", 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.936507684 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841230+0000", 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138245 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841385+0000", 2023-10-09T04:26:35.701 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015503399999999999 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844512+0000", 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031265519999999999 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850232+0000", 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057197300000000001 2023-10-09T04:26:35.702 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850519+0000", 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000287342 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862488+0000", 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011968807999999999 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.703 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862529+0000", 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1087999999999997e-05 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862637+0000", 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108094 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.704 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87523 9.d 9:b2b1ff3b:::benchmark_data_smithi163_464650_object87522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.58167472600002, 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8252149690000001, 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87523 2023-10-09T04:26:35.705 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.706 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037713+0000", 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037758+0000", 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5336000000000001e-05 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.707 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037769+0000", 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1139e-05 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037774+0000", 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6249999999999998e-06 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037811+0000", 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7234999999999998e-05 2023-10-09T04:26:35.708 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037819+0000", 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8599999999999993e-06 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904623+0000", 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86680350299999998 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.709 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904630+0000", 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5780000000000001e-06 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844552+0000", 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9399218949999999 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844685+0000", 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013337900000000001 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844852+0000", 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166682 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850445+0000", 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055933379999999998 2023-10-09T04:26:35.711 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850487+0000", 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1508000000000003e-05 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856104+0000", 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056174129999999999 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.712 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862788+0000", 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066833480000000004 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862832+0000", 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000002e-05 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862929+0000", 2023-10-09T04:26:35.713 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7040999999999995e-05 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87524 9.d 9:bf5a5ce2:::benchmark_data_smithi163_464650_object87523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.578950615, 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8227152589999998, 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.714 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87524 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.715 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040437+0000", 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.716 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040544+0000", 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010666399999999999 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040556+0000", 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697999999999999e-05 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040562+0000", 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0839999999999997e-06 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.717 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040647+0000", 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4971999999999999e-05 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040656+0000", 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9479999999999997e-06 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.718 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904660+0000", 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86400389600000005 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904667+0000", 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847526+0000", 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9428587660000001 2023-10-09T04:26:35.719 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847651+0000", 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125136 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847838+0000", 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018702700000000001 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.720 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850552+0000", 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002713853 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856113+0000", 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055611580000000001 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862179+0000", 2023-10-09T04:26:35.721 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060656729999999997 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863021+0000", 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000842759 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863060+0000", 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8272000000000002e-05 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.722 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863153+0000", 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3684000000000006e-05 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87531 9.d 9:b4d1cf98:::benchmark_data_smithi163_464650_object87530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.57792310399998, 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.836533765, 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.723 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87531 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.724 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041465+0000", 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.725 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041513+0000", 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8566999999999998e-05 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041522+0000", 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9020000000000005e-06 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041528+0000", 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:26:35.726 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041602+0000", 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4530000000000006e-05 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041611+0000", 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2819999999999996e-06 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.727 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904688+0000", 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86307752000000004 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904695+0000", 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850572+0000", 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9458773759999999 2023-10-09T04:26:35.728 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850685+0000", 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011277699999999999 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850827+0000", 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014244099999999999 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.729 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862146+0000", 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011318701 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862451+0000", 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030508599999999999 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.730 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862741+0000", 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029011400000000001 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877820+0000", 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015078775000000001 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877869+0000", 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9407999999999999e-05 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.731 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877999+0000", 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129868 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87536 9.d 9:b13d2384:::benchmark_data_smithi163_464650_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.567440691, 2023-10-09T04:26:35.732 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.826285253, 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87536 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:26:35.733 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051947+0000", 2023-10-09T04:26:35.734 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052032+0000", 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5885999999999997e-05 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052049+0000", 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6867e-05 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.735 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052056+0000", 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9349999999999997e-06 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052117+0000", 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0758999999999999e-05 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052127+0000", 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301e-05 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904719+0000", 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85259186799999997 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904726+0000", 2023-10-09T04:26:35.737 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7989999999999996e-06 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853627+0000", 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9489006930000001 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853769+0000", 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142619 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.738 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853901+0000", 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205399999999999 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862421+0000", 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085194190000000003 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862735+0000", 2023-10-09T04:26:35.739 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031399200000000003 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862747+0000", 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2238e-05 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878064+0000", 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015316585000000001 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.740 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878120+0000", 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6338999999999997e-05 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878233+0000", 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113268 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.741 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87542 9.d 9:b8ae8bf6:::benchmark_data_smithi163_464650_object87541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.557939111, 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.817112244, 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87542 2023-10-09T04:26:35.742 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.743 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061448+0000", 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061495+0000", 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6932999999999999e-05 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.744 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061515+0000", 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061522+0000", 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8499999999999996e-06 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.745 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061580+0000", 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8037e-05 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061591+0000", 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1099e-05 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904748+0000", 2023-10-09T04:26:35.746 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84315744100000001 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904755+0000", 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7880000000000001e-06 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856351+0000", 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9515959619999999 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.747 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856435+0000", 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3497000000000004e-05 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856590+0000", 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015579299999999999 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862697+0000", 2023-10-09T04:26:35.748 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0061066360000000004 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862753+0000", 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6106000000000001e-05 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862991+0000", 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023799100000000001 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878381+0000", 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015390452000000001 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878432+0000", 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0779999999999998e-05 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.750 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878562+0000", 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012949399999999999 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87544 9.d 9:b2959611:::benchmark_data_smithi163_464650_object87543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.557794086, 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8172143809999999, 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.751 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87544 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.752 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061594+0000", 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.753 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061631+0000", 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6940000000000002e-05 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061641+0000", 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5650000000000007e-06 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061645+0000", 2023-10-09T04:26:35.754 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9400000000000001e-06 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061692+0000", 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6638000000000003e-05 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061705+0000", 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2525e-05 2023-10-09T04:26:35.755 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904777+0000", 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84307196799999995 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904783+0000", 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.756 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859252+0000", 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.954468756 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859377+0000", 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124423 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859542+0000", 2023-10-09T04:26:35.757 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016522800000000001 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862758+0000", 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032164730000000001 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862963+0000", 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000205105 2023-10-09T04:26:35.758 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871931+0000", 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089675880000000003 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878651+0000", 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067196959999999998 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.759 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878706+0000", 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5047999999999999e-05 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878809+0000", 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103266 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.760 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87579 9.d 9:b6955889:::benchmark_data_smithi163_464650_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.50049708300003, 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.760215788, 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87579 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.761 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118890+0000", 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118943+0000", 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2917000000000002e-05 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118957+0000", 2023-10-09T04:26:35.763 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3553000000000001e-05 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118963+0000", 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9329999999999998e-06 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119027+0000", 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4450999999999997e-05 2023-10-09T04:26:35.764 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119039+0000", 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1657999999999999e-05 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904808+0000", 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78576894799999997 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.765 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904814+0000", 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.472e-06 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863179+0000", 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.958364789 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.766 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863297+0000", 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011818 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863439+0000", 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000141836 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871995+0000", 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085565290000000002 2023-10-09T04:26:35.767 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872008+0000", 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2779000000000001e-05 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878306+0000", 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062979710000000003 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.768 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878882+0000", 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057614199999999997 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878977+0000", 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4944000000000004e-05 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879107+0000", 2023-10-09T04:26:35.769 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013005199999999999 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87597 9.d 9:ba722c6f:::benchmark_data_smithi163_464650_object87596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.47211836700001, 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.741504747, 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.770 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87597 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.771 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.772 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147306+0000", 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6227999999999999e-05 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147315+0000", 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4420000000000004e-06 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147321+0000", 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3900000000000001e-06 2023-10-09T04:26:35.773 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147373+0000", 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1922000000000002e-05 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147383+0000", 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8160000000000005e-06 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.774 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904836+0000", 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75745349399999995 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904843+0000", 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9360000000000002e-06 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.775 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866166+0000", 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9613231419999999 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866285+0000", 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011870399999999999 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866435+0000", 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015030299999999999 2023-10-09T04:26:35.776 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.875034+0000", 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085985039999999999 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878346+0000", 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033125379999999999 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.777 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878607+0000", 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000261158 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888598+0000", 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099901480000000008 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888652+0000", 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4688999999999998e-05 2023-10-09T04:26:35.778 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888775+0000", 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012282100000000001 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87606 9.d 9:bd98b1c5:::benchmark_data_smithi163_464650_object87605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:26:35.779 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.45126896099998, 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7216060710000001, 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87606 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:26:35.780 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168119+0000", 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.781 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168170+0000", 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1724000000000001e-05 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168183+0000", 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2322e-05 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.782 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168188+0000", 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.011e-06 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168245+0000", 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7197999999999997e-05 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168257+0000", 2023-10-09T04:26:35.783 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1742999999999999e-05 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904867+0000", 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.73660977999999999 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904873+0000", 2023-10-09T04:26:35.784 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3450000000000002e-06 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.868939+0000", 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9640662180000001 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869059+0000", 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012025 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.785 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869200+0000", 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014040799999999999 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878017+0000", 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0088170209999999995 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.786 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878849+0000", 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00083247799999999999 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879166+0000", 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000316373 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889551+0000", 2023-10-09T04:26:35.787 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010385553 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889609+0000", 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-05 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889726+0000", 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116743 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.788 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87628 9.d 9:b09d3eb4:::benchmark_data_smithi163_464650_object87627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 429.36680831000001, 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6374180369999998, 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:26:35.789 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87628 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.790 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252579+0000", 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252624+0000", 2023-10-09T04:26:35.791 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5071999999999998e-05 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252637+0000", 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.272e-05 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252642+0000", 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8999999999999997e-06 2023-10-09T04:26:35.792 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252685+0000", 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3056000000000002e-05 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252691+0000", 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3790000000000003e-06 2023-10-09T04:26:35.793 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905181+0000", 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.65248916000000001 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905191+0000", 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0231000000000001e-05 2023-10-09T04:26:35.794 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872040+0000", 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.966849243 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872156+0000", 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115611 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.795 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872308+0000", 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152707 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878257+0000", 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059484389999999998 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879226+0000", 2023-10-09T04:26:35.796 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096947500000000002 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888556+0000", 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093301310000000002 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889825+0000", 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001268948 2023-10-09T04:26:35.797 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889882+0000", 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6885999999999997e-05 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889998+0000", 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011613899999999999 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.798 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:81266 10.10 10:0ba0e01a:::benchmark_data_smithi163_523108_object7250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 143.450873194, 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.55363772300000003, 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:26:35.799 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 81266 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:26:35.800 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168512+0000", 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999971 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168529+0000", 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6223999999999999e-05 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.801 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168551+0000", 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2459e-05 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168562+0000", 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1326e-05 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.802 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.313120+0000", 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14455803 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.313133+0000", 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2466000000000001e-05 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718112+0000", 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.404979266 2023-10-09T04:26:35.803 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718376+0000", 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026423799999999997 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718728+0000", 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00035156 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.804 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.720984+0000", 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022562839999999999 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721631+0000", 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064677500000000004 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:26:35.805 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721710+0000", 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8672999999999997e-05 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721809+0000", 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8904e-05 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.722153+0000", 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00034452899999999999 2023-10-09T04:26:35.806 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.807 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.807 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.807 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:26:35.807 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:26:35.807 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:26:35.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:35.869 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:35.962 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:35.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:35.970 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:35 smithi181 ceph-mon[196979]: 10.9 scrub starts 2023-10-09T04:26:36.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:36.133 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:36.166 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:35 smithi163 ceph-mon[213841]: 10.9 scrub starts 2023-10-09T04:26:36.167 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:35 smithi163 ceph-mon[221067]: 10.9 scrub starts 2023-10-09T04:26:36.230 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:36.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:36.288 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:36.339 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:36.406 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:36.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:26:36.824 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:26:36.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:36.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:37.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:36 smithi181 ceph-mon[196979]: pgmap v1263: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1580/62232 objects degraded (2.539%); 6969/62232 objects misplaced (11.198%); 3.5 MiB/s, 114 objects/s recovering 2023-10-09T04:26:37.045 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:37.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:36 smithi163 ceph-mon[213841]: pgmap v1263: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1580/62232 objects degraded (2.539%); 6969/62232 objects misplaced (11.198%); 3.5 MiB/s, 114 objects/s recovering 2023-10-09T04:26:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:36 smithi163 ceph-mon[221067]: pgmap v1263: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1580/62232 objects degraded (2.539%); 6969/62232 objects misplaced (11.198%); 3.5 MiB/s, 114 objects/s recovering 2023-10-09T04:26:37.115 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:37.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:37.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:37.302 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:37.371 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:37.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:37.477 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:37.520 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:37.885 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:37.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:37.978 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:26:37.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:38.060 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:38.163 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:38.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:38.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4064598887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:26:38.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:37 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:26:38.176 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:37 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/62232 objects degraded (2.539%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:38.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4064598887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:26:38.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:37 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:26:38.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:37 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/62232 objects degraded (2.539%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:38.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4064598887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:26:38.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:37 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:26:38.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:37 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/62232 objects degraded (2.539%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:38.275 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:38.358 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:38.459 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:38.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:38.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:38.611 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:38.727 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:38.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:38.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:38.906 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:39.001 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:39.009 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:38 smithi181 ceph-mon[196979]: failed to encode map e478 with expected crc 2023-10-09T04:26:39.009 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:38 smithi181 ceph-mon[196979]: failed to encode map e478 with expected crc 2023-10-09T04:26:39.010 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:38 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:26:39.010 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:38 smithi181 ceph-mon[196979]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T04:26:39.010 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:38 smithi181 ceph-mon[196979]: pgmap v1265: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 1580/53574 objects degraded (2.949%); 6085/53574 objects misplaced (11.358%); 4.0 MiB/s, 135 objects/s recovering 2023-10-09T04:26:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:39.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:39.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:26:39.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:39.151 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:39.158 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:39.159 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:39.240 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:39.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[221067]: failed to encode map e478 with expected crc 2023-10-09T04:26:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[221067]: failed to encode map e478 with expected crc 2023-10-09T04:26:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:26:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[221067]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T04:26:39.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[221067]: pgmap v1265: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 1580/53574 objects degraded (2.949%); 6085/53574 objects misplaced (11.358%); 4.0 MiB/s, 135 objects/s recovering 2023-10-09T04:26:39.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[213841]: failed to encode map e478 with expected crc 2023-10-09T04:26:39.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[213841]: failed to encode map e478 with expected crc 2023-10-09T04:26:39.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:26:39.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[213841]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T04:26:39.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:38 smithi163 ceph-mon[213841]: pgmap v1265: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 1580/53574 objects degraded (2.949%); 6085/53574 objects misplaced (11.358%); 4.0 MiB/s, 135 objects/s recovering 2023-10-09T04:26:39.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:39.450 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:39.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:39.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:39.785 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:39.860 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:40.035 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:40.038 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:26:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:40.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:40.043 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3236459968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:40.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3236459968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:40.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:40.220 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3236459968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:26:40.923 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:26:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:26:40] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.43.0" 2023-10-09T04:26:40.958 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:40.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:41.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:41.133 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:40 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3236459968' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:40 smithi181 ceph-mon[196979]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T04:26:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:40 smithi181 ceph-mon[196979]: failed to encode map e479 with expected crc 2023-10-09T04:26:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:40 smithi181 ceph-mon[196979]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1580/50484 objects degraded (3.130%); 5338/50484 objects misplaced (10.574%); 3.9 MiB/s, 135 objects/s recovering 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3236459968' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[221067]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[221067]: failed to encode map e479 with expected crc 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[221067]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1580/50484 objects degraded (3.130%); 5338/50484 objects misplaced (10.574%); 3.9 MiB/s, 135 objects/s recovering 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3236459968' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:26:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[213841]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T04:26:41.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[213841]: failed to encode map e479 with expected crc 2023-10-09T04:26:41.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:40 smithi163 ceph-mon[213841]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1580/50484 objects degraded (3.130%); 5338/50484 objects misplaced (10.574%); 3.9 MiB/s, 135 objects/s recovering 2023-10-09T04:26:41.558 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:41.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:41.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:41.662 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:41.690 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:41.739 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:41.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:41.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:41.876 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:42.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:42 smithi181 ceph-mon[196979]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T04:26:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:42 smithi181 ceph-mon[196979]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:42 smithi181 ceph-mon[196979]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:42 smithi181 ceph-mon[196979]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[213841]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T04:26:42.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[213841]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[213841]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[213841]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[221067]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T04:26:42.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[221067]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[221067]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:42 smithi163 ceph-mon[221067]: failed to encode map e480 with expected crc 2023-10-09T04:26:42.421 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:42.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:42.528 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:42.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:42.623 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:42.912 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:42.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:43.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:43.061 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:43.065 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:43.101 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:43.135 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:43.144 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:43.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:43.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:43.169 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:43.209 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:43.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: pgmap v1269: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 933 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 964 op/s; 1580/44853 objects degraded (3.523%); 4845/44853 objects misplaced (10.802%); 3.4 MiB/s, 142 objects/s recovering 2023-10-09T04:26:43.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 1580/44853 objects degraded (3.523%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: failed to encode map e482 with expected crc 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[213841]: failed to encode map e482 with expected crc 2023-10-09T04:26:43.218 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: pgmap v1269: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 933 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 964 op/s; 1580/44853 objects degraded (3.523%); 4845/44853 objects misplaced (10.802%); 3.4 MiB/s, 142 objects/s recovering 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 1580/44853 objects degraded (3.523%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: failed to encode map e482 with expected crc 2023-10-09T04:26:43.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:43 smithi163 ceph-mon[221067]: failed to encode map e482 with expected crc 2023-10-09T04:26:43.240 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:43.251 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:43.267 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:43.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: pgmap v1269: 129 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 116 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 933 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 964 op/s; 1580/44853 objects degraded (3.523%); 4845/44853 objects misplaced (10.802%); 3.4 MiB/s, 142 objects/s recovering 2023-10-09T04:26:43.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: failed to encode map e481 with expected crc 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 1580/44853 objects degraded (3.523%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: failed to encode map e482 with expected crc 2023-10-09T04:26:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:43 smithi181 ceph-mon[196979]: failed to encode map e482 with expected crc 2023-10-09T04:26:43.300 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:43.304 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:43.310 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:43.310 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:43.312 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.312 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.313 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.313 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.313 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.313 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.313 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:26:43.313 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:26:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: failed to encode map e482 with expected crc 2023-10-09T04:26:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2833578064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T04:26:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[221067]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: failed to encode map e482 with expected crc 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2833578064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:44 smithi163 ceph-mon[213841]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: failed to encode map e482 with expected crc 2023-10-09T04:26:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2833578064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T04:26:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:44 smithi181 ceph-mon[196979]: failed to encode map e483 with expected crc 2023-10-09T04:26:44.559 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:26:44.821 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:44.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:44.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:44.927 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:45.015 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:45.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:45 smithi163 ceph-mon[221067]: failed to encode map e483 with expected crc 2023-10-09T04:26:45.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:45 smithi163 ceph-mon[221067]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 117 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 847 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 774/40719 objects degraded (1.901%); 4602/40719 objects misplaced (11.302%); 3.5 MiB/s, 373 objects/s recovering 2023-10-09T04:26:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:45 smithi163 ceph-mon[213841]: failed to encode map e483 with expected crc 2023-10-09T04:26:45.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:45 smithi163 ceph-mon[213841]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 117 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 847 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 774/40719 objects degraded (1.901%); 4602/40719 objects misplaced (11.302%); 3.5 MiB/s, 373 objects/s recovering 2023-10-09T04:26:45.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:45.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:45.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:45.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:45.233 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:45.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:45 smithi181 ceph-mon[196979]: failed to encode map e483 with expected crc 2023-10-09T04:26:45.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:45 smithi181 ceph-mon[196979]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 117 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 847 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 774/40719 objects degraded (1.901%); 4602/40719 objects misplaced (11.302%); 3.5 MiB/s, 373 objects/s recovering 2023-10-09T04:26:45.287 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:45.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:45.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:45.418 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:45.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:45.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:45.572 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:45.606 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:45.796 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:26:45.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:45.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:45.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:45.941 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:45.999 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:46.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:46.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T04:26:46.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[213841]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:46 smithi163 ceph-mon[221067]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.090 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:46.104 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:46.146 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:46.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:46.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:46.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T04:26:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.356 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.356 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.356 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:46 smithi181 ceph-mon[196979]: failed to encode map e484 with expected crc 2023-10-09T04:26:46.356 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:46.422 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:46.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:46.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:46.527 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:46.558 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:46.615 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:46.759 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:46.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:46.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:46.893 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:47.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: pgmap v1275: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 117 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 811 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 774/38985 objects degraded (1.985%); 4064/38985 objects misplaced (10.425%); 2.3 MiB/s, 354 objects/s recovering 2023-10-09T04:26:47.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.049 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.052 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/285335772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: pgmap v1275: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 117 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 811 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 774/38985 objects degraded (1.985%); 4064/38985 objects misplaced (10.425%); 2.3 MiB/s, 354 objects/s recovering 2023-10-09T04:26:47.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T04:26:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/285335772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:26:47.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:26:47.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: pgmap v1275: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 117 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait; 811 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 774/38985 objects degraded (1.985%); 4064/38985 objects misplaced (10.425%); 2.3 MiB/s, 354 objects/s recovering 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: failed to encode map e485 with expected crc 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/285335772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:26:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:26:47.565 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:26:47.722 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:47.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:47.833 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:47.867 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:47.931 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:47.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:48.039 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:48.067 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: failed to encode map e486 with expected crc 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: failed to encode map e486 with expected crc 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: Health check update: Degraded data redundancy: 774/38985 objects degraded (1.985%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T04:26:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:48 smithi181 ceph-mon[196979]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: failed to encode map e486 with expected crc 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: failed to encode map e486 with expected crc 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: Health check update: Degraded data redundancy: 774/38985 objects degraded (1.985%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[221067]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: failed to encode map e486 with expected crc 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: failed to encode map e486 with expected crc 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: Health check update: Degraded data redundancy: 774/38985 objects degraded (1.985%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:48 smithi163 ceph-mon[213841]: failed to encode map e487 with expected crc 2023-10-09T04:26:48.422 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:48.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:48.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:48.526 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:48.552 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:48.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:26:48.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:48.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:48.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:48.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:48.729 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:48.810 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:48.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:48.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:48.946 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:49.008 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:49.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:49.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: failed to encode map e487 with expected crc 2023-10-09T04:26:49.055 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: pgmap v1279: 129 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+clean+remapped, 120 active+clean, 6 active+remapped+backfill_wait; 699 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 2683/33621 objects misplaced (7.980%); 19 MiB/s, 431 objects/s recovering 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 774/38985 objects degraded (1.985%), 1 pg degraded, 2 pgs undersized) 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[213841]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: failed to encode map e487 with expected crc 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: pgmap v1279: 129 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+clean+remapped, 120 active+clean, 6 active+remapped+backfill_wait; 699 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 2683/33621 objects misplaced (7.980%); 19 MiB/s, 431 objects/s recovering 2023-10-09T04:26:49.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 774/38985 objects degraded (1.985%), 1 pg degraded, 2 pgs undersized) 2023-10-09T04:26:49.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T04:26:49.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.057 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:49 smithi163 ceph-mon[221067]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.114 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:49.164 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:49.241 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:26:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:49.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:26:49.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: failed to encode map e487 with expected crc 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: pgmap v1279: 129 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+clean+remapped, 120 active+clean, 6 active+remapped+backfill_wait; 699 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 2683/33621 objects misplaced (7.980%); 19 MiB/s, 431 objects/s recovering 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 774/38985 objects degraded (1.985%), 1 pg degraded, 2 pgs undersized) 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:49 smithi181 ceph-mon[196979]: failed to encode map e488 with expected crc 2023-10-09T04:26:49.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:49.377 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:49.426 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:49.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:49.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:49.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.556 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:49.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:49.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:49.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:49.696 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:26:49.701 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.730 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:49.818 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:49.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:49.930 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:49.990 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:50.104 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:50.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:50.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: failed to encode map e488 with expected crc 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4289115071' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4289115071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.116 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.117 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:50 smithi181 ceph-mon[196979]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:50.260 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: failed to encode map e488 with expected crc 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4289115071' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4289115071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[213841]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: failed to encode map e488 with expected crc 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:50.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4289115071' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:26:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:26:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4289115071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:26:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T04:26:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:50 smithi163 ceph-mon[221067]: failed to encode map e489 with expected crc 2023-10-09T04:26:50.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:50.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:50.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:50.462 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:50.488 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:50.789 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:26:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:26:50] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.43.0" 2023-10-09T04:26:50.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:50.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:50.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:50.934 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:50.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:51.025 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:51.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:51 smithi163 ceph-mon[213841]: failed to encode map e489 with expected crc 2023-10-09T04:26:51.063 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:51 smithi163 ceph-mon[213841]: pgmap v1282: 129 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+clean+remapped, 120 active+clean, 6 active+remapped+backfill_wait; 681 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 389 op/s; 2399/32739 objects misplaced (7.328%); 8.7 MiB/s, 260 objects/s recovering 2023-10-09T04:26:51.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:51.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:51.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:51.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:51 smithi181 ceph-mon[196979]: failed to encode map e489 with expected crc 2023-10-09T04:26:51.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:51 smithi181 ceph-mon[196979]: pgmap v1282: 129 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+clean+remapped, 120 active+clean, 6 active+remapped+backfill_wait; 681 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 389 op/s; 2399/32739 objects misplaced (7.328%); 8.7 MiB/s, 260 objects/s recovering 2023-10-09T04:26:51.271 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:51.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:51 smithi163 ceph-mon[221067]: failed to encode map e489 with expected crc 2023-10-09T04:26:51.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:51 smithi163 ceph-mon[221067]: pgmap v1282: 129 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+clean+remapped, 120 active+clean, 6 active+remapped+backfill_wait; 681 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 389 op/s; 2399/32739 objects misplaced (7.328%); 8.7 MiB/s, 260 objects/s recovering 2023-10-09T04:26:51.324 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:51.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:51.334 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:51.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:51.460 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:51.480 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:26:51.480 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:26:51.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:51.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:51.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:51.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:51.652 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:52.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[213841]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T04:26:52.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[213841]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[213841]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[213841]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[213841]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.080 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[213841]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[221067]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T04:26:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[221067]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[221067]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[221067]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[221067]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:52 smithi163 ceph-mon[221067]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:52 smithi181 ceph-mon[196979]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T04:26:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:52 smithi181 ceph-mon[196979]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:52 smithi181 ceph-mon[196979]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:52 smithi181 ceph-mon[196979]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:52 smithi181 ceph-mon[196979]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:52 smithi181 ceph-mon[196979]: failed to encode map e490 with expected crc 2023-10-09T04:26:52.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: pgmap v1284: 129 pgs: 1 peering, 123 active+clean, 5 active+remapped+backfill_wait; 596 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 1736/28638 objects misplaced (6.062%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T04:26:53.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[221067]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: pgmap v1284: 129 pgs: 1 peering, 123 active+clean, 5 active+remapped+backfill_wait; 596 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 1736/28638 objects misplaced (6.062%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T04:26:53.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:53 smithi163 ceph-mon[213841]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: pgmap v1284: 129 pgs: 1 peering, 123 active+clean, 5 active+remapped+backfill_wait; 596 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 1736/28638 objects misplaced (6.062%); 8.6 MiB/s, 138 objects/s recovering 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e491 with expected crc 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:53 smithi181 ceph-mon[196979]: failed to encode map e492 with expected crc 2023-10-09T04:26:53.574 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:53.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:53.613 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:26:53.679 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:53.706 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:53.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:53.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:53.859 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:53.918 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[221067]: failed to encode map e492 with expected crc 2023-10-09T04:26:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[221067]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[221067]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T04:26:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[221067]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[221067]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[213841]: failed to encode map e492 with expected crc 2023-10-09T04:26:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[213841]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[213841]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T04:26:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[213841]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:54 smithi163 ceph-mon[213841]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:26:54.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:26:54.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:26:54.433 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:26:54.442 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:26:54.466 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:26:54.493 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:26:54.515 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.518 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:26:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:54 smithi181 ceph-mon[196979]: failed to encode map e492 with expected crc 2023-10-09T04:26:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:54 smithi181 ceph-mon[196979]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:54 smithi181 ceph-mon[196979]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T04:26:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:54 smithi181 ceph-mon[196979]: failed to encode map e493 with expected crc 2023-10-09T04:26:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:54 smithi181 ceph-mon[196979]: failed to encode map e493 with expected crc 2023-10-09T04:26:55.188 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:55.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:55.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:55.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:55 smithi181 ceph-mon[196979]: failed to encode map e493 with expected crc 2023-10-09T04:26:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:55 smithi181 ceph-mon[196979]: pgmap v1288: 129 pgs: 125 active+clean, 4 active+remapped+backfill_wait; 528 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 1394/25383 objects misplaced (5.492%); 14 MiB/s, 286 objects/s recovering 2023-10-09T04:26:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4132994998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:55 smithi181 ceph-mon[196979]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T04:26:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:55 smithi181 ceph-mon[196979]: failed to encode map e494 with expected crc 2023-10-09T04:26:55.348 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:55 smithi181 ceph-mon[196979]: failed to encode map e494 with expected crc 2023-10-09T04:26:55.353 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:55.462 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:55.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:55.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:26:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[213841]: failed to encode map e493 with expected crc 2023-10-09T04:26:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[213841]: pgmap v1288: 129 pgs: 125 active+clean, 4 active+remapped+backfill_wait; 528 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 1394/25383 objects misplaced (5.492%); 14 MiB/s, 286 objects/s recovering 2023-10-09T04:26:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4132994998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[213841]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T04:26:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[213841]: failed to encode map e494 with expected crc 2023-10-09T04:26:55.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[213841]: failed to encode map e494 with expected crc 2023-10-09T04:26:55.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[221067]: failed to encode map e493 with expected crc 2023-10-09T04:26:55.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[221067]: pgmap v1288: 129 pgs: 125 active+clean, 4 active+remapped+backfill_wait; 528 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 1394/25383 objects misplaced (5.492%); 14 MiB/s, 286 objects/s recovering 2023-10-09T04:26:55.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4132994998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:26:55.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[221067]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T04:26:55.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[221067]: failed to encode map e494 with expected crc 2023-10-09T04:26:55.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:55 smithi163 ceph-mon[221067]: failed to encode map e494 with expected crc 2023-10-09T04:26:55.637 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:55.710 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:26:55.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:55.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:26:55.823 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:55.878 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:56.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:56 smithi163 ceph-mon[213841]: failed to encode map e494 with expected crc 2023-10-09T04:26:56.179 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:26:56.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:56 smithi163 ceph-mon[221067]: failed to encode map e494 with expected crc 2023-10-09T04:26:56.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:26:56.258 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:56.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:56.281 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:56.362 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:26:56.393 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:56 smithi181 ceph-mon[196979]: failed to encode map e494 with expected crc 2023-10-09T04:26:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:26:56.674 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:26:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:56.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:26:56.779 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:56.806 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:56.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:56.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:56.954 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:56.983 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:57.007 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:57.029 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:57.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:57.112 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:26:57.134 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:57.161 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:57.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:57.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:57.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: pgmap v1290: 129 pgs: 1 active+remapped+backfilling, 125 active+clean, 3 active+remapped+backfill_wait; 515 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 321 op/s; 926/24777 objects misplaced (3.737%); 8.7 MiB/s, 196 objects/s recovering 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1482914229' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:26:57.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: pgmap v1290: 129 pgs: 1 active+remapped+backfilling, 125 active+clean, 3 active+remapped+backfill_wait; 515 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 321 op/s; 926/24777 objects misplaced (3.737%); 8.7 MiB/s, 196 objects/s recovering 2023-10-09T04:26:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:26:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T04:26:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1482914229' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:26:57.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: pgmap v1290: 129 pgs: 1 active+remapped+backfilling, 125 active+clean, 3 active+remapped+backfill_wait; 515 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 321 op/s; 926/24777 objects misplaced (3.737%); 8.7 MiB/s, 196 objects/s recovering 2023-10-09T04:26:57.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:26:57.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T04:26:57.353 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: failed to encode map e495 with expected crc 2023-10-09T04:26:57.354 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1482914229' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:26:57.354 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:57.406 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:26:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:57.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:26:57.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:26:57.540 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:57.600 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:26:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:57.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:26:57.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:26:57.734 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:58.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:26:58.386 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[213841]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[213841]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1482914229' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[213841]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[213841]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[213841]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[221067]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[221067]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1482914229' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:26:58.387 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[221067]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T04:26:58.388 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[221067]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.388 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:58 smithi163 ceph-mon[221067]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.433 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:26:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:58.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:26:58.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:26:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:58 smithi181 ceph-mon[196979]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:58 smithi181 ceph-mon[196979]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1482914229' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:26:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:58 smithi181 ceph-mon[196979]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T04:26:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:58 smithi181 ceph-mon[196979]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:58 smithi181 ceph-mon[196979]: failed to encode map e496 with expected crc 2023-10-09T04:26:58.543 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:58.589 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:58.660 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:58.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:58.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:58.766 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:26:58.856 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:59.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: pgmap v1293: 129 pgs: 1 remapped+peering, 128 active+clean; 315 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 15 MiB/s, 236 objects/s recovering 2023-10-09T04:26:59.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T04:26:59.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: 10.10 starting backfill to osd.7 from (0'0,443'5482] MIN to 494'6954 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[221067]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: pgmap v1293: 129 pgs: 1 remapped+peering, 128 active+clean; 315 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 15 MiB/s, 236 objects/s recovering 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: 10.10 starting backfill to osd.7 from (0'0,443'5482] MIN to 494'6954 2023-10-09T04:26:59.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:26:59 smithi163 ceph-mon[213841]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.499 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:26:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:59.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: pgmap v1293: 129 pgs: 1 remapped+peering, 128 active+clean; 315 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 15 MiB/s, 236 objects/s recovering 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: 10.10 starting backfill to osd.7 from (0'0,443'5482] MIN to 494'6954 2023-10-09T04:26:59.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:26:59 smithi181 ceph-mon[196979]: failed to encode map e497 with expected crc 2023-10-09T04:26:59.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:26:59.654 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:26:59.748 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:26:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:26:59.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:26:59.755 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:59.801 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:26:59.855 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:26:59.946 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:00.039 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:00.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:00.144 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:00.189 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97814 9.8 9:1e2e17f2:::benchmark_data_smithi163_464650_object97813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:27:00.374 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.65974126899999, 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0107446179999999, 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97814 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.375 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710189+0000", 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710188+0000", 2023-10-09T04:27:00.376 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710223+0000", 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5763000000000003e-05 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710243+0000", 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9536000000000001e-05 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.377 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.378 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.378 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710260+0000", 2023-10-09T04:27:00.378 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7271000000000001e-05 2023-10-09T04:27:00.378 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.378 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710304+0000", 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3745000000000002e-05 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710446+0000", 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014181999999999999 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.379 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.710552+0000", 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010643500000000001 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.713530+0000", 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029773289999999999 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717690+0000", 2023-10-09T04:27:00.380 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041607049999999998 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.742122+0000", 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.024431905 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.742134+0000", 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1542000000000001e-05 2023-10-09T04:27:00.381 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506765+0000", 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76463178499999995 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506780+0000", 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4603000000000001e-05 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.382 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.720842+0000", 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2140615079999999 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.720934+0000", 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2258999999999999e-05 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.383 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97818 9.8 9:11ed80ac:::benchmark_data_smithi163_464650_object97817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.655310112, 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0067211999999999, 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97818 2023-10-09T04:27:00.384 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714620+0000", 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.385 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714618+0000", 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714643+0000", 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4170999999999999e-05 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.386 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714655+0000", 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2646000000000001e-05 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.714667+0000", 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1525000000000001e-05 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717718+0000", 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003051649 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.387 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717853+0000", 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013490299999999999 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.717991+0000", 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137861 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.388 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.723501+0000", 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055102470000000002 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.726686+0000", 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031843050000000001 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.743057+0000", 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016371198999999999 2023-10-09T04:27:00.389 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.743067+0000", 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0241000000000001e-05 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506817+0000", 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76374998100000002 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.390 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.506828+0000", 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0964999999999999e-05 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721244+0000", 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2144160310000001 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721342+0000", 2023-10-09T04:27:00.391 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7343999999999995e-05 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97832 9.8 9:1e55ad63:::benchmark_data_smithi163_464650_object97831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.63925427300001, 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.99079312, 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.392 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97832 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.393 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730676+0000", 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.394 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730695+0000", 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8825999999999999e-05 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730702+0000", 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1600000000000001e-06 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730707+0000", 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8840000000000002e-06 2023-10-09T04:27:00.395 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730736+0000", 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9824000000000001e-05 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.730895+0000", 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015838500000000001 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.396 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.731058+0000", 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016319699999999999 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.736028+0000", 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049698779999999996 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745215+0000", 2023-10-09T04:27:00.397 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091867310000000001 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745227+0000", 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1897e-05 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507474+0000", 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76224778000000004 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.398 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507486+0000", 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1707e-05 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721391+0000", 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.213904777 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721469+0000", 2023-10-09T04:27:00.399 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8479999999999994e-05 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97836 9.8 9:11733f2c:::benchmark_data_smithi163_464650_object97835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.62583837400001, 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.996319304, 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.400 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97836 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.401 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744092+0000", 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744090+0000", 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744115+0000", 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4522999999999999e-05 2023-10-09T04:27:00.402 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744128+0000", 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3434999999999999e-05 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744138+0000", 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0118e-05 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.403 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745249+0000", 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011104089999999999 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745254+0000", 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8379999999999998e-06 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507512+0000", 2023-10-09T04:27:00.404 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.762257929 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507520+0000", 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9240000000000007e-06 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721492+0000", 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.213971412 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.405 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721620+0000", 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000128085 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.721817+0000", 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000196839 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.406 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725055+0000", 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032383199999999998 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732136+0000", 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070812619999999996 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732296+0000", 2023-10-09T04:27:00.407 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015994499999999999 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740237+0000", 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079407409999999994 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740292+0000", 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5081999999999999e-05 2023-10-09T04:27:00.408 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740411+0000", 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000119461 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97838 9.8 9:1b793ee6:::benchmark_data_smithi163_464650_object97837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2023-10-09T04:27:00.409 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.62573586899998, 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0048248380000002, 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97838 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.410 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744195+0000", 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.411 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744194+0000", 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744242+0000", 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8263999999999997e-05 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744253+0000", 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0642000000000001e-05 2023-10-09T04:27:00.412 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.744259+0000", 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3040000000000001e-06 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745272+0000", 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010125170000000001 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.413 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.745278+0000", 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.127e-06 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507541+0000", 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.76226340000000004 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507549+0000", 2023-10-09T04:27:00.414 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6620000000000005e-06 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725084+0000", 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2175345179999999 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725227+0000", 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000143406 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.415 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.725417+0000", 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000189956 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732176+0000", 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067589599999999996 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.416 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732240+0000", 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4319000000000005e-05 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740463+0000", 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082230529999999993 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.748849+0000", 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0083855000000000006 2023-10-09T04:27:00.417 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.748896+0000", 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7463000000000003e-05 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749019+0000", 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012314499999999999 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.418 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97866 9.8 9:1cea5b31:::benchmark_data_smithi163_464650_object97865:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.594228461, 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.973609446, 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.419 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97866 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.420 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775702+0000", 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775722+0000", 2023-10-09T04:27:00.421 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0279999999999999e-05 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775729+0000", 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775734+0000", 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1359999999999996e-06 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.422 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775796+0000", 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2526000000000004e-05 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.775811+0000", 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4799e-05 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.423 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507569+0000", 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.73175752299999997 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507576+0000", 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.424 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728482+0000", 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2209053270000001 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728643+0000", 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016166299999999999 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.728783+0000", 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000139185 2023-10-09T04:27:00.425 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732184+0000", 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034017660000000001 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732369+0000", 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018494 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.426 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740499+0000", 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0081298159999999998 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749147+0000", 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086481760000000005 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749200+0000", 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2543999999999997e-05 2023-10-09T04:27:00.427 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749311+0000", 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111702 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97872 9.8 9:10a54521:::benchmark_data_smithi163_464650_object97871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:27:00.428 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.584225242, 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.963828709, 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97872 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:27:00.429 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785705+0000", 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785703+0000", 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.430 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785769+0000", 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6533999999999997e-05 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785789+0000", 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.431 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785799+0000", 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5719999999999992e-06 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785910+0000", 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111128 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785917+0000", 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3359999999999997e-06 2023-10-09T04:27:00.432 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507597+0000", 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.72167993600000002 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507606+0000", 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1589999999999993e-06 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.433 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732411+0000", 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.224805452 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732576+0000", 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016478 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.732767+0000", 2023-10-09T04:27:00.434 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019163200000000001 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740425+0000", 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076572660000000002 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740534+0000", 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000109341 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.435 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749083+0000", 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085489380000000007 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749377+0000", 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029441100000000001 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749423+0000", 2023-10-09T04:27:00.436 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5676000000000001e-05 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749534+0000", 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110911 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97874 9.8 9:191c8b34:::benchmark_data_smithi163_464650_object97873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:27:00.437 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.58411572400001, 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9835905089999999, 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97874 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.438 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785815+0000", 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.439 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785814+0000", 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785855+0000", 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0932e-05 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785866+0000", 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0762e-05 2023-10-09T04:27:00.440 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785871+0000", 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7550000000000002e-06 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785935+0000", 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4851999999999995e-05 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.441 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.785940+0000", 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2119999999999997e-06 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507627+0000", 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.72168717900000001 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507635+0000", 2023-10-09T04:27:00.442 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3699999999999995e-06 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736110+0000", 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2284743970000001 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736281+0000", 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017120699999999999 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.443 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.736480+0000", 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019965799999999999 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740574+0000", 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0040938049999999998 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.444 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.749102+0000", 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085278049999999994 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753493+0000", 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043906470000000001 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769243+0000", 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015749984000000002 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.445 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769290+0000", 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7129000000000002e-05 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769405+0000", 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115487 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.446 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97892 9.8 9:1aaa6a58:::benchmark_data_smithi163_464650_object97891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.56130038499998, 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9609627970000001, 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97892 2023-10-09T04:27:00.447 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808630+0000", 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.448 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808629+0000", 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808672+0000", 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3235000000000002e-05 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.449 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808684+0000", 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2139e-05 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808692+0000", 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6420000000000007e-06 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808752+0000", 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9707999999999998e-05 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.450 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.808760+0000", 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3569999999999998e-06 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507656+0000", 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.69889576899999994 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.451 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507663+0000", 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6120000000000001e-06 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740602+0000", 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2329382529999999 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740744+0000", 2023-10-09T04:27:00.452 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000142769 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.740917+0000", 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017210500000000001 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753421+0000", 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012504678999999999 2023-10-09T04:27:00.453 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753524+0000", 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102972 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.761433+0000", 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0079086420000000004 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.454 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769451+0000", 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080184650000000007 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769498+0000", 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6409999999999998e-05 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769593+0000", 2023-10-09T04:27:00.455 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5168999999999996e-05 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:97904 9.8 9:1009d856:::benchmark_data_smithi163_464650_object97903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 438.54828538300001, 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9481932799999999, 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.456 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 97904 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.457 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821645+0000", 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821643+0000", 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821692+0000", 2023-10-09T04:27:00.458 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9016e-05 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821706+0000", 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3781000000000001e-05 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821713+0000", 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3730000000000004e-06 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.459 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821759+0000", 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5587999999999998e-05 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:41.821769+0000", 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9369999999999999e-06 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507684+0000", 2023-10-09T04:27:00.460 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.68591571799999995 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:42.507693+0000", 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1689999999999992e-06 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745049+0000", 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2373563329999999 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.461 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745227+0000", 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017849599999999999 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,5,7", 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.745426+0000", 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019907700000000001 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.462 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753455+0000", 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0080286279999999995 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.753558+0000", 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010336399999999999 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.761441+0000", 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078828909999999995 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.463 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769701+0000", 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0082594929999999997 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769746+0000", 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4821999999999999e-05 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.464 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:43.769838+0000", 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2744000000000005e-05 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18935 10.b 10:da4c922f:::benchmark_data_smithi163_523108_object18934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 273.38951564400003, 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0962053709999999, 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.465 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18935 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.466 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980415+0000", 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980413+0000", 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.467 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980478+0000", 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4868000000000003e-05 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980501+0000", 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3396000000000001e-05 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980530+0000", 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9082999999999999e-05 2023-10-09T04:27:00.468 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980605+0000", 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5334999999999996e-05 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op must wait for map", 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980613+0000", 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6529999999999994e-06 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.469 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985700+0000", 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0050872310000000002 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985710+0000", 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0077e-05 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.470 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.054356+0000", 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0686453579999999 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.054995+0000", 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00063910900000000001 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.055173+0000", 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000178314 2023-10-09T04:27:00.471 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058584+0000", 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034106679999999999 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058630+0000", 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6400999999999998e-05 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.472 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060716+0000", 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020859139999999999 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.075527+0000", 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014810489 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.075578+0000", 2023-10-09T04:27:00.473 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0995999999999998e-05 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076620+0000", 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001042608 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18945 10.b 10:db72c492:::benchmark_data_smithi163_523108_object18944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:27:00.474 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 273.38937414700001, 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.097054776, 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18945 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.475 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980556+0000", 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.476 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980610+0000", 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4863e-05 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980622+0000", 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1056000000000001e-05 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.477 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980628+0000", 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4520000000000002e-06 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980665+0000", 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7508000000000001e-05 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:27:00.478 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980671+0000", 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1889999999999998e-06 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985732+0000", 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005061767 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985739+0000", 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8000000000000001e-06 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.479 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056187+0000", 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0704480949999999 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056380+0000", 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019281100000000001 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:27:00.480 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.056554+0000", 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017387899999999999 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058639+0000", 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002084776 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058684+0000", 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5627000000000002e-05 2023-10-09T04:27:00.481 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060847+0000", 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002162298 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076681+0000", 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015834559000000002 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.482 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.076730+0000", 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8708000000000002e-05 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077611+0000", 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00088110100000000002 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.483 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18948 10.b 10:d3e53c2e:::benchmark_data_smithi163_523108_object18947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e390)", 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 273.38929560000003, 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0978110240000001, 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18948 2023-10-09T04:27:00.484 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980635+0000", 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.485 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980634+0000", 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980675+0000", 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0608999999999997e-05 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.486 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980682+0000", 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7939999999999997e-06 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.980686+0000", 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0589999999999996e-06 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.487 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.981834+0000", 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011482910000000001 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.981840+0000", 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3459999999999999e-06 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985759+0000", 2023-10-09T04:27:00.488 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039191210000000002 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.985767+0000", 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.4109999999999999e-06 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057486+0000", 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.071719098 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.489 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057617+0000", 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013104999999999999 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057754+0000", 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137408 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.490 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058774+0000", 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001020137 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058886+0000", 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111945 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060884+0000", 2023-10-09T04:27:00.491 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001997619 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077700+0000", 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016816296000000001 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.077760+0000", 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0031999999999997e-05 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.492 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.078446+0000", 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00068554800000000002 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57312 10.8 10.a8007c28 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.334509+0000", 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 235.03542178800001, 2023-10-09T04:27:00.493 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99230618599999998, 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57312 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334509+0000", 2023-10-09T04:27:00.494 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334509+0000", 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334507+0000", 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.495 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334532+0000", 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5179999999999999e-05 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334542+0000", 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5999999999999996e-06 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334551+0000", 2023-10-09T04:27:00.496 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.1279999999999997e-06 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334879+0000", 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032774200000000001 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334890+0000", 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0776e-05 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.497 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326799+0000", 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99190986299999995 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326815+0000", 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5413e-05 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.498 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57318 10.8 10.a56ae5f8 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 235.03536745100001, 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99228850099999999, 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.499 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57318 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.500 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334563+0000", 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.501 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334583+0000", 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0534e-05 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334590+0000", 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1060000000000002e-06 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.502 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334596+0000", 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.694e-06 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334912+0000", 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00031580300000000003 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334918+0000", 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7220000000000002e-06 2023-10-09T04:27:00.503 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326841+0000", 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99192297100000004 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326852+0000", 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0145e-05 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.504 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57319 10.a 10.930077a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.334601+0000", 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 235.03532996199999, 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99232309100000005, 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.505 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57319 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334601+0000", 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334601+0000", 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.506 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334600+0000", 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334621+0000", 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0503999999999999e-05 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.507 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334628+0000", 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.968e-06 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334634+0000", 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4999999999999996e-06 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.508 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334953+0000", 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000319303 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.334961+0000", 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9719999999999992e-06 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326912+0000", 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99195034800000004 2023-10-09T04:27:00.509 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.326924+0000", 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1855000000000001e-05 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGPush(10.d 434/415 [PushOp(10:b0c90586:::benchmark_data_smithi163_523108_object20642:head, version: 392'1242, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:b0c90586:::benchmark_data_smithi163_523108_object20642:head@392'1242, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:27:00.510 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 157.790791122, 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.445266852, 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.511 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579139+0000", 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.512 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579201+0000", 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2989000000000003e-05 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579216+0000", 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4229999999999999e-05 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.579227+0000", 2023-10-09T04:27:00.513 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1024e-05 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:25.003833+0000", 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.424606394 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:25.004367+0000", 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053335800000000005 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.514 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:25.024406+0000", 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.020039699000000001 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:91037 10.14 10:2b87ac6f:::benchmark_data_smithi163_523108_object17021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:42.555201+0000", 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 137.81472913100001, 2023-10-09T04:27:00.515 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1080779919999999, 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 91037 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555201+0000", 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.516 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555201+0000", 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555200+0000", 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555206+0000", 2023-10-09T04:27:00.517 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7009999999999999e-06 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555219+0000", 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2549e-05 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.555226+0000", 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2289999999999997e-06 2023-10-09T04:27:00.518 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.559168+0000", 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0039416590000000001 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.559289+0000", 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121478 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:27:00.519 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.559613+0000", 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00032357399999999998 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.562434+0000", 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0028212699999999999 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.562439+0000", 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8949999999999997e-06 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.520 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.662683+0000", 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.100244059 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.662697+0000", 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3648999999999999e-05 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663224+0000", 2023-10-09T04:27:00.521 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0005272300000001 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663279+0000", 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5408000000000002e-05 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:91049 10.14 10:2bda9338:::benchmark_data_smithi163_523108_object17033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e439)", 2023-10-09T04:27:00.522 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:42.680082+0000", 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 137.68984835800001, 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99102864800000001, 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 91049 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.523 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680082+0000", 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680082+0000", 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680081+0000", 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.524 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680087+0000", 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5880000000000003e-06 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680097+0000", 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.842e-06 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.525 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680102+0000", 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3090000000000003e-06 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680717+0000", 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00061489499999999996 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:42.680732+0000", 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.473e-05 2023-10-09T04:27:00.526 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663300+0000", 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98256744699999998 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663401+0000", 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101123 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.527 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.663699+0000", 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029785499999999997 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.669391+0000", 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005692679 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.670991+0000", 2023-10-09T04:27:00.528 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015994620000000001 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.671025+0000", 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3853000000000003e-05 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.671067+0000", 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2527000000000001e-05 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.529 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:43.671111+0000", 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3696999999999999e-05 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:121925 10.14 10:2fd1a8e0:::benchmark_data_smithi163_523108_object47909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:23.680650+0000", 2023-10-09T04:27:00.530 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 36.689280388, 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.016767508, 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 121925 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680650+0000", 2023-10-09T04:27:00.531 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680650+0000", 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680648+0000", 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.532 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680655+0000", 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0110000000000003e-06 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680671+0000", 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5221e-05 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:00.533 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680678+0000", 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6510000000000001e-06 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680736+0000", 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7306000000000003e-05 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:23.680746+0000", 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0672e-05 2023-10-09T04:27:00.534 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.692520+0000", 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0117737170000001 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.692655+0000", 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013488199999999999 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.535 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.692931+0000", 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027616199999999999 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.697199+0000", 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0042684560000000003 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.697300+0000", 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000100455 2023-10-09T04:27:00.536 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.697350+0000", 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0416e-05 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.697393+0000", 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2967999999999999e-05 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:00.537 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:24.697418+0000", 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4314000000000001e-05 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:00.538 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:27:00.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:00.539 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:00.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:00.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:00.666 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:00.699 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:27:00.712 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:00.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:00.818 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:00.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T04:27:00.821 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1545828152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.822 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[221067]: pgmap v1296: 129 pgs: 1 remapped+peering, 128 active+clean; 315 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 12 MiB/s, 195 objects/s recovering 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1545828152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: failed to encode map e498 with expected crc 2023-10-09T04:27:00.823 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:00 smithi163 ceph-mon[213841]: pgmap v1296: 129 pgs: 1 remapped+peering, 128 active+clean; 315 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 12 MiB/s, 195 objects/s recovering 2023-10-09T04:27:00.824 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:27:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:27:00] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.43.0" 2023-10-09T04:27:00.852 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: failed to encode map e498 with expected crc 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: failed to encode map e498 with expected crc 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1545828152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: failed to encode map e498 with expected crc 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: failed to encode map e498 with expected crc 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: failed to encode map e498 with expected crc 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: failed to encode map e498 with expected crc 2023-10-09T04:27:01.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:00 smithi181 ceph-mon[196979]: pgmap v1296: 129 pgs: 1 remapped+peering, 128 active+clean; 315 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 12 MiB/s, 195 objects/s recovering 2023-10-09T04:27:02.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: 1.1c scrub starts 2023-10-09T04:27:02.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:01 smithi181 ceph-mon[196979]: 1.1c scrub ok 2023-10-09T04:27:02.020 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:27:01 smithi181 conmon[175350]: logger=cleanup t=2023-10-09T04:27:01.579346819Z level=info msg="Completed cleanup jobs" duration=1.049188284s 2023-10-09T04:27:02.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:02.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:02.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T04:27:02.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: 1.1c scrub starts 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[213841]: 1.1c scrub ok 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: failed to encode map e499 with expected crc 2023-10-09T04:27:02.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: 1.1c scrub starts 2023-10-09T04:27:02.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:01 smithi163 ceph-mon[221067]: 1.1c scrub ok 2023-10-09T04:27:02.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:27:02.250 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:02.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:02.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:02.415 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:03.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:02 smithi181 ceph-mon[196979]: pgmap v1298: 129 pgs: 1 remapped+peering, 128 active+clean; 204 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 937 KiB/s, 14 objects/s recovering 2023-10-09T04:27:03.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:02 smithi163 ceph-mon[213841]: pgmap v1298: 129 pgs: 1 remapped+peering, 128 active+clean; 204 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 937 KiB/s, 14 objects/s recovering 2023-10-09T04:27:03.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:02 smithi163 ceph-mon[221067]: pgmap v1298: 129 pgs: 1 remapped+peering, 128 active+clean; 204 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 937 KiB/s, 14 objects/s recovering 2023-10-09T04:27:03.294 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:03.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:03.408 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.497 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:03.510 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 74013 objects 2023-10-09T04:27:03.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :194.734 2023-10-09T04:27:03.535 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:03.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:03.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:03.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:03.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:03.680 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:03.805 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:03.806 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:03 smithi163 ceph-mon[221067]: 1.11 scrub starts 2023-10-09T04:27:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:03.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:03.808 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:03 smithi163 ceph-mon[213841]: 1.11 scrub starts 2023-10-09T04:27:03.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:03.939 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:03.986 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:03.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:04.002 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:27:04.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:03 smithi181 ceph-mon[196979]: 1.11 scrub starts 2023-10-09T04:27:04.022 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:27:04.042 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:27:04.066 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:27:04.085 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:27:04.090 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:04.107 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:27:04.117 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:04.127 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:27:04.150 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:27:04.151 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.152 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:27:04.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:04.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:04.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:04.270 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:04.304 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:04.356 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:04.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:04.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:04.491 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:04.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:04.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:04.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:04.637 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:04.663 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:04.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:04.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:04.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:04.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:27:04.731 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:27:04.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:04.839 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:04.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:04.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:04.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:04.975 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: 1.11 scrub ok 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: 1.19 scrub starts 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: 1.19 scrub ok 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3942615465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: pgmap v1299: 129 pgs: 129 active+clean; 98 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 316 KiB/s, 4 objects/s recovering 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3206938223' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:27:04.976 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: 1.11 scrub ok 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: 1.19 scrub starts 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: 1.19 scrub ok 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3942615465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: pgmap v1299: 129 pgs: 129 active+clean; 98 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 316 KiB/s, 4 objects/s recovering 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3206938223' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:27:04.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:04.980 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T04:27:04.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T04:27:04.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:05.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: 1.11 scrub ok 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: 1.19 scrub starts 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: 1.19 scrub ok 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3942615465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: pgmap v1299: 129 pgs: 129 active+clean; 98 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 316 KiB/s, 4 objects/s recovering 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3206938223' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:27:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:05.051 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: 1.e scrub starts 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: 1.e scrub ok 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: failed to encode map e500 with expected crc 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: failed to encode map e500 with expected crc 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:27:05.811 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: 10.4 scrub starts 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[213841]: 10.4 scrub ok 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: 1.e scrub starts 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: 1.e scrub ok 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: failed to encode map e500 with expected crc 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: failed to encode map e500 with expected crc 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:27:05.812 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T04:27:05.813 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: 10.4 scrub starts 2023-10-09T04:27:05.813 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:05 smithi163 ceph-mon[221067]: 10.4 scrub ok 2023-10-09T04:27:05.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:27:05.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:05.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:05.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:06.013 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: 1.e scrub starts 2023-10-09T04:27:06.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: 1.e scrub ok 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: failed to encode map e500 with expected crc 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: failed to encode map e500 with expected crc 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: 10.4 scrub starts 2023-10-09T04:27:06.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:05 smithi181 ceph-mon[196979]: 10.4 scrub ok 2023-10-09T04:27:06.044 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:06.093 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:06.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:06.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:06.227 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:06.239 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:06.272 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:06.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:06.300 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:27:06.300 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:27:06.380 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:06.413 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:06.793 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:07.002 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[221067]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[221067]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[221067]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T04:27:07.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[221067]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[221067]: pgmap v1302: 129 pgs: 129 active+clean; 98 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 316 KiB/s, 4 objects/s recovering 2023-10-09T04:27:07.003 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:27:07.003 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[213841]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[213841]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[213841]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T04:27:07.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[213841]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[213841]: pgmap v1302: 129 pgs: 129 active+clean; 98 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 316 KiB/s, 4 objects/s recovering 2023-10-09T04:27:07.004 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:27:07.004 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:07.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:06 smithi181 ceph-mon[196979]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:06 smithi181 ceph-mon[196979]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:06 smithi181 ceph-mon[196979]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T04:27:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:06 smithi181 ceph-mon[196979]: failed to encode map e501 with expected crc 2023-10-09T04:27:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:06 smithi181 ceph-mon[196979]: pgmap v1302: 129 pgs: 129 active+clean; 98 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 316 KiB/s, 4 objects/s recovering 2023-10-09T04:27:07.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:27:07.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:27:07.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:07.142 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:07.169 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:07.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:27:08.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: 1.1a scrub starts 2023-10-09T04:27:08.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: 1.1a scrub ok 2023-10-09T04:27:08.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[221067]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: 1.1a scrub starts 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: 1.1a scrub ok 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T04:27:08.010 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.011 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:07 smithi163 ceph-mon[213841]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T04:27:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: 1.1a scrub starts 2023-10-09T04:27:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: 1.1a scrub ok 2023-10-09T04:27:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:27:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T04:27:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: failed to encode map e502 with expected crc 2023-10-09T04:27:08.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:07 smithi181 ceph-mon[196979]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T04:27:08.184 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:08.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:08.289 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:08.315 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:08.358 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:08.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:08.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:08.493 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:08.824 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:08.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:08.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:08.931 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:08.958 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:09.002 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:09.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:08 smithi181 ceph-mon[196979]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:08 smithi181 ceph-mon[196979]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:08 smithi181 ceph-mon[196979]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:08 smithi181 ceph-mon[196979]: pgmap v1305: 129 pgs: 1 remapped+peering, 128 active+clean; 19 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:09.007 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:08 smithi181 ceph-mon[196979]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T04:27:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:09.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[221067]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[221067]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[221067]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[221067]: pgmap v1305: 129 pgs: 1 remapped+peering, 128 active+clean; 19 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[221067]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T04:27:09.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[213841]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[213841]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[213841]: failed to encode map e503 with expected crc 2023-10-09T04:27:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[213841]: pgmap v1305: 129 pgs: 1 remapped+peering, 128 active+clean; 19 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:09.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:08 smithi163 ceph-mon[213841]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T04:27:09.109 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:09.136 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:09.178 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:09.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:09.285 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:09.311 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:09.355 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:09.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:09.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:09.486 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:09.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:09.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:09.636 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:09.664 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:09.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:09 smithi163 ceph-mon[213841]: 10.1 scrub starts 2023-10-09T04:27:09.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:09 smithi163 ceph-mon[213841]: failed to encode map e504 with expected crc 2023-10-09T04:27:09.977 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:09 smithi163 ceph-mon[213841]: failed to encode map e504 with expected crc 2023-10-09T04:27:09.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:09 smithi163 ceph-mon[221067]: 10.1 scrub starts 2023-10-09T04:27:09.977 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:09 smithi163 ceph-mon[221067]: failed to encode map e504 with expected crc 2023-10-09T04:27:09.978 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:09 smithi163 ceph-mon[221067]: failed to encode map e504 with expected crc 2023-10-09T04:27:10.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:09 smithi181 ceph-mon[196979]: 10.1 scrub starts 2023-10-09T04:27:10.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:09 smithi181 ceph-mon[196979]: failed to encode map e504 with expected crc 2023-10-09T04:27:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:09 smithi181 ceph-mon[196979]: failed to encode map e504 with expected crc 2023-10-09T04:27:10.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:10.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:10.872 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:10.896 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:27:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:27:10] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-10-09T04:27:10.900 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:11.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[213841]: pgmap v1307: 129 pgs: 1 remapped+peering, 128 active+clean; 19 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:11.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:11.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[213841]: 1.5 scrub starts 2023-10-09T04:27:11.203 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[213841]: 1.5 scrub ok 2023-10-09T04:27:11.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[221067]: pgmap v1307: 129 pgs: 1 remapped+peering, 128 active+clean; 19 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:11.203 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:11.204 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[221067]: 1.5 scrub starts 2023-10-09T04:27:11.204 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:11 smithi163 ceph-mon[221067]: 1.5 scrub ok 2023-10-09T04:27:11.206 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:11.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:11.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:11.343 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:11.367 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:11.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:11.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:11 smithi181 ceph-mon[196979]: pgmap v1307: 129 pgs: 1 remapped+peering, 128 active+clean; 19 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:11.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:11 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:11.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:11 smithi181 ceph-mon[196979]: 1.5 scrub starts 2023-10-09T04:27:11.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:11 smithi181 ceph-mon[196979]: 1.5 scrub ok 2023-10-09T04:27:11.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:11.542 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:11.558 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:11.904 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:11.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:11.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:12.010 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:12.037 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:12.079 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:12.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:12.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:12.108 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_4' removed 2023-10-09T04:27:12.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:12.211 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:12.254 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:12.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:12.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2627263847' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:27:12.339 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2627263847' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:27:12.359 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:12.391 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:12.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2627263847' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:27:13.157 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: pgmap v1308: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 17 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 4632/867 objects degraded (534.256%); 2316/867 objects misplaced (267.128%) 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: Health check failed: Degraded data redundancy: 4632/867 objects degraded (534.256%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2627263847' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3509972034' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/927288380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:13.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:13 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: pgmap v1308: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 17 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 4632/867 objects degraded (534.256%); 2316/867 objects misplaced (267.128%) 2023-10-09T04:27:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: Health check failed: Degraded data redundancy: 4632/867 objects degraded (534.256%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:27:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2627263847' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:27:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3509972034' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/927288380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: pgmap v1308: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 17 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 4632/867 objects degraded (534.256%); 2316/867 objects misplaced (267.128%) 2023-10-09T04:27:13.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: Health check failed: Degraded data redundancy: 4632/867 objects degraded (534.256%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2627263847' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3509972034' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: failed to encode map e505 with expected crc 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/927288380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:13.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:13 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:14 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:27:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:27:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:14 smithi181 ceph-mon[196979]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T04:27:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:14 smithi181 ceph-mon[196979]: failed to encode map e506 with expected crc 2023-10-09T04:27:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:14 smithi181 ceph-mon[196979]: failed to encode map e506 with expected crc 2023-10-09T04:27:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:27:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:27:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[213841]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T04:27:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[213841]: failed to encode map e506 with expected crc 2023-10-09T04:27:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[213841]: failed to encode map e506 with expected crc 2023-10-09T04:27:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:27:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:27:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[221067]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T04:27:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[221067]: failed to encode map e506 with expected crc 2023-10-09T04:27:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:14 smithi163 ceph-mon[221067]: failed to encode map e506 with expected crc 2023-10-09T04:27:14.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:27:14.664 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:27:15.153 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:15.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:15.166 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:27:15.167 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:27:15.167 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T04:27:15.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T04:27:15.258 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:15.285 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:15 smithi181 ceph-mon[196979]: pgmap v1311: 103 pgs: 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:15 smithi181 ceph-mon[196979]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4632/867 objects degraded (534.256%), 1 pg degraded) 2023-10-09T04:27:15.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:15 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:27:15.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:15 smithi163 ceph-mon[221067]: pgmap v1311: 103 pgs: 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:15.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:15 smithi163 ceph-mon[221067]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4632/867 objects degraded (534.256%), 1 pg degraded) 2023-10-09T04:27:15.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:15 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:27:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:15 smithi163 ceph-mon[213841]: pgmap v1311: 103 pgs: 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:15 smithi163 ceph-mon[213841]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4632/867 objects degraded (534.256%), 1 pg degraded) 2023-10-09T04:27:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:15 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:27:15.688 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:15.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:15.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:15.829 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:15.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:15.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:15.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:15.920 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:15.959 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:15.983 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:16.014 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:16.027 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:27:16.027 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:27:16.133 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:16.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:16.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:16.270 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:16.318 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:16.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:16.378 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:27:16.398 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:27:16.418 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:27:16.426 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:16.437 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:27:16.455 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:16.456 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:27:16.474 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:27:16.496 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:27:16.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:16.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:16.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:16.516 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:27:16.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.517 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.518 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.518 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.518 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.518 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:27:16.609 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:16.640 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:17.381 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:17 smithi163 ceph-mon[221067]: pgmap v1312: 103 pgs: 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:17.381 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/776771912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:17.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:17 smithi163 ceph-mon[213841]: pgmap v1312: 103 pgs: 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:17.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/776771912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:17 smithi181 ceph-mon[196979]: pgmap v1312: 103 pgs: 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/776771912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:17.839 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:27:18.144 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:18.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:18.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:18.248 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:18.274 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:18.409 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:27:18.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:18.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:18.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:18.563 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:18.591 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:18.702 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:18.862 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:18.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:18.966 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:18.995 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:18 smithi181 ceph-mon[196979]: pgmap v1313: 103 pgs: 103 active+clean; 599 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:18.998 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:19.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:19.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:19.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:18 smithi163 ceph-mon[213841]: pgmap v1313: 103 pgs: 103 active+clean; 599 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:19.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:18 smithi163 ceph-mon[221067]: pgmap v1313: 103 pgs: 103 active+clean; 599 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:19.151 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:19.180 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:19.225 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:19.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:19.266 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:19.330 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:19.357 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:19.399 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:19.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:19.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:19.505 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:19.535 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:19.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:19.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:19.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:19.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:19.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/865083048' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:27:19.697 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/351954397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:27:19.713 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:27:19.714 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_5' created 2023-10-09T04:27:19.717 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:19.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/865083048' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:27:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/351954397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:27:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:19.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:19.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:27:20.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/865083048' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:27:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/351954397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:27:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:20.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:20.169 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:20.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:20.274 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:20.303 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:20.350 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:20.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:20.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:20.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:20.486 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:20.535 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:20.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:20.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:20.672 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:20.724 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:20.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:20.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: failed to encode map e507 with expected crc 2023-10-09T04:27:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: failed to encode map e507 with expected crc 2023-10-09T04:27:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/865083048' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:27:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/351954397' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:27:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T04:27:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: pgmap v1315: 119 pgs: 16 unknown, 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:20.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:20.832 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:20.859 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:27:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:27:20] "GET /metrics HTTP/1.1" 200 34205 "" "Prometheus/2.43.0" 2023-10-09T04:27:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: failed to encode map e507 with expected crc 2023-10-09T04:27:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: failed to encode map e507 with expected crc 2023-10-09T04:27:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/865083048' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:27:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/351954397' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:27:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: pgmap v1315: 119 pgs: 16 unknown, 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: failed to encode map e507 with expected crc 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: failed to encode map e507 with expected crc 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/865083048' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/351954397' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: pgmap v1315: 119 pgs: 16 unknown, 103 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:21.122 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:27:21.442 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T04:27:21.458 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:21.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:21.563 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:21.592 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:21.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:21.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:21.953 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:21.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:22.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:21 smithi181 ceph-mon[196979]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:21 smithi181 ceph-mon[196979]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T04:27:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:21 smithi181 ceph-mon[196979]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:21 smithi181 ceph-mon[196979]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:21 smithi181 ceph-mon[196979]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[213841]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[213841]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T04:27:22.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[213841]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[213841]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.039 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[213841]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.039 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[221067]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[221067]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T04:27:22.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[221067]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[221067]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.040 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:21 smithi163 ceph-mon[221067]: failed to encode map e508 with expected crc 2023-10-09T04:27:22.042 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:22.132 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:22.215 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:22.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:22.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:22.349 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:22.516 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:22.726 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:27:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:22 smithi181 ceph-mon[196979]: failed to encode map e509 with expected crc 2023-10-09T04:27:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:22 smithi181 ceph-mon[196979]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T04:27:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:22 smithi181 ceph-mon[196979]: failed to encode map e509 with expected crc 2023-10-09T04:27:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2086068843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:27:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:22 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:27:23.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:22 smithi181 ceph-mon[196979]: pgmap v1318: 119 pgs: 1 creating+peering, 4 unknown, 114 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[221067]: failed to encode map e509 with expected crc 2023-10-09T04:27:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[221067]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T04:27:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[221067]: failed to encode map e509 with expected crc 2023-10-09T04:27:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2086068843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:27:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:27:23.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[221067]: pgmap v1318: 119 pgs: 1 creating+peering, 4 unknown, 114 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[213841]: failed to encode map e509 with expected crc 2023-10-09T04:27:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[213841]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T04:27:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[213841]: failed to encode map e509 with expected crc 2023-10-09T04:27:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2086068843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:27:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:27:23.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:22 smithi163 ceph-mon[213841]: pgmap v1318: 119 pgs: 1 creating+peering, 4 unknown, 114 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:23.114 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:23.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.35475.0:170514 8.1 8:8f7d6c46:::benchmark_data_smithi163_404719_object79930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-10-09T04:27:23.203 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 591.33172045200001, 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27164164200000002, 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.35475", 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/1593451486", 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 170514 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868514+0000", 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868512+0000", 2023-10-09T04:27:23.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868530+0000", 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8127000000000001e-05 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868545+0000", 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4941e-05 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.868556+0000", 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1137e-05 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.871810+0000", 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0032543070000000001 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.871939+0000", 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012900100000000001 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.872059+0000", 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120219 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.878504+0000", 2023-10-09T04:27:23.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0064444419999999999 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:31.883814+0000", 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053105230000000002 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140056+0000", 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25624188199999998 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140112+0000", 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5835000000000003e-05 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:17:32.140156+0000", 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3615999999999997e-05 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.210 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:63536 9.0 9:0aceaddd:::benchmark_data_smithi163_464650_object63535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 502.83162497400002, 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28875457799999998, 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:23.211 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 63536 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368609+0000", 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368608+0000", 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368630+0000", 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2226e-05 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368640+0000", 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8500000000000006e-06 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.368647+0000", 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2179999999999996e-06 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534087+0000", 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16544025500000001 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534242+0000", 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000155162 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.534420+0000", 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017757 2023-10-09T04:27:23.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.543991+0000", 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095711679999999997 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544001+0000", 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0030000000000001e-05 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.216 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.656468+0000", 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.112466861 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.656522+0000", 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4855000000000002e-05 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657364+0000", 2023-10-09T04:27:23.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00084163200000000004 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:63537 9.0 9:03499599:::benchmark_data_smithi163_464650_object63536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e325)", 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 502.82980286600002, 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28755178100000001, 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.218 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 63537 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370432+0000", 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370430+0000", 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370480+0000", 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9536000000000001e-05 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370503+0000", 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3405e-05 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.370518+0000", 2023-10-09T04:27:23.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4299e-05 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536564+0000", 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16604591899999999 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536746+0000", 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018275999999999999 2023-10-09T04:27:23.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.536962+0000", 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021562400000000001 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544049+0000", 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0070868019999999997 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.544171+0000", 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012230300000000001 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657426+0000", 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.113255047 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657481+0000", 2023-10-09T04:27:23.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4827999999999998e-05 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:00.657983+0000", 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00050239800000000004 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.225 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:108768 9.2 9:40bf3da0:::benchmark_data_smithi163_464650_object10042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 445.20360227200001, 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26903813999999998, 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 108768 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996632+0000", 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996634+0000", 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3300000000000001e-06 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996640+0000", 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9089999999999997e-06 2023-10-09T04:27:23.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996645+0000", 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7729999999999999e-06 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.996907+0000", 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00026236199999999998 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.997011+0000", 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010401800000000001 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:57.997115+0000", 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000103387 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.003248+0000", 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006133416 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.008657+0000", 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0054094549999999996 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265580+0000", 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25692238499999998 2023-10-09T04:27:23.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265624+0000", 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3671999999999998e-05 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:19:58.265670+0000", 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6819e-05 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.232 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181567 9.2 9:4eb3f07f:::benchmark_data_smithi163_464650_object82841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.364667+0000", 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 347.83556777500002, 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27748786399999997, 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:23.233 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181567 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364667+0000", 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364667+0000", 2023-10-09T04:27:23.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364665+0000", 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364670+0000", 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8149999999999996e-06 2023-10-09T04:27:23.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364679+0000", 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.166e-06 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364686+0000", 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7560000000000002e-06 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364774+0000", 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8299000000000003e-05 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.364908+0000", 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000133788 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:27:23.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.365032+0000", 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012386499999999999 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.368771+0000", 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0037393410000000002 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371873+0000", 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003101295 2023-10-09T04:27:23.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642059+0000", 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27018651900000001 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642104+0000", 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4570000000000002e-05 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642155+0000", 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0759e-05 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181570 9.2 9:4981e7d2:::benchmark_data_smithi163_464650_object82844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.371071+0000", 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 347.829163725, 2023-10-09T04:27:23.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27127820699999999, 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181570 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371071+0000", 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371071+0000", 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371070+0000", 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:23.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371073+0000", 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8999999999999999e-06 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371082+0000", 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1450000000000008e-06 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.371087+0000", 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3970000000000002e-06 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.372364+0000", 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012767259999999999 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.372466+0000", 2023-10-09T04:27:23.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000102143 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.372568+0000", 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101891 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.375746+0000", 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031787429999999999 2023-10-09T04:27:23.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642225+0000", 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26647827499999999 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642259+0000", 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3791999999999998e-05 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642302+0000", 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3707e-05 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.642349+0000", 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6697999999999999e-05 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.55504.0:181585 9.2 9:40695247:::benchmark_data_smithi163_464650_object82859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:21:35.378714+0000", 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 347.82152050299999, 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27045982299999999, 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.55504", 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 181585 2023-10-09T04:27:23.248 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378714+0000", 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378714+0000", 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378713+0000", 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378719+0000", 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8810000000000001e-06 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378731+0000", 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2203e-05 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.378739+0000", 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8660000000000006e-06 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.381455+0000", 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027158270000000001 2023-10-09T04:27:23.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.381579+0000", 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012458799999999999 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.639098+0000", 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25751881399999998 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.645662+0000", 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0065638379999999998 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.648839+0000", 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0031765949999999999 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649077+0000", 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023882100000000001 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649128+0000", 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0918000000000003e-05 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:21:35.649174+0000", 2023-10-09T04:27:23.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5463000000000002e-05 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18929 10.2 10:4b019e5b:::benchmark_data_smithi163_523108_object18928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.977843+0000", 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 297.22239189099997, 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.073312767, 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.255 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18929 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977843+0000", 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977843+0000", 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977840+0000", 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977867+0000", 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6684000000000001e-05 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977892+0000", 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4726e-05 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977903+0000", 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1025e-05 2023-10-09T04:27:23.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977980+0000", 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7068000000000001e-05 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978175+0000", 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000195582 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978346+0000", 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017094699999999999 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.980370+0000", 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020234089999999999 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.982534+0000", 2023-10-09T04:27:23.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021639900000000002 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.001908+0000", 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019374031999999999 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.001922+0000", 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3845e-05 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.988348+0000", 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98642666099999998 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.988365+0000", 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6818000000000001e-05 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.051057+0000", 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.062691794 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.051155+0000", 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8371999999999999e-05 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57314 10.2 10:4147658d:::benchmark_data_smithi163_523108_object57313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.325214+0000", 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 257.87502017499997, 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1597814309999999, 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57314 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.264 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325214+0000", 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325214+0000", 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325213+0000", 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325262+0000", 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9441999999999999e-05 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325276+0000", 2023-10-09T04:27:23.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4296e-05 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.325287+0000", 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0152999999999999e-05 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326696+0000", 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0014094929999999999 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326849+0000", 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000152876 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326988+0000", 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000139398 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.336681+0000", 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0096929470000000004 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.341192+0000", 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0045103360000000002 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.341202+0000", 2023-10-09T04:27:23.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0454e-05 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329833+0000", 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98863050699999999 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329846+0000", 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3923e-05 2023-10-09T04:27:23.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470799+0000", 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1409526269999999 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.471670+0000", 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00087087700000000004 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.484877+0000", 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013206542999999999 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.484996+0000", 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011919200000000001 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57316 10.2 10:4066bc76:::benchmark_data_smithi163_523108_object57315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.326433+0000", 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 257.87380187500003, 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1601299439999999, 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57316 2023-10-09T04:27:23.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326433+0000", 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326433+0000", 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326432+0000", 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326466+0000", 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4625999999999997e-05 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326483+0000", 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7314000000000001e-05 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326493+0000", 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.2620000000000004e-06 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.329254+0000", 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027612349999999999 2023-10-09T04:27:23.276 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.329406+0000", 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000152328 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.329576+0000", 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016951800000000001 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.277 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.336689+0000", 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0071129790000000002 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342259+0000", 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055705290000000003 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342272+0000", 2023-10-09T04:27:23.278 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2525e-05 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329893+0000", 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98762081999999995 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329904+0000", 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.114e-05 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.279 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.472026+0000", 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1421222550000001 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.472797+0000", 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000771256 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.280 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486491+0000", 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.013694045 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486563+0000", 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1188999999999999e-05 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57320 10.2 10:473b6798:::benchmark_data_smithi163_523108_object57319:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e406)", 2023-10-09T04:27:23.281 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.326501+0000", 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 257.87373373499997, 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1599652659999999, 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57320 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.282 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326501+0000", 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326501+0000", 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.283 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326500+0000", 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326524+0000", 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3416000000000001e-05 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326531+0000", 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3289999999999996e-06 2023-10-09T04:27:23.284 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.326537+0000", 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7470000000000002e-06 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.331699+0000", 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.005161845 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.285 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.331842+0000", 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014325699999999999 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.331981+0000", 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013920499999999999 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.336694+0000", 2023-10-09T04:27:23.286 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047133059999999996 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342967+0000", 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062731410000000003 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.342980+0000", 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2309e-05 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.287 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329939+0000", 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.98695938800000005 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329950+0000", 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1323e-05 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.288 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.472855+0000", 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1429041090000001 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.473758+0000", 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00090376599999999997 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486384+0000", 2023-10-09T04:27:23.289 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.012625716 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486466+0000", 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1926000000000001e-05 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57329 10.2 10:47781a5f:::benchmark_data_smithi163_523108_object57328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:27:23.290 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:05.358571+0000", 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 257.84166356200001, 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1352695960000001, 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57329 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.291 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358571+0000", 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358571+0000", 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.292 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358569+0000", 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358592+0000", 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2637e-05 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358609+0000", 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6674e-05 2023-10-09T04:27:23.293 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358620+0000", 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0896e-05 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358699+0000", 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9841999999999999e-05 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.294 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:05.358716+0000", 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6494000000000002e-05 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329977+0000", 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97126138699999998 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:23.295 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.329986+0000", 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8079999999999994e-06 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.477576+0000", 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1475897989999999 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.477764+0000", 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188104 2023-10-09T04:27:23.296 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.477940+0000", 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000175589 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.483913+0000", 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0059738120000000002 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.297 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.485670+0000", 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001756514 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486360+0000", 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00069008199999999996 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.493667+0000", 2023-10-09T04:27:23.298 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0073070449999999999 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.493726+0000", 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8751e-05 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.493841+0000", 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000114654 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.299 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:61232 10.2 10:4b04ec52:::benchmark_data_smithi163_523108_object61231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:11.662119+0000", 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 251.538115113, 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29074994799999998, 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.300 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 61232 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662119+0000", 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662119+0000", 2023-10-09T04:27:23.301 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662118+0000", 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662148+0000", 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9969000000000001e-05 2023-10-09T04:27:23.302 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662162+0000", 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4095e-05 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662171+0000", 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8570000000000007e-06 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.303 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662230+0000", 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.821e-05 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662446+0000", 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021639000000000001 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662675+0000", 2023-10-09T04:27:23.304 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00022903800000000001 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668635+0000", 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0059601699999999999 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668670+0000", 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4845000000000003e-05 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.305 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668677+0000", 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2679999999999997e-06 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.952695+0000", 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28401721600000002 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.306 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.952752+0000", 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7858000000000001e-05 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.952869+0000", 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116857 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.307 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:61235 10.2 10:4472a39f:::benchmark_data_smithi163_523108_object61234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:11.662486+0000", 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 251.537748502, 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.290560178, 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 61235 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.308 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662486+0000", 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662486+0000", 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.309 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662485+0000", 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662511+0000", 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6307000000000001e-05 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662523+0000", 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1596e-05 2023-10-09T04:27:23.310 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662530+0000", 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2189999999999998e-06 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.664956+0000", 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.002426115 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.311 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.665096+0000", 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013919099999999999 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.665219+0000", 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012321599999999999 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668683+0000", 2023-10-09T04:27:23.312 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034638949999999998 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668706+0000", 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3390000000000001e-05 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.670236+0000", 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001529891 2023-10-09T04:27:23.313 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.952909+0000", 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28267330000000002 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.952949+0000", 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0204999999999999e-05 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.314 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.953046+0000", 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.6676999999999996e-05 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:61237 10.2 10:4f19ea81:::benchmark_data_smithi163_523108_object61236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:11.662543+0000", 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 251.53769194700001, 2023-10-09T04:27:23.315 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.29318258600000002, 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 61237 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662543+0000", 2023-10-09T04:27:23.316 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662543+0000", 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662542+0000", 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.317 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662561+0000", 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.942e-05 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662568+0000", 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4010000000000002e-06 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.318 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.662574+0000", 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7409999999999998e-06 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.666717+0000", 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0041430089999999996 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.319 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.666851+0000", 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013412600000000001 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.666994+0000", 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014336399999999999 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668733+0000", 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001739174 2023-10-09T04:27:23.320 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.668739+0000", 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9699999999999996e-06 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.670357+0000", 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0016177299999999999 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.321 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.955542+0000", 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.28518459699999998 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.955608+0000", 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6166000000000003e-05 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:11.955725+0000", 2023-10-09T04:27:23.322 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117366 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:120903 10.10 10:0c77eaca:::benchmark_data_smithi163_523108_object46887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:21.592448+0000", 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 61.607786181000002, 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.38329649999999998, 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.323 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 120903 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592448+0000", 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.324 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592448+0000", 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592446+0000", 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.325 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592457+0000", 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1197e-05 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592483+0000", 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5043999999999999e-05 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.326 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592493+0000", 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0057e-05 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592553+0000", 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0123e-05 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.592691+0000", 2023-10-09T04:27:23.327 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000137937 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.593076+0000", 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00038531699999999998 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.595183+0000", 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0021075299999999998 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.328 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.596580+0000", 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0013966359999999999 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.975390+0000", 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.378810325 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.329 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.975448+0000", 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7533e-05 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.975745+0000", 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00029682200000000003 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.330 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:134419 10.10 10:0d7c9333:::benchmark_data_smithi163_523108_object60403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e479)", 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:40.996421+0000", 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 42.203812986000003, 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.118153959, 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 134419 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.331 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:40.996421+0000", 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:40.996421+0000", 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.332 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:40.996420+0000", 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:40.996428+0000", 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6879999999999999e-06 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.333 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:40.996446+0000", 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8547999999999999e-05 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:40.996457+0000", 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0443e-05 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.004132+0000", 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0076749940000000001 2023-10-09T04:27:23.334 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.004288+0000", 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015635099999999999 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.004708+0000", 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000420092 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.335 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.006973+0000", 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022645709999999999 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.015771+0000", 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0087985170000000005 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.336 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114389+0000", 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.09861789 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114440+0000", 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0485000000000002e-05 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114575+0000", 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013575800000000001 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.337 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:134440 10.0 10:05cc7956:::benchmark_data_smithi163_523108_object60424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e480)", 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:41.019926+0000", 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 42.180308377000003, 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.108132467, 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:23.338 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 134440 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.019926+0000", 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.339 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.019926+0000", 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.019924+0000", 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.019932+0000", 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0669999999999992e-06 2023-10-09T04:27:23.340 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.113086+0000", 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.093153896 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.113099+0000", 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2197e-05 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.341 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.113853+0000", 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00075449399999999998 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.113990+0000", 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013737100000000001 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:27:23.342 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114457+0000", 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046651899999999998 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.120048+0000", 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055907320000000002 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.123113+0000", 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030649309999999999 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.343 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.127714+0000", 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046014339999999997 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.127772+0000", 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8248999999999997e-05 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.344 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.128059+0000", 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00028627200000000001 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46073.0:134419 10.10 e480/445 ondisk, result = 0)", 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:41.020923+0000", 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 42.179311644000002, 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.093658136000000003, 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.345 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.020923+0000", 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.020923+0000", 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.346 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.020923+0000", 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.020925+0000", 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2539999999999999e-06 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.347 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.113940+0000", 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.093015020000000004 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.113946+0000", 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1070000000000001e-06 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.348 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114375+0000", 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00042913000000000001 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114383+0000", 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0790000000000001e-06 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.114581+0000", 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00019771 2023-10-09T04:27:23.349 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "MOSDPGRecoveryDeleteReply(10.10 e480,445 10:094604b5:::benchmark_data_smithi163_523108_object5899:head,424'4956)", 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:41.909580+0000", 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 41.290654941, 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.16960762200000001, 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:23.350 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909580+0000", 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909580+0000", 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.351 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909579+0000", 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909584+0000", 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0719999999999999e-06 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909593+0000", 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4630000000000007e-06 2023-10-09T04:27:23.352 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909599+0000", 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5400000000000003e-06 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:41.909628+0000", 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8736000000000001e-05 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:23.353 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:42.079187+0000", 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.169559395 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:23.354 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:27:23.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:23.387 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:23.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:23.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:23.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:23.572 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[221067]: failed to encode map e510 with expected crc 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[221067]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[213841]: failed to encode map e510 with expected crc 2023-10-09T04:27:23.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:27:23.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:23 smithi163 ceph-mon[213841]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T04:27:23.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:23.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:23 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:23 smithi181 ceph-mon[196979]: failed to encode map e510 with expected crc 2023-10-09T04:27:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:27:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:23 smithi181 ceph-mon[196979]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T04:27:24.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:24.102 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:24.286 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:27:25.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:24 smithi181 ceph-mon[196979]: pgmap v1320: 119 pgs: 1 creating+peering, 118 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:25.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:24 smithi163 ceph-mon[221067]: pgmap v1320: 119 pgs: 1 creating+peering, 118 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:25.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:24 smithi163 ceph-mon[213841]: pgmap v1320: 119 pgs: 1 creating+peering, 118 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:25.151 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:25.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:25.255 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:25.283 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:25.383 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:25.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:25.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:25.488 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:25.506 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:25.519 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:25.568 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:25.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:25.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:25.704 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:25.750 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:25.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:25.855 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:25.882 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:25.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:25.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:25.940 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:27:25.974 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:27:25.994 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:27:26.016 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:25 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/332650780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:26.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:27:26.017 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:25 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/332650780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:26.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:26.036 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:27:26.056 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:27:26.057 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:26.079 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:27:26.101 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:26.101 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:27:26.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.102 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.103 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.103 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.103 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:27:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:26.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:26.207 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:26.231 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:25 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/332650780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:26.233 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:26.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:26.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:26.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:26.383 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:26.413 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:26.984 INFO:teuthology.orchestra.run.smithi163.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T04:27:26.984 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:27:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:26 smithi181 ceph-mon[196979]: pgmap v1321: 119 pgs: 119 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3764880399' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:26 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3106583466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:27:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:26 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:27:27.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[221067]: pgmap v1321: 119 pgs: 119 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:27.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3764880399' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:27.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:27.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3106583466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:27:27.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:27:27.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[213841]: pgmap v1321: 119 pgs: 119 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:27.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3764880399' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:27.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:27:27.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3106583466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:27:27.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:26 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:27:27.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:27:27.464 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:27.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:27.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:27.600 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:27.649 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:27.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:27.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:27.758 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:27.786 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[221067]: failed to encode map e511 with expected crc 2023-10-09T04:27:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[221067]: failed to encode map e511 with expected crc 2023-10-09T04:27:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:27:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:27:28.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[221067]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T04:27:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[213841]: failed to encode map e511 with expected crc 2023-10-09T04:27:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[213841]: failed to encode map e511 with expected crc 2023-10-09T04:27:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:27:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:27:28.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:27 smithi163 ceph-mon[213841]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T04:27:28.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:28.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:28.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:27 smithi181 ceph-mon[196979]: failed to encode map e511 with expected crc 2023-10-09T04:27:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:27 smithi181 ceph-mon[196979]: failed to encode map e511 with expected crc 2023-10-09T04:27:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:27:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:27:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:27 smithi181 ceph-mon[196979]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T04:27:28.295 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:27:28.295 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:27:28.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:28.368 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:28.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:28.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:28.456 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:27:28.456 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:27:28.484 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:28.523 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:28.552 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:28.649 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:28.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:28.754 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:28.780 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:29.025 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:29.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:29.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:29.157 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:29.190 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:27:29.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:29.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:29.233 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:28 smithi163 ceph-mon[221067]: pgmap v1323: 119 pgs: 119 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:29.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:28 smithi163 ceph-mon[213841]: pgmap v1323: 119 pgs: 119 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:28 smithi181 ceph-mon[196979]: pgmap v1323: 119 pgs: 119 active+clean; 599 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:29.280 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:27:29.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:29.341 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:29.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:29.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:29.407 DEBUG:teuthology.orchestra.run.smithi163:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T04:27:29.409 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:27:29.409 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:27:29.467 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:hints = 1 2023-10-09T04:27:29.467 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:27:29.468 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object prefix: benchmark_data_smithi163_601242 2023-10-09T04:27:29.468 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:27:29.468 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:27:29.492 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:29.541 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:29.617 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:29.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:29.726 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:29.777 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:29.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:29.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:29.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:29.975 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:30.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:30.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:29 smithi163 ceph-mon[213841]: failed to encode map e511 with expected crc 2023-10-09T04:27:30.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:29 smithi163 ceph-mon[221067]: failed to encode map e511 with expected crc 2023-10-09T04:27:30.110 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:30.226 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:30.227 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:30.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:29 smithi181 ceph-mon[196979]: failed to encode map e511 with expected crc 2023-10-09T04:27:30.318 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:30.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:30.424 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:30.467 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 1 16 3229 3213 200.795 200.812 0.00742481 0.00495684 2023-10-09T04:27:30.479 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:30.854 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:30.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:30.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:30.963 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:31.019 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:31.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:27:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:27:30] "GET /metrics HTTP/1.1" 200 34205 "" "Prometheus/2.43.0" 2023-10-09T04:27:31.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:31.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:31.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:31.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:31 smithi181 ceph-mon[196979]: pgmap v1324: 119 pgs: 119 active+clean; 599 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:31.276 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:31.468 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 2 16 6236 6220 194.357 187.938 0.00327853 0.00513307 2023-10-09T04:27:31.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:31 smithi163 ceph-mon[213841]: pgmap v1324: 119 pgs: 119 active+clean; 599 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:31.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:31.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:31.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:31 smithi163 ceph-mon[221067]: pgmap v1324: 119 pgs: 119 active+clean; 599 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:27:31.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:31.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:32.144 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:27:32.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:32 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2082785776' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:32.401 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:32.401 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:32 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2082785776' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:32.401 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:32.469 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 3 16 9352 9336 194.421 194.75 0.00437084 0.00513636 2023-10-09T04:27:32.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:32 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2082785776' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:32.728 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:32.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:32.834 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:32.903 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:33.445 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[221067]: pgmap v1325: 119 pgs: 119 active+clean; 249 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[221067]: failed to encode map e512 with expected crc 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[221067]: failed to encode map e512 with expected crc 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[221067]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T04:27:33.450 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[213841]: pgmap v1325: 119 pgs: 119 active+clean; 249 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s 2023-10-09T04:27:33.451 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:33.451 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[213841]: failed to encode map e512 with expected crc 2023-10-09T04:27:33.451 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[213841]: failed to encode map e512 with expected crc 2023-10-09T04:27:33.451 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:33.451 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:33 smithi163 ceph-mon[213841]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T04:27:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:33.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:33.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 4 16 11883 11867 185.313 158.188 0.0035092 0.00538962 2023-10-09T04:27:33.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:33 smithi181 ceph-mon[196979]: pgmap v1325: 119 pgs: 119 active+clean; 249 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 27 MiB/s wr, 427 op/s 2023-10-09T04:27:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:33 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:33 smithi181 ceph-mon[196979]: failed to encode map e512 with expected crc 2023-10-09T04:27:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:33 smithi181 ceph-mon[196979]: failed to encode map e512 with expected crc 2023-10-09T04:27:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:33 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:33 smithi181 ceph-mon[196979]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T04:27:33.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:33.607 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:33.894 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:33.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:33.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:33.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:34.054 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:34.174 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:34.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:34.282 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:34.332 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:34.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 5 16 14002 13986 174.74 132.438 0.00313946 0.00571308 2023-10-09T04:27:35.048 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:35.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:35.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:35.155 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:35.207 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:35 smithi163 ceph-mon[213841]: pgmap v1327: 119 pgs: 119 active+clean; 399 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-10-09T04:27:35.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:35.208 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:35.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:35 smithi163 ceph-mon[221067]: pgmap v1327: 119 pgs: 119 active+clean; 399 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-10-09T04:27:35.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:35.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:35.209 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:35.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 6 16 16058 16042 167.034 128.5 0.00163423 0.00593619 2023-10-09T04:27:35.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:35 smithi181 ceph-mon[196979]: pgmap v1327: 119 pgs: 119 active+clean; 399 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-10-09T04:27:35.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:35.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:36.092 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:36.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:36.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:36.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:36.283 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:36.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:36.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:36.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:36.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 7 16 17803 17787 158.752 109.062 0.00564675 0.00629402 2023-10-09T04:27:36.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:36.553 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:36.663 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:36.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:36.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:36.731 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:27:36.757 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:27:36.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:36.817 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:36.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:36.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:36.833 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:36 smithi163 ceph-mon[213841]: pgmap v1328: 119 pgs: 119 active+clean; 449 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T04:27:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:36.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:36 smithi163 ceph-mon[221067]: pgmap v1328: 119 pgs: 119 active+clean; 449 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T04:27:36.881 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:36.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:36.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:36.957 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:27:36.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:36.993 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:27:37.005 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:27:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:37.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:36 smithi181 ceph-mon[196979]: pgmap v1328: 119 pgs: 119 active+clean; 449 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T04:27:37.045 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:27:37.084 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:27:37.088 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:37.129 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:27:37.168 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:27:37.214 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:27:37.215 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:27:37.215 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:27:37.215 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:27:37.215 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:27:37.215 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:27:37.222 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:37.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:37.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:37.257 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:27:37.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.266 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:27:37.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:37.419 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:37.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 8 16 19571 19555 152.72 110.5 0.00392934 0.00653884 2023-10-09T04:27:37.658 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:37 smithi163 ceph-mon[221067]: from='client.56374 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:37.659 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/947048034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:37.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:37 smithi163 ceph-mon[213841]: from='client.56374 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:37.659 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/947048034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:38.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:37 smithi181 ceph-mon[196979]: from='client.56374 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:38.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/947048034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:38.472 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 9 16 21632 21616 150.063 128.812 0.00668618 0.0066591 2023-10-09T04:27:38.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:38 smithi163 ceph-mon[213841]: pgmap v1329: 119 pgs: 119 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:27:38.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:38 smithi163 ceph-mon[221067]: pgmap v1329: 119 pgs: 119 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:27:39.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:38 smithi181 ceph-mon[196979]: pgmap v1329: 119 pgs: 119 active+clean; 1.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:27:39.120 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1328,"stamp":"2023-10-09T04:27:36.003849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":470506882,"num_objects":7194,"num_object_clones":0,"num_object_copies":21582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7194,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7238,"num_write_kb":458982,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":1901380,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12216,"ondisk_log_size":12216,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8455652,"kb_used_data":1899812,"kb_used_omap":126,"kb_used_meta":6555649,"kb_avail":741505564,"statfs":{"total":767960285184,"available":759301697536,"internally_reserved":0,"allocated":1945407488,"data_stored":1930217416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6712985211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":469893120,"num_objects":7170,"num_object_clones":0,"num_object_copies":21510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7170,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.027452"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":683,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724126+0000","last_change":"2023-10-09T04:26:24.679564+0000","last_active":"2023-10-09T04:27:22.724126+0000","last_peered":"2023-10-09T04:27:22.724126+0000","last_clean":"2023-10-09T04:27:22.724126+0000","last_became_active":"2023-10-09T04:26:24.679142+0000","last_became_peered":"2023-10-09T04:26:24.679142+0000","last_unstale":"2023-10-09T04:27:22.724126+0000","last_undegraded":"2023-10-09T04:27:22.724126+0000","last_fullsized":"2023-10-09T04:27:22.724126+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:37.095828+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":97,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200721+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:27:28.200721+0000","last_peered":"2023-10-09T04:27:28.200721+0000","last_clean":"2023-10-09T04:27:28.200721+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:27:28.200721+0000","last_undegraded":"2023-10-09T04:27:28.200721+0000","last_fullsized":"2023-10-09T04:27:28.200721+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201242+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:27:28.201242+0000","last_peered":"2023-10-09T04:27:28.201242+0000","last_clean":"2023-10-09T04:27:28.201242+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:27:28.201242+0000","last_undegraded":"2023-10-09T04:27:28.201242+0000","last_fullsized":"2023-10-09T04:27:28.201242+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":708,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726348+0000","last_change":"2023-10-09T04:24:58.734627+0000","last_active":"2023-10-09T04:27:22.726348+0000","last_peered":"2023-10-09T04:27:22.726348+0000","last_clean":"2023-10-09T04:27:22.726348+0000","last_became_active":"2023-10-09T04:24:58.734248+0000","last_became_peered":"2023-10-09T04:24:58.734248+0000","last_unstale":"2023-10-09T04:27:22.726348+0000","last_undegraded":"2023-10-09T04:27:22.726348+0000","last_fullsized":"2023-10-09T04:27:22.726348+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:59.934563+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":519,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176469+0000","last_change":"2023-10-09T04:24:56.158228+0000","last_active":"2023-10-09T04:27:28.176469+0000","last_peered":"2023-10-09T04:27:28.176469+0000","last_clean":"2023-10-09T04:27:28.176469+0000","last_became_active":"2023-10-09T04:24:56.157946+0000","last_became_peered":"2023-10-09T04:24:56.157946+0000","last_unstale":"2023-10-09T04:27:28.176469+0000","last_undegraded":"2023-10-09T04:27:28.176469+0000","last_fullsized":"2023-10-09T04:27:28.176469+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:37.279836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":116,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724779+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:27:22.724779+0000","last_peered":"2023-10-09T04:27:22.724779+0000","last_clean":"2023-10-09T04:27:22.724779+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:27:22.724779+0000","last_undegraded":"2023-10-09T04:27:22.724779+0000","last_fullsized":"2023-10-09T04:27:22.724779+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":504,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200681+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:27:28.200681+0000","last_peered":"2023-10-09T04:27:28.200681+0000","last_clean":"2023-10-09T04:27:28.200681+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:27:28.200681+0000","last_undegraded":"2023-10-09T04:27:28.200681+0000","last_fullsized":"2023-10-09T04:27:28.200681+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":115,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.723937+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:27:22.723937+0000","last_peered":"2023-10-09T04:27:22.723937+0000","last_clean":"2023-10-09T04:27:22.723937+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:27:22.723937+0000","last_undegraded":"2023-10-09T04:27:22.723937+0000","last_fullsized":"2023-10-09T04:27:22.723937+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":474,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724683+0000","last_change":"2023-10-09T04:26:09.747631+0000","last_active":"2023-10-09T04:27:22.724683+0000","last_peered":"2023-10-09T04:27:22.724683+0000","last_clean":"2023-10-09T04:27:22.724683+0000","last_became_active":"2023-10-09T04:26:09.747326+0000","last_became_peered":"2023-10-09T04:26:09.747326+0000","last_unstale":"2023-10-09T04:27:22.724683+0000","last_undegraded":"2023-10-09T04:27:22.724683+0000","last_fullsized":"2023-10-09T04:27:22.724683+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.315346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":910,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726414+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:27:22.726414+0000","last_peered":"2023-10-09T04:27:22.726414+0000","last_clean":"2023-10-09T04:27:22.726414+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:27:22.726414+0000","last_undegraded":"2023-10-09T04:27:22.726414+0000","last_fullsized":"2023-10-09T04:27:22.726414+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":115,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724459+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:27:22.724459+0000","last_peered":"2023-10-09T04:27:22.724459+0000","last_clean":"2023-10-09T04:27:22.724459+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:27:22.724459+0000","last_undegraded":"2023-10-09T04:27:22.724459+0000","last_fullsized":"2023-10-09T04:27:22.724459+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.139772+0000","last_change":"2023-10-09T04:26:09.741543+0000","last_active":"2023-10-09T04:27:32.139772+0000","last_peered":"2023-10-09T04:27:32.139772+0000","last_clean":"2023-10-09T04:27:32.139772+0000","last_became_active":"2023-10-09T04:26:09.741193+0000","last_became_peered":"2023-10-09T04:26:09.741193+0000","last_unstale":"2023-10-09T04:27:32.139772+0000","last_undegraded":"2023-10-09T04:27:32.139772+0000","last_fullsized":"2023-10-09T04:27:32.139772+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:43.560332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":807,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726973+0000","last_change":"2023-10-09T04:26:30.562322+0000","last_active":"2023-10-09T04:27:22.726973+0000","last_peered":"2023-10-09T04:27:22.726973+0000","last_clean":"2023-10-09T04:27:22.726973+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:27:22.726973+0000","last_undegraded":"2023-10-09T04:27:22.726973+0000","last_fullsized":"2023-10-09T04:27:22.726973+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":470,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725171+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:27:22.725171+0000","last_peered":"2023-10-09T04:27:22.725171+0000","last_clean":"2023-10-09T04:27:22.725171+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:27:22.725171+0000","last_undegraded":"2023-10-09T04:27:22.725171+0000","last_fullsized":"2023-10-09T04:27:22.725171+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176460+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:27:28.176460+0000","last_peered":"2023-10-09T04:27:28.176460+0000","last_clean":"2023-10-09T04:27:28.176460+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:27:28.176460+0000","last_undegraded":"2023-10-09T04:27:28.176460+0000","last_fullsized":"2023-10-09T04:27:28.176460+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":715,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724525+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:27:22.724525+0000","last_peered":"2023-10-09T04:27:22.724525+0000","last_clean":"2023-10-09T04:27:22.724525+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:27:22.724525+0000","last_undegraded":"2023-10-09T04:27:22.724525+0000","last_fullsized":"2023-10-09T04:27:22.724525+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":585,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726762+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:27:22.726762+0000","last_peered":"2023-10-09T04:27:22.726762+0000","last_clean":"2023-10-09T04:27:22.726762+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:27:22.726762+0000","last_undegraded":"2023-10-09T04:27:22.726762+0000","last_fullsized":"2023-10-09T04:27:22.726762+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":158,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983491+0000","last_change":"2023-10-09T04:26:21.351148+0000","last_active":"2023-10-09T04:27:26.983491+0000","last_peered":"2023-10-09T04:27:26.983491+0000","last_clean":"2023-10-09T04:27:26.983491+0000","last_became_active":"2023-10-09T04:26:21.349344+0000","last_became_peered":"2023-10-09T04:26:21.349344+0000","last_unstale":"2023-10-09T04:27:26.983491+0000","last_undegraded":"2023-10-09T04:27:26.983491+0000","last_fullsized":"2023-10-09T04:27:26.983491+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:41.495268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":690,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:29.080555+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:27:29.080555+0000","last_peered":"2023-10-09T04:27:29.080555+0000","last_clean":"2023-10-09T04:27:29.080555+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:27:29.080555+0000","last_undegraded":"2023-10-09T04:27:29.080555+0000","last_fullsized":"2023-10-09T04:27:29.080555+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":87,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201006+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:27:28.201006+0000","last_peered":"2023-10-09T04:27:28.201006+0000","last_clean":"2023-10-09T04:27:28.201006+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:27:28.201006+0000","last_undegraded":"2023-10-09T04:27:28.201006+0000","last_fullsized":"2023-10-09T04:27:28.201006+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201011+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:27:28.201011+0000","last_peered":"2023-10-09T04:27:28.201011+0000","last_clean":"2023-10-09T04:27:28.201011+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:27:28.201011+0000","last_undegraded":"2023-10-09T04:27:28.201011+0000","last_fullsized":"2023-10-09T04:27:28.201011+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":542,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724303+0000","last_change":"2023-10-09T04:27:02.906226+0000","last_active":"2023-10-09T04:27:22.724303+0000","last_peered":"2023-10-09T04:27:22.724303+0000","last_clean":"2023-10-09T04:27:22.724303+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:27:22.724303+0000","last_undegraded":"2023-10-09T04:27:22.724303+0000","last_fullsized":"2023-10-09T04:27:22.724303+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:21.516499+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":554,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725508+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:27:22.725508+0000","last_peered":"2023-10-09T04:27:22.725508+0000","last_clean":"2023-10-09T04:27:22.725508+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:27:22.725508+0000","last_undegraded":"2023-10-09T04:27:22.725508+0000","last_fullsized":"2023-10-09T04:27:22.725508+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":286,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726849+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:27:22.726849+0000","last_peered":"2023-10-09T04:27:22.726849+0000","last_clean":"2023-10-09T04:27:22.726849+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:27:22.726849+0000","last_undegraded":"2023-10-09T04:27:22.726849+0000","last_fullsized":"2023-10-09T04:27:22.726849+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":675,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724126+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:27:22.724126+0000","last_peered":"2023-10-09T04:27:22.724126+0000","last_clean":"2023-10-09T04:27:22.724126+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:27:22.724126+0000","last_undegraded":"2023-10-09T04:27:22.724126+0000","last_fullsized":"2023-10-09T04:27:22.724126+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":115,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.847851+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:27:22.847851+0000","last_peered":"2023-10-09T04:27:22.847851+0000","last_clean":"2023-10-09T04:27:22.847851+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:27:22.847851+0000","last_undegraded":"2023-10-09T04:27:22.847851+0000","last_fullsized":"2023-10-09T04:27:22.847851+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":489,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724948+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:27:22.724948+0000","last_peered":"2023-10-09T04:27:22.724948+0000","last_clean":"2023-10-09T04:27:22.724948+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:27:22.724948+0000","last_undegraded":"2023-10-09T04:27:22.724948+0000","last_fullsized":"2023-10-09T04:27:22.724948+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":657,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176277+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:27:28.176277+0000","last_peered":"2023-10-09T04:27:28.176277+0000","last_clean":"2023-10-09T04:27:28.176277+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:27:28.176277+0000","last_undegraded":"2023-10-09T04:27:28.176277+0000","last_fullsized":"2023-10-09T04:27:28.176277+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200477+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:27:28.200477+0000","last_peered":"2023-10-09T04:27:28.200477+0000","last_clean":"2023-10-09T04:27:28.200477+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:27:28.200477+0000","last_undegraded":"2023-10-09T04:27:28.200477+0000","last_fullsized":"2023-10-09T04:27:28.200477+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":115,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724743+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:27:22.724743+0000","last_peered":"2023-10-09T04:27:22.724743+0000","last_clean":"2023-10-09T04:27:22.724743+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:27:22.724743+0000","last_undegraded":"2023-10-09T04:27:22.724743+0000","last_fullsized":"2023-10-09T04:27:22.724743+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":675,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725819+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:27:22.725819+0000","last_peered":"2023-10-09T04:27:22.725819+0000","last_clean":"2023-10-09T04:27:22.725819+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:27:22.725819+0000","last_undegraded":"2023-10-09T04:27:22.725819+0000","last_fullsized":"2023-10-09T04:27:22.725819+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":125,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724034+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:27:22.724034+0000","last_peered":"2023-10-09T04:27:22.724034+0000","last_clean":"2023-10-09T04:27:22.724034+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:27:22.724034+0000","last_undegraded":"2023-10-09T04:27:22.724034+0000","last_fullsized":"2023-10-09T04:27:22.724034+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":177,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176758+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:27:28.176758+0000","last_peered":"2023-10-09T04:27:28.176758+0000","last_clean":"2023-10-09T04:27:28.176758+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:27:28.176758+0000","last_undegraded":"2023-10-09T04:27:28.176758+0000","last_fullsized":"2023-10-09T04:27:28.176758+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"511'295","reported_seq":307,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.028970+0000","last_change":"2023-10-09T04:27:20.720040+0000","last_active":"2023-10-09T04:27:31.028970+0000","last_peered":"2023-10-09T04:27:31.028970+0000","last_clean":"2023-10-09T04:27:31.028970+0000","last_became_active":"2023-10-09T04:27:20.719715+0000","last_became_peered":"2023-10-09T04:27:20.719715+0000","last_unstale":"2023-10-09T04:27:31.028970+0000","last_undegraded":"2023-10-09T04:27:31.028970+0000","last_fullsized":"2023-10-09T04:27:31.028970+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":295,"log_dups_size":0,"ondisk_log_size":295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:37.979204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":295,"num_write_kb":18880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":953,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726535+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:27:22.726535+0000","last_peered":"2023-10-09T04:27:22.726535+0000","last_clean":"2023-10-09T04:27:22.726535+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:27:22.726535+0000","last_undegraded":"2023-10-09T04:27:22.726535+0000","last_fullsized":"2023-10-09T04:27:22.726535+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":555,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724851+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:27:22.724851+0000","last_peered":"2023-10-09T04:27:22.724851+0000","last_clean":"2023-10-09T04:27:22.724851+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:27:22.724851+0000","last_undegraded":"2023-10-09T04:27:22.724851+0000","last_fullsized":"2023-10-09T04:27:22.724851+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":472,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724029+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:27:22.724029+0000","last_peered":"2023-10-09T04:27:22.724029+0000","last_clean":"2023-10-09T04:27:22.724029+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:27:22.724029+0000","last_undegraded":"2023-10-09T04:27:22.724029+0000","last_fullsized":"2023-10-09T04:27:22.724029+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"511'411","reported_seq":423,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.573962+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:27:31.573962+0000","last_peered":"2023-10-09T04:27:31.573962+0000","last_clean":"2023-10-09T04:27:31.573962+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:27:31.573962+0000","last_undegraded":"2023-10-09T04:27:31.573962+0000","last_fullsized":"2023-10-09T04:27:31.573962+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":411,"log_dups_size":0,"ondisk_log_size":411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26935296,"num_objects":411,"num_object_clones":0,"num_object_copies":1233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":411,"num_write_kb":26304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":745,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176530+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:27:28.176530+0000","last_peered":"2023-10-09T04:27:28.176530+0000","last_clean":"2023-10-09T04:27:28.176530+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:27:28.176530+0000","last_undegraded":"2023-10-09T04:27:28.176530+0000","last_fullsized":"2023-10-09T04:27:28.176530+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":531,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726076+0000","last_change":"2023-10-09T04:24:56.212579+0000","last_active":"2023-10-09T04:27:22.726076+0000","last_peered":"2023-10-09T04:27:22.726076+0000","last_clean":"2023-10-09T04:27:22.726076+0000","last_became_active":"2023-10-09T04:24:56.184403+0000","last_became_peered":"2023-10-09T04:24:56.184403+0000","last_unstale":"2023-10-09T04:27:22.726076+0000","last_undegraded":"2023-10-09T04:27:22.726076+0000","last_fullsized":"2023-10-09T04:27:22.726076+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:10.486526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200537+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:27:28.200537+0000","last_peered":"2023-10-09T04:27:28.200537+0000","last_clean":"2023-10-09T04:27:28.200537+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:27:28.200537+0000","last_undegraded":"2023-10-09T04:27:28.200537+0000","last_fullsized":"2023-10-09T04:27:28.200537+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"511'311","reported_seq":323,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.041664+0000","last_change":"2023-10-09T04:27:20.720676+0000","last_active":"2023-10-09T04:27:31.041664+0000","last_peered":"2023-10-09T04:27:31.041664+0000","last_clean":"2023-10-09T04:27:31.041664+0000","last_became_active":"2023-10-09T04:27:20.720342+0000","last_became_peered":"2023-10-09T04:27:20.720342+0000","last_unstale":"2023-10-09T04:27:31.041664+0000","last_undegraded":"2023-10-09T04:27:31.041664+0000","last_fullsized":"2023-10-09T04:27:31.041664+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":311,"log_dups_size":0,"ondisk_log_size":311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:37.485070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20381696,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":311,"num_write_kb":19904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":715,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724421+0000","last_change":"2023-10-09T04:24:58.648512+0000","last_active":"2023-10-09T04:27:22.724421+0000","last_peered":"2023-10-09T04:27:22.724421+0000","last_clean":"2023-10-09T04:27:22.724421+0000","last_became_active":"2023-10-09T04:24:58.648143+0000","last_became_peered":"2023-10-09T04:24:58.648143+0000","last_unstale":"2023-10-09T04:27:22.724421+0000","last_undegraded":"2023-10-09T04:27:22.724421+0000","last_fullsized":"2023-10-09T04:27:22.724421+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:26.096956+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":482,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726638+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:27:22.726638+0000","last_peered":"2023-10-09T04:27:22.726638+0000","last_clean":"2023-10-09T04:27:22.726638+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:27:22.726638+0000","last_undegraded":"2023-10-09T04:27:22.726638+0000","last_fullsized":"2023-10-09T04:27:22.726638+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":138,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983367+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:27:26.983367+0000","last_peered":"2023-10-09T04:27:26.983367+0000","last_clean":"2023-10-09T04:27:26.983367+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:27:26.983367+0000","last_undegraded":"2023-10-09T04:27:26.983367+0000","last_fullsized":"2023-10-09T04:27:26.983367+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"511'309","reported_seq":320,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.026425+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:27:31.026425+0000","last_peered":"2023-10-09T04:27:31.026425+0000","last_clean":"2023-10-09T04:27:31.026425+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:27:31.026425+0000","last_undegraded":"2023-10-09T04:27:31.026425+0000","last_fullsized":"2023-10-09T04:27:31.026425+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":309,"log_dups_size":0,"ondisk_log_size":309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20250624,"num_objects":309,"num_object_clones":0,"num_object_copies":927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":309,"num_write_kb":19776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":768,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201366+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:27:28.201366+0000","last_peered":"2023-10-09T04:27:28.201366+0000","last_clean":"2023-10-09T04:27:28.201366+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:27:28.201366+0000","last_undegraded":"2023-10-09T04:27:28.201366+0000","last_fullsized":"2023-10-09T04:27:28.201366+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":510,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724475+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:27:22.724475+0000","last_peered":"2023-10-09T04:27:22.724475+0000","last_clean":"2023-10-09T04:27:22.724475+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:27:22.724475+0000","last_undegraded":"2023-10-09T04:27:22.724475+0000","last_fullsized":"2023-10-09T04:27:22.724475+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":133,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724341+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:27:22.724341+0000","last_peered":"2023-10-09T04:27:22.724341+0000","last_clean":"2023-10-09T04:27:22.724341+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:27:22.724341+0000","last_undegraded":"2023-10-09T04:27:22.724341+0000","last_fullsized":"2023-10-09T04:27:22.724341+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"511'302","reported_seq":314,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.040201+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:27:31.040201+0000","last_peered":"2023-10-09T04:27:31.040201+0000","last_clean":"2023-10-09T04:27:31.040201+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:27:31.040201+0000","last_undegraded":"2023-10-09T04:27:31.040201+0000","last_fullsized":"2023-10-09T04:27:31.040201+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":302,"log_dups_size":0,"ondisk_log_size":302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19791872,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":302,"num_write_kb":19328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":974,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726613+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:27:22.726613+0000","last_peered":"2023-10-09T04:27:22.726613+0000","last_clean":"2023-10-09T04:27:22.726613+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:27:22.726613+0000","last_undegraded":"2023-10-09T04:27:22.726613+0000","last_fullsized":"2023-10-09T04:27:22.726613+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":145,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724240+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:27:22.724240+0000","last_peered":"2023-10-09T04:27:22.724240+0000","last_clean":"2023-10-09T04:27:22.724240+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:27:22.724240+0000","last_undegraded":"2023-10-09T04:27:22.724240+0000","last_fullsized":"2023-10-09T04:27:22.724240+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200994+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:27:28.200994+0000","last_peered":"2023-10-09T04:27:28.200994+0000","last_clean":"2023-10-09T04:27:28.200994+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:27:28.200994+0000","last_undegraded":"2023-10-09T04:27:28.200994+0000","last_fullsized":"2023-10-09T04:27:28.200994+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"511'445","reported_seq":458,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.563778+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:27:31.563778+0000","last_peered":"2023-10-09T04:27:31.563778+0000","last_clean":"2023-10-09T04:27:31.563778+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:27:31.563778+0000","last_undegraded":"2023-10-09T04:27:31.563778+0000","last_fullsized":"2023-10-09T04:27:31.563778+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":445,"log_dups_size":0,"ondisk_log_size":445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29163520,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":445,"num_write_kb":28480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":720,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724230+0000","last_change":"2023-10-09T04:26:23.679008+0000","last_active":"2023-10-09T04:27:22.724230+0000","last_peered":"2023-10-09T04:27:22.724230+0000","last_clean":"2023-10-09T04:27:22.724230+0000","last_became_active":"2023-10-09T04:26:23.678629+0000","last_became_peered":"2023-10-09T04:26:23.678629+0000","last_unstale":"2023-10-09T04:27:22.724230+0000","last_undegraded":"2023-10-09T04:27:22.724230+0000","last_fullsized":"2023-10-09T04:27:22.724230+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:41.534992+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.199931+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:27:28.199931+0000","last_peered":"2023-10-09T04:27:28.199931+0000","last_clean":"2023-10-09T04:27:28.199931+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:27:28.199931+0000","last_undegraded":"2023-10-09T04:27:28.199931+0000","last_fullsized":"2023-10-09T04:27:28.199931+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":94,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201237+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:27:28.201237+0000","last_peered":"2023-10-09T04:27:28.201237+0000","last_clean":"2023-10-09T04:27:28.201237+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:27:28.201237+0000","last_undegraded":"2023-10-09T04:27:28.201237+0000","last_fullsized":"2023-10-09T04:27:28.201237+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"512'1032","reported_seq":1046,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:35.545212+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:27:35.545212+0000","last_peered":"2023-10-09T04:27:35.545212+0000","last_clean":"2023-10-09T04:27:35.545212+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:27:35.545212+0000","last_undegraded":"2023-10-09T04:27:35.545212+0000","last_fullsized":"2023-10-09T04:27:35.545212+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1032,"log_dups_size":0,"ondisk_log_size":1032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633152,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1032,"num_write_kb":66048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":616,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724429+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:27:22.724429+0000","last_peered":"2023-10-09T04:27:22.724429+0000","last_clean":"2023-10-09T04:27:22.724429+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:27:22.724429+0000","last_undegraded":"2023-10-09T04:27:22.724429+0000","last_fullsized":"2023-10-09T04:27:22.724429+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200599+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:27:28.200599+0000","last_peered":"2023-10-09T04:27:28.200599+0000","last_clean":"2023-10-09T04:27:28.200599+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:27:28.200599+0000","last_undegraded":"2023-10-09T04:27:28.200599+0000","last_fullsized":"2023-10-09T04:27:28.200599+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200046+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:27:28.200046+0000","last_peered":"2023-10-09T04:27:28.200046+0000","last_clean":"2023-10-09T04:27:28.200046+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:27:28.200046+0000","last_undegraded":"2023-10-09T04:27:28.200046+0000","last_fullsized":"2023-10-09T04:27:28.200046+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"512'764","reported_seq":778,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:33.754538+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:27:33.754538+0000","last_peered":"2023-10-09T04:27:33.754538+0000","last_clean":"2023-10-09T04:27:33.754538+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:27:33.754538+0000","last_undegraded":"2023-10-09T04:27:33.754538+0000","last_fullsized":"2023-10-09T04:27:33.754538+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":764,"log_dups_size":0,"ondisk_log_size":764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50069504,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":764,"num_write_kb":48896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":715,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725304+0000","last_change":"2023-10-09T04:26:23.680981+0000","last_active":"2023-10-09T04:27:22.725304+0000","last_peered":"2023-10-09T04:27:22.725304+0000","last_clean":"2023-10-09T04:27:22.725304+0000","last_became_active":"2023-10-09T04:26:23.680615+0000","last_became_peered":"2023-10-09T04:26:23.680615+0000","last_unstale":"2023-10-09T04:27:22.725304+0000","last_undegraded":"2023-10-09T04:27:22.725304+0000","last_fullsized":"2023-10-09T04:27:22.725304+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:36.804345+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":177,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176107+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:27:28.176107+0000","last_peered":"2023-10-09T04:27:28.176107+0000","last_clean":"2023-10-09T04:27:28.176107+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:27:28.176107+0000","last_undegraded":"2023-10-09T04:27:28.176107+0000","last_fullsized":"2023-10-09T04:27:28.176107+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":87,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201238+0000","last_change":"2023-10-09T04:26:09.746900+0000","last_active":"2023-10-09T04:27:28.201238+0000","last_peered":"2023-10-09T04:27:28.201238+0000","last_clean":"2023-10-09T04:27:28.201238+0000","last_became_active":"2023-10-09T04:26:09.746607+0000","last_became_peered":"2023-10-09T04:26:09.746607+0000","last_unstale":"2023-10-09T04:27:28.201238+0000","last_undegraded":"2023-10-09T04:27:28.201238+0000","last_fullsized":"2023-10-09T04:27:28.201238+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:07.695043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"512'533","reported_seq":546,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.145669+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:27:32.145669+0000","last_peered":"2023-10-09T04:27:32.145669+0000","last_clean":"2023-10-09T04:27:32.145669+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:27:32.145669+0000","last_undegraded":"2023-10-09T04:27:32.145669+0000","last_fullsized":"2023-10-09T04:27:32.145669+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":533,"log_dups_size":0,"ondisk_log_size":533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34930688,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":533,"num_write_kb":34112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":638,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983192+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:27:26.983192+0000","last_peered":"2023-10-09T04:27:26.983192+0000","last_clean":"2023-10-09T04:27:26.983192+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:27:26.983192+0000","last_undegraded":"2023-10-09T04:27:26.983192+0000","last_fullsized":"2023-10-09T04:27:26.983192+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":114,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724185+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:27:22.724185+0000","last_peered":"2023-10-09T04:27:22.724185+0000","last_clean":"2023-10-09T04:27:22.724185+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:27:22.724185+0000","last_undegraded":"2023-10-09T04:27:22.724185+0000","last_fullsized":"2023-10-09T04:27:22.724185+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":527,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724798+0000","last_change":"2023-10-09T04:26:21.351382+0000","last_active":"2023-10-09T04:27:22.724798+0000","last_peered":"2023-10-09T04:27:22.724798+0000","last_clean":"2023-10-09T04:27:22.724798+0000","last_became_active":"2023-10-09T04:26:21.348023+0000","last_became_peered":"2023-10-09T04:26:21.348023+0000","last_unstale":"2023-10-09T04:27:22.724798+0000","last_undegraded":"2023-10-09T04:27:22.724798+0000","last_fullsized":"2023-10-09T04:27:22.724798+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:45.177721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":390,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725707+0000","last_change":"2023-10-09T04:26:09.749151+0000","last_active":"2023-10-09T04:27:22.725707+0000","last_peered":"2023-10-09T04:27:22.725707+0000","last_clean":"2023-10-09T04:27:22.725707+0000","last_became_active":"2023-10-09T04:26:09.748857+0000","last_became_peered":"2023-10-09T04:26:09.748857+0000","last_unstale":"2023-10-09T04:27:22.725707+0000","last_undegraded":"2023-10-09T04:27:22.725707+0000","last_fullsized":"2023-10-09T04:27:22.725707+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:38.228161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"511'389","reported_seq":402,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.575799+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:27:31.575799+0000","last_peered":"2023-10-09T04:27:31.575799+0000","last_clean":"2023-10-09T04:27:31.575799+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:27:31.575799+0000","last_undegraded":"2023-10-09T04:27:31.575799+0000","last_fullsized":"2023-10-09T04:27:31.575799+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":389,"log_dups_size":0,"ondisk_log_size":389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25493504,"num_objects":389,"num_object_clones":0,"num_object_copies":1167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":389,"num_write_kb":24896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":718,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726038+0000","last_change":"2023-10-09T04:26:23.684553+0000","last_active":"2023-10-09T04:27:22.726038+0000","last_peered":"2023-10-09T04:27:22.726038+0000","last_clean":"2023-10-09T04:27:22.726038+0000","last_became_active":"2023-10-09T04:26:23.684125+0000","last_became_peered":"2023-10-09T04:26:23.684125+0000","last_unstale":"2023-10-09T04:27:22.726038+0000","last_undegraded":"2023-10-09T04:27:22.726038+0000","last_fullsized":"2023-10-09T04:27:22.726038+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:01.715214+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":494,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725406+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:27:22.725406+0000","last_peered":"2023-10-09T04:27:22.725406+0000","last_clean":"2023-10-09T04:27:22.725406+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:27:22.725406+0000","last_undegraded":"2023-10-09T04:27:22.725406+0000","last_fullsized":"2023-10-09T04:27:22.725406+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.199966+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:27:28.199966+0000","last_peered":"2023-10-09T04:27:28.199966+0000","last_clean":"2023-10-09T04:27:28.199966+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:27:28.199966+0000","last_undegraded":"2023-10-09T04:27:28.199966+0000","last_fullsized":"2023-10-09T04:27:28.199966+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":476,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:29.080477+0000","last_change":"2023-10-09T04:26:21.345296+0000","last_active":"2023-10-09T04:27:29.080477+0000","last_peered":"2023-10-09T04:27:29.080477+0000","last_clean":"2023-10-09T04:27:29.080477+0000","last_became_active":"2023-10-09T04:26:21.345003+0000","last_became_peered":"2023-10-09T04:26:21.345003+0000","last_unstale":"2023-10-09T04:27:29.080477+0000","last_undegraded":"2023-10-09T04:27:29.080477+0000","last_fullsized":"2023-10-09T04:27:29.080477+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:33.938068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"511'279","reported_seq":291,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.023922+0000","last_change":"2023-10-09T04:27:20.717622+0000","last_active":"2023-10-09T04:27:31.023922+0000","last_peered":"2023-10-09T04:27:31.023922+0000","last_clean":"2023-10-09T04:27:31.023922+0000","last_became_active":"2023-10-09T04:27:20.717291+0000","last_became_peered":"2023-10-09T04:27:20.717291+0000","last_unstale":"2023-10-09T04:27:31.023922+0000","last_undegraded":"2023-10-09T04:27:31.023922+0000","last_fullsized":"2023-10-09T04:27:31.023922+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":279,"log_dups_size":0,"ondisk_log_size":279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:34.329924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18284544,"num_objects":279,"num_object_clones":0,"num_object_copies":837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":279,"num_write_kb":17856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":685,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627353+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:27:28.627353+0000","last_peered":"2023-10-09T04:27:28.627353+0000","last_clean":"2023-10-09T04:27:28.627353+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:27:28.627353+0000","last_undegraded":"2023-10-09T04:27:28.627353+0000","last_fullsized":"2023-10-09T04:27:28.627353+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":361,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724131+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:27:22.724131+0000","last_peered":"2023-10-09T04:27:22.724131+0000","last_clean":"2023-10-09T04:27:22.724131+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:27:22.724131+0000","last_undegraded":"2023-10-09T04:27:22.724131+0000","last_fullsized":"2023-10-09T04:27:22.724131+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":451,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:29.080117+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:27:29.080117+0000","last_peered":"2023-10-09T04:27:29.080117+0000","last_clean":"2023-10-09T04:27:29.080117+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:27:29.080117+0000","last_undegraded":"2023-10-09T04:27:29.080117+0000","last_fullsized":"2023-10-09T04:27:29.080117+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":216,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724827+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:27:22.724827+0000","last_peered":"2023-10-09T04:27:22.724827+0000","last_clean":"2023-10-09T04:27:22.724827+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:27:22.724827+0000","last_undegraded":"2023-10-09T04:27:22.724827+0000","last_fullsized":"2023-10-09T04:27:22.724827+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"511'276","reported_seq":288,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.018838+0000","last_change":"2023-10-09T04:27:20.717505+0000","last_active":"2023-10-09T04:27:31.018838+0000","last_peered":"2023-10-09T04:27:31.018838+0000","last_clean":"2023-10-09T04:27:31.018838+0000","last_became_active":"2023-10-09T04:27:20.717202+0000","last_became_peered":"2023-10-09T04:27:20.717202+0000","last_unstale":"2023-10-09T04:27:31.018838+0000","last_undegraded":"2023-10-09T04:27:31.018838+0000","last_fullsized":"2023-10-09T04:27:31.018838+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":276,"log_dups_size":0,"ondisk_log_size":276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:51.034299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":276,"num_write_kb":17664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":514,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726241+0000","last_change":"2023-10-09T04:26:21.429887+0000","last_active":"2023-10-09T04:27:22.726241+0000","last_peered":"2023-10-09T04:27:22.726241+0000","last_clean":"2023-10-09T04:27:22.726241+0000","last_became_active":"2023-10-09T04:26:21.392380+0000","last_became_peered":"2023-10-09T04:26:21.392380+0000","last_unstale":"2023-10-09T04:27:22.726241+0000","last_undegraded":"2023-10-09T04:27:22.726241+0000","last_fullsized":"2023-10-09T04:27:22.726241+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:05:55.650685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":965,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.727266+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:27:22.727266+0000","last_peered":"2023-10-09T04:27:22.727266+0000","last_clean":"2023-10-09T04:27:22.727266+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:27:22.727266+0000","last_undegraded":"2023-10-09T04:27:22.727266+0000","last_fullsized":"2023-10-09T04:27:22.727266+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":447,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725613+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:27:22.725613+0000","last_peered":"2023-10-09T04:27:22.725613+0000","last_clean":"2023-10-09T04:27:22.725613+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:27:22.725613+0000","last_undegraded":"2023-10-09T04:27:22.725613+0000","last_fullsized":"2023-10-09T04:27:22.725613+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":74,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201124+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:27:28.201124+0000","last_peered":"2023-10-09T04:27:28.201124+0000","last_clean":"2023-10-09T04:27:28.201124+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:27:28.201124+0000","last_undegraded":"2023-10-09T04:27:28.201124+0000","last_fullsized":"2023-10-09T04:27:28.201124+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"511'400","reported_seq":413,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.571551+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:27:31.571551+0000","last_peered":"2023-10-09T04:27:31.571551+0000","last_clean":"2023-10-09T04:27:31.571551+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:27:31.571551+0000","last_undegraded":"2023-10-09T04:27:31.571551+0000","last_fullsized":"2023-10-09T04:27:31.571551+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":400,"log_dups_size":0,"ondisk_log_size":400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26214400,"num_objects":400,"num_object_clones":0,"num_object_copies":1200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":400,"num_write_kb":25600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":607,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201342+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:27:28.201342+0000","last_peered":"2023-10-09T04:27:28.201342+0000","last_clean":"2023-10-09T04:27:28.201342+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:27:28.201342+0000","last_undegraded":"2023-10-09T04:27:28.201342+0000","last_fullsized":"2023-10-09T04:27:28.201342+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":511,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.727164+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:27:22.727164+0000","last_peered":"2023-10-09T04:27:22.727164+0000","last_clean":"2023-10-09T04:27:22.727164+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:27:22.727164+0000","last_undegraded":"2023-10-09T04:27:22.727164+0000","last_fullsized":"2023-10-09T04:27:22.727164+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":152,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.847789+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:27:22.847789+0000","last_peered":"2023-10-09T04:27:22.847789+0000","last_clean":"2023-10-09T04:27:22.847789+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:27:22.847789+0000","last_undegraded":"2023-10-09T04:27:22.847789+0000","last_fullsized":"2023-10-09T04:27:22.847789+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":134,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.577048+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:27:26.577048+0000","last_peered":"2023-10-09T04:27:26.577048+0000","last_clean":"2023-10-09T04:27:26.577048+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:27:26.577048+0000","last_undegraded":"2023-10-09T04:27:26.577048+0000","last_fullsized":"2023-10-09T04:27:26.577048+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"512'559","reported_seq":572,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.152416+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:27:32.152416+0000","last_peered":"2023-10-09T04:27:32.152416+0000","last_clean":"2023-10-09T04:27:32.152416+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:27:32.152416+0000","last_undegraded":"2023-10-09T04:27:32.152416+0000","last_fullsized":"2023-10-09T04:27:32.152416+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":559,"log_dups_size":0,"ondisk_log_size":559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36634624,"num_objects":559,"num_object_clones":0,"num_object_copies":1677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":559,"num_write_kb":35776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":630,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983334+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:27:26.983334+0000","last_peered":"2023-10-09T04:27:26.983334+0000","last_clean":"2023-10-09T04:27:26.983334+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:27:26.983334+0000","last_undegraded":"2023-10-09T04:27:26.983334+0000","last_fullsized":"2023-10-09T04:27:26.983334+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":534,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726641+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:27:22.726641+0000","last_peered":"2023-10-09T04:27:22.726641+0000","last_clean":"2023-10-09T04:27:22.726641+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:27:22.726641+0000","last_undegraded":"2023-10-09T04:27:22.726641+0000","last_fullsized":"2023-10-09T04:27:22.726641+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":269,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724373+0000","last_change":"2023-10-09T04:26:21.355315+0000","last_active":"2023-10-09T04:27:22.724373+0000","last_peered":"2023-10-09T04:27:22.724373+0000","last_clean":"2023-10-09T04:27:22.724373+0000","last_became_active":"2023-10-09T04:26:21.355038+0000","last_became_peered":"2023-10-09T04:26:21.355038+0000","last_unstale":"2023-10-09T04:27:22.724373+0000","last_undegraded":"2023-10-09T04:27:22.724373+0000","last_fullsized":"2023-10-09T04:27:22.724373+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:45.819056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726147+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:27:22.726147+0000","last_peered":"2023-10-09T04:27:22.726147+0000","last_clean":"2023-10-09T04:27:22.726147+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:27:22.726147+0000","last_undegraded":"2023-10-09T04:27:22.726147+0000","last_fullsized":"2023-10-09T04:27:22.726147+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"511'318","reported_seq":330,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.009715+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:27:31.009715+0000","last_peered":"2023-10-09T04:27:31.009715+0000","last_clean":"2023-10-09T04:27:31.009715+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:27:31.009715+0000","last_undegraded":"2023-10-09T04:27:31.009715+0000","last_fullsized":"2023-10-09T04:27:31.009715+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":318,"log_dups_size":0,"ondisk_log_size":318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20840448,"num_objects":318,"num_object_clones":0,"num_object_copies":954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":318,"num_write_kb":20352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":569,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200098+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:27:28.200098+0000","last_peered":"2023-10-09T04:27:28.200098+0000","last_clean":"2023-10-09T04:27:28.200098+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:27:28.200098+0000","last_undegraded":"2023-10-09T04:27:28.200098+0000","last_fullsized":"2023-10-09T04:27:28.200098+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":471,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.725164+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:27:22.725164+0000","last_peered":"2023-10-09T04:27:22.725164+0000","last_clean":"2023-10-09T04:27:22.725164+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:27:22.725164+0000","last_undegraded":"2023-10-09T04:27:22.725164+0000","last_fullsized":"2023-10-09T04:27:22.725164+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200033+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:27:28.200033+0000","last_peered":"2023-10-09T04:27:28.200033+0000","last_clean":"2023-10-09T04:27:28.200033+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:27:28.200033+0000","last_undegraded":"2023-10-09T04:27:28.200033+0000","last_fullsized":"2023-10-09T04:27:28.200033+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":419,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.727511+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:27:22.727511+0000","last_peered":"2023-10-09T04:27:22.727511+0000","last_clean":"2023-10-09T04:27:22.727511+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:27:22.727511+0000","last_undegraded":"2023-10-09T04:27:22.727511+0000","last_fullsized":"2023-10-09T04:27:22.727511+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"512'547","reported_seq":560,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.165524+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:27:32.165524+0000","last_peered":"2023-10-09T04:27:32.165524+0000","last_clean":"2023-10-09T04:27:32.165524+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:27:32.165524+0000","last_undegraded":"2023-10-09T04:27:32.165524+0000","last_fullsized":"2023-10-09T04:27:32.165524+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":547,"log_dups_size":0,"ondisk_log_size":547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35848192,"num_objects":547,"num_object_clones":0,"num_object_copies":1641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":547,"num_write_kb":35008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":687,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176488+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:27:28.176488+0000","last_peered":"2023-10-09T04:27:28.176488+0000","last_clean":"2023-10-09T04:27:28.176488+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:27:28.176488+0000","last_undegraded":"2023-10-09T04:27:28.176488+0000","last_fullsized":"2023-10-09T04:27:28.176488+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":311,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983491+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:27:26.983491+0000","last_peered":"2023-10-09T04:27:26.983491+0000","last_clean":"2023-10-09T04:27:26.983491+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:27:26.983491+0000","last_undegraded":"2023-10-09T04:27:26.983491+0000","last_fullsized":"2023-10-09T04:27:26.983491+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":146,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.577077+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:27:26.577077+0000","last_peered":"2023-10-09T04:27:26.577077+0000","last_clean":"2023-10-09T04:27:26.577077+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:27:26.577077+0000","last_undegraded":"2023-10-09T04:27:26.577077+0000","last_fullsized":"2023-10-09T04:27:26.577077+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.199931+0000","last_change":"2023-10-09T04:24:56.169203+0000","last_active":"2023-10-09T04:27:28.199931+0000","last_peered":"2023-10-09T04:27:28.199931+0000","last_clean":"2023-10-09T04:27:28.199931+0000","last_became_active":"2023-10-09T04:24:56.168737+0000","last_became_peered":"2023-10-09T04:24:56.168737+0000","last_unstale":"2023-10-09T04:27:28.199931+0000","last_undegraded":"2023-10-09T04:27:28.199931+0000","last_fullsized":"2023-10-09T04:27:28.199931+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:21.515886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":553,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724217+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:27:22.724217+0000","last_peered":"2023-10-09T04:27:22.724217+0000","last_clean":"2023-10-09T04:27:22.724217+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:27:22.724217+0000","last_undegraded":"2023-10-09T04:27:22.724217+0000","last_fullsized":"2023-10-09T04:27:22.724217+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":133,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724648+0000","last_change":"2023-10-09T04:24:56.170052+0000","last_active":"2023-10-09T04:27:22.724648+0000","last_peered":"2023-10-09T04:27:22.724648+0000","last_clean":"2023-10-09T04:27:22.724648+0000","last_became_active":"2023-10-09T04:24:56.169607+0000","last_became_peered":"2023-10-09T04:24:56.169607+0000","last_unstale":"2023-10-09T04:27:22.724648+0000","last_undegraded":"2023-10-09T04:27:22.724648+0000","last_fullsized":"2023-10-09T04:27:22.724648+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.849923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":51,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.615097+0000","last_change":"2023-10-09T04:26:21.345230+0000","last_active":"2023-10-09T04:27:28.615097+0000","last_peered":"2023-10-09T04:27:28.615097+0000","last_clean":"2023-10-09T04:27:28.615097+0000","last_became_active":"2023-10-09T04:26:21.344939+0000","last_became_peered":"2023-10-09T04:26:21.344939+0000","last_unstale":"2023-10-09T04:27:28.615097+0000","last_undegraded":"2023-10-09T04:27:28.615097+0000","last_fullsized":"2023-10-09T04:27:28.615097+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.495703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":586,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983310+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:27:26.983310+0000","last_peered":"2023-10-09T04:27:26.983310+0000","last_clean":"2023-10-09T04:27:26.983310+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:27:26.983310+0000","last_undegraded":"2023-10-09T04:27:26.983310+0000","last_fullsized":"2023-10-09T04:27:26.983310+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":528,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724592+0000","last_change":"2023-10-09T04:24:56.181978+0000","last_active":"2023-10-09T04:27:22.724592+0000","last_peered":"2023-10-09T04:27:22.724592+0000","last_clean":"2023-10-09T04:27:22.724592+0000","last_became_active":"2023-10-09T04:24:56.164297+0000","last_became_peered":"2023-10-09T04:24:56.164297+0000","last_unstale":"2023-10-09T04:27:22.724592+0000","last_undegraded":"2023-10-09T04:27:22.724592+0000","last_fullsized":"2023-10-09T04:27:22.724592+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:15.703672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200960+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:27:28.200960+0000","last_peered":"2023-10-09T04:27:28.200960+0000","last_clean":"2023-10-09T04:27:28.200960+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:27:28.200960+0000","last_undegraded":"2023-10-09T04:27:28.200960+0000","last_fullsized":"2023-10-09T04:27:28.200960+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":539,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724708+0000","last_change":"2023-10-09T04:24:56.202659+0000","last_active":"2023-10-09T04:27:22.724708+0000","last_peered":"2023-10-09T04:27:22.724708+0000","last_clean":"2023-10-09T04:27:22.724708+0000","last_became_active":"2023-10-09T04:24:56.166522+0000","last_became_peered":"2023-10-09T04:24:56.166522+0000","last_unstale":"2023-10-09T04:27:22.724708+0000","last_undegraded":"2023-10-09T04:27:22.724708+0000","last_fullsized":"2023-10-09T04:27:22.724708+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:15.858204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200451+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:27:28.200451+0000","last_peered":"2023-10-09T04:27:28.200451+0000","last_clean":"2023-10-09T04:27:28.200451+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:27:28.200451+0000","last_undegraded":"2023-10-09T04:27:28.200451+0000","last_fullsized":"2023-10-09T04:27:28.200451+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":764,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.726796+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:27:22.726796+0000","last_peered":"2023-10-09T04:27:22.726796+0000","last_clean":"2023-10-09T04:27:22.726796+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:27:22.726796+0000","last_undegraded":"2023-10-09T04:27:22.726796+0000","last_fullsized":"2023-10-09T04:27:22.726796+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":245,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176776+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:27:28.176776+0000","last_peered":"2023-10-09T04:27:28.176776+0000","last_clean":"2023-10-09T04:27:28.176776+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:27:28.176776+0000","last_undegraded":"2023-10-09T04:27:28.176776+0000","last_fullsized":"2023-10-09T04:27:28.176776+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":361,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724044+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:27:22.724044+0000","last_peered":"2023-10-09T04:27:22.724044+0000","last_clean":"2023-10-09T04:27:22.724044+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:27:22.724044+0000","last_undegraded":"2023-10-09T04:27:22.724044+0000","last_fullsized":"2023-10-09T04:27:22.724044+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":623,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201126+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:27:28.201126+0000","last_peered":"2023-10-09T04:27:28.201126+0000","last_clean":"2023-10-09T04:27:28.201126+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:27:28.201126+0000","last_undegraded":"2023-10-09T04:27:28.201126+0000","last_fullsized":"2023-10-09T04:27:28.201126+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":138,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-09T04:27:22.724549+0000","last_change":"2023-10-09T04:26:21.347607+0000","last_active":"2023-10-09T04:27:22.724549+0000","last_peered":"2023-10-09T04:27:22.724549+0000","last_clean":"2023-10-09T04:27:22.724549+0000","last_became_active":"2023-10-09T04:26:21.347136+0000","last_became_peered":"2023-10-09T04:26:21.347136+0000","last_unstale":"2023-10-09T04:27:22.724549+0000","last_undegraded":"2023-10-09T04:27:22.724549+0000","last_fullsized":"2023-10-09T04:27:22.724549+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:52.721389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200343+0000","last_change":"2023-10-09T04:24:56.183973+0000","last_active":"2023-10-09T04:27:28.200343+0000","last_peered":"2023-10-09T04:27:28.200343+0000","last_clean":"2023-10-09T04:27:28.200343+0000","last_became_active":"2023-10-09T04:24:56.181453+0000","last_became_peered":"2023-10-09T04:24:56.181453+0000","last_unstale":"2023-10-09T04:27:28.200343+0000","last_undegraded":"2023-10-09T04:27:28.200343+0000","last_fullsized":"2023-10-09T04:27:28.200343+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:35.829983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":469893120,"num_objects":7170,"num_object_clones":0,"num_object_copies":21510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7170,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2028929024,"data_stored":2028929024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7170,"ondisk_log_size":7170,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":11614,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1889766,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201700,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020896,"kb_used_data":125080,"kb_used_omap":0,"kb_used_meta":895808,"kb_avail":92724256,"statfs":{"total":95995035648,"available":94949638144,"internally_reserved":0,"allocated":128081920,"data_stored":126580267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917307392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.77,"15min":3.0190000000000001},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.20799999999999999},"max":{"1min":4.4580000000000002,"5min":70.614000000000004,"15min":148.65899999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.24,"5min":2.0489999999999999,"15min":3.0670000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.221},"max":{"1min":6.0389999999999997,"5min":70.721999999999994,"15min":148.642},"last":0.315}]},{"osd":1,"last update":"Mon Oct 9 04:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":4.4359999999999999,"15min":5.5069999999999997},"min":{"1min":0.316,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.4870000000000001,"5min":261.935,"15min":261.935},"last":0.249},{"interface":"front","average":{"1min":1.5569999999999999,"5min":4.4740000000000002,"15min":5.3600000000000003},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":6.0229999999999997,"5min":262.07600000000002,"15min":262.07600000000002},"last":0.373}]},{"osd":2,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":4.383,"15min":2.778},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.194,"5min":253.41200000000001,"15min":253.41200000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.3049999999999999,"5min":4.5090000000000003,"15min":3.1720000000000002},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.246},"max":{"1min":5.9900000000000002,"5min":253.39400000000001,"15min":253.39400000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":3.758,"15min":2.8599999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.9219999999999997,"5min":255.97300000000001,"15min":255.97300000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.5169999999999999,"5min":3.9319999999999999,"15min":3.1219999999999999},"min":{"1min":0.38500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.0069999999999997,"5min":256.06099999999998,"15min":256.06099999999998},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":0.76400000000000001,"15min":0.76500000000000001},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":5.8689999999999998,"5min":5.8689999999999998,"15min":8.9359999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.319,"5min":0.81399999999999995,"15min":1.131},"min":{"1min":0.27600000000000002,"5min":0.251,"15min":0.251},"max":{"1min":6.0670000000000002,"5min":7.2549999999999999,"15min":12.566000000000001},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.1599999999999999,"15min":1.665},"min":{"1min":0.32300000000000001,"5min":0.254,"15min":0.254},"max":{"1min":7.4820000000000002,"5min":17.882999999999999,"15min":120.066},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.202,"5min":1.2270000000000001,"15min":1.4730000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":5.9539999999999997,"5min":17.568999999999999,"15min":120.09699999999999},"last":0.5}]},{"osd":6,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.121,"15min":1.2230000000000001},"min":{"1min":0.378,"5min":0.311,"15min":0.307},"max":{"1min":7.5170000000000003,"5min":7.5170000000000003,"15min":8.4550000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.5940000000000001,"5min":0.995,"15min":0.93799999999999994},"min":{"1min":0.40300000000000002,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":9.4179999999999993,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.54700000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463336,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":892500,"kb_used_data":298636,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":92852652,"statfs":{"total":95995035648,"available":95081115648,"internally_reserved":0,"allocated":305803264,"data_stored":304320929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.528,"15min":3.6160000000000001},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.193},"max":{"1min":4.5460000000000003,"5min":65.439999999999998,"15min":133.78999999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.23,"5min":1.6259999999999999,"15min":3.7469999999999999},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.218},"max":{"1min":5.8019999999999996,"5min":65.427999999999997,"15min":133.69300000000001},"last":0.58899999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":5.0700000000000003,"15min":5.1040000000000001},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":10.083,"5min":315.73599999999999,"15min":315.73599999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.9199999999999999,"5min":5.335,"15min":5.1909999999999998},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":15.159000000000001,"5min":316.91500000000002,"15min":316.91500000000002},"last":1.784}]},{"osd":2,"last update":"Mon Oct 9 04:26:47 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":3.3100000000000001,"15min":2.7879999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.23899999999999999},"max":{"1min":5.1040000000000001,"5min":70.358999999999995,"15min":139.06899999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.456,"5min":3.359,"15min":2.8340000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1360000000000001,"5min":70.391000000000005,"15min":139.173},"last":0.77000000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.397,"15min":2.0600000000000001},"min":{"1min":0.35999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.048,"5min":14.083,"15min":51.082000000000001},"last":1.3149999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.3149999999999999,"15min":1.9970000000000001},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":3.677,"5min":14.052,"15min":51.031999999999996},"last":0.625}]},{"osd":4,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.4350000000000001,"15min":1.427},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.24399999999999999},"max":{"1min":3.117,"5min":161.27500000000001,"15min":161.27500000000001},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.3780000000000001,"15min":1.496},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":5.1189999999999998,"5min":160.32599999999999,"15min":160.32599999999999},"last":0.69699999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":4.883,"5min":1.6120000000000001,"15min":1.2649999999999999},"min":{"1min":0.40799999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":74.043000000000006,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.92600000000000005},{"interface":"front","average":{"1min":5.0579999999999998,"5min":1.532,"15min":1.268},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":74.073999999999998,"5min":74.073999999999998,"15min":74.073999999999998},"last":0.876}]},{"osd":7,"last update":"Mon Oct 9 04:26:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.5049999999999999,"15min":1.6319999999999999},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":4.6189999999999998,"5min":161.39400000000001,"15min":161.39400000000001},"last":2.3540000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":2.3100000000000001,"15min":1.629},"min":{"1min":0.36299999999999999,"5min":0.34499999999999997,"15min":0.27800000000000002},"max":{"1min":5.1520000000000001,"5min":161.41200000000001,"15min":161.41200000000001},"last":2.3319999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089646,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1118708,"kb_used_data":373420,"kb_used_omap":9,"kb_used_meta":745270,"kb_avail":92626444,"statfs":{"total":95995035648,"available":94849478656,"internally_reserved":0,"allocated":382382080,"data_stored":380121503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":763157481},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":5.6790000000000003,"5min":6.21,"15min":4.7789999999999999},"min":{"1min":0.188,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":86.963999999999999,"5min":259.26799999999997,"15min":259.26799999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":5.798,"5min":6.2130000000000001,"15min":5.4530000000000003},"min":{"1min":0.222,"5min":0.222,"15min":0.182},"max":{"1min":83.213999999999999,"5min":258.46899999999999,"15min":258.46899999999999},"last":1.27}]},{"osd":2,"last update":"Mon Oct 9 04:26:56 2023","interfaces":[{"interface":"back","average":{"1min":4.5830000000000002,"5min":5.2039999999999997,"15min":3.2440000000000002},"min":{"1min":0.22900000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":74.474000000000004,"5min":259.315,"15min":259.315},"last":0.60099999999999998},{"interface":"front","average":{"1min":4.4340000000000002,"5min":5.3739999999999997,"15min":3.6080000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":75.653000000000006,"5min":259.733,"15min":259.733},"last":1.145}]},{"osd":3,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.7989999999999999,"15min":2.798},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8680000000000001,"5min":71.745999999999995,"15min":71.745999999999995},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.665,"15min":3.2189999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8520000000000003,"5min":75.613,"15min":75.613},"last":1.2430000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":4.3529999999999998,"15min":3.008},"min":{"1min":0.318,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":6.9370000000000003,"5min":257.83300000000003,"15min":257.83300000000003},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":4.2830000000000004,"15min":2.8420000000000001},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.4989999999999997,"5min":257.863,"15min":257.863},"last":0.41599999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.97399999999999998,"15min":2.177},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":6.9649999999999999,"5min":8.2300000000000004,"15min":38.848999999999997},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.018,"15min":2.2839999999999998},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":7.452,"5min":7.452,"15min":39.593000000000004},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.4179999999999999,"15min":2.028},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5590000000000002,"5min":35.045999999999999,"15min":35.045999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.21,"5min":1.502,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0259999999999998,"5min":34.960999999999999,"15min":34.960999999999999},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":4.577,"15min":3.9039999999999999},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.5289999999999999,"5min":259.08499999999998,"15min":259.08499999999998},"last":0.496},{"interface":"front","average":{"1min":1.119,"5min":4.6970000000000001,"15min":3.5489999999999999},"min":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":4.9790000000000001,"5min":259.13299999999998,"15min":259.13299999999998},"last":0.54800000000000004}]}]},{"osd":0,"up_from":84,"seq":360777253171,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":868340,"kb_used_data":81004,"kb_used_omap":28,"kb_used_meta":787299,"kb_avail":92876812,"statfs":{"total":95995035648,"available":95105855488,"internally_reserved":0,"allocated":82948096,"data_stored":80559847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":806195024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.849,"15min":4.4390000000000001},"min":{"1min":0.20499999999999999,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":4.2549999999999999,"5min":96.305000000000007,"15min":209.215},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.655,"5min":2.2149999999999999,"15min":4.4859999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":16.131,"5min":95.861000000000004,"15min":120.934},"last":0.44900000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.546,"15min":2.8340000000000001},"min":{"1min":0.23899999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":17.562999999999999,"5min":17.562999999999999,"15min":88.088999999999999},"last":0.496},{"interface":"front","average":{"1min":1.544,"5min":1.5009999999999999,"15min":3.1110000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.4499999999999993,"5min":17.655000000000001,"15min":88.004999999999995},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.95199999999999996,"15min":2.9740000000000002},"min":{"1min":0.27100000000000002,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":10.853999999999999,"5min":10.853999999999999,"15min":99.709000000000003},"last":1.3740000000000001},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.1539999999999999,"15min":3.036},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.292,"5min":12.292,"15min":99.560000000000002},"last":1.21}]},{"osd":4,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":1.786,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":86.906999999999996,"5min":86.906999999999996,"15min":88.058000000000007},"last":0.53300000000000003},{"interface":"front","average":{"1min":4.7919999999999998,"5min":1.78,"15min":2.4689999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":86.891000000000005,"5min":86.891000000000005,"15min":86.891000000000005},"last":0.65400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":6.8380000000000001,"5min":2.194,"15min":3.4670000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":134.93000000000001,"5min":134.93000000000001,"15min":156.96100000000001},"last":1.048},{"interface":"front","average":{"1min":6.7880000000000003,"5min":2.0779999999999998,"15min":2.972},"min":{"1min":0.29499999999999998,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":135.04599999999999,"5min":135.04599999999999,"15min":156.97300000000001},"last":1.284}]},{"osd":6,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.97699999999999998,"15min":1.766},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":4.7160000000000002,"5min":4.7160000000000002,"15min":63.468000000000004},"last":1.3440000000000001},{"interface":"front","average":{"1min":1.851,"5min":1.1719999999999999,"15min":2.29},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.34100000000000003},"max":{"1min":12.26,"5min":12.26,"15min":88.180000000000007},"last":0.74099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.119,"15min":1.915},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":11.962,"5min":11.962,"15min":63.509},"last":1.3129999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.242,"15min":2.4390000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":12.321,"5min":12.321,"15min":88.120000000000005},"last":0.78900000000000003}]}]},{"osd":2,"up_from":94,"seq":403726926120,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108580,"kb_used_data":351772,"kb_used_omap":81,"kb_used_meta":756718,"kb_avail":92636572,"statfs":{"total":95995035648,"available":94859849728,"internally_reserved":0,"allocated":360214528,"data_stored":357907192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":774879450},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:36 2023","interfaces":[{"interface":"back","average":{"1min":4.4800000000000004,"5min":1.9239999999999999,"15min":3.8900000000000001},"min":{"1min":0.192,"5min":0.183,"15min":0.17699999999999999},"max":{"1min":60.192,"5min":60.192,"15min":81.385999999999996},"last":0.23499999999999999},{"interface":"front","average":{"1min":4.4080000000000004,"5min":2.206,"15min":3.6720000000000002},"min":{"1min":0.23999999999999999,"5min":0.216,"15min":0.20000000000000001},"max":{"1min":58.289999999999999,"5min":58.289999999999999,"15min":79.453999999999994},"last":0.33100000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:26:36 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":3.1019999999999999,"15min":4.4699999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.191},"max":{"1min":1.085,"5min":115.184,"15min":158.13399999999999},"last":0.254},{"interface":"front","average":{"1min":1.637,"5min":3.1070000000000002,"15min":4.617},"min":{"1min":0.26600000000000001,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":6.5449999999999999,"5min":113.94,"15min":161.12},"last":0.219}]},{"osd":3,"last update":"Mon Oct 9 04:27:06 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.0089999999999999,"15min":2.3879999999999999},"min":{"1min":0.23699999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":15.201000000000001,"5min":15.201000000000001,"15min":58.997},"last":45.023000000000003},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.369,"15min":2.5920000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":15.102,"5min":15.102,"15min":66.135000000000005},"last":44.923999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:26:36 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":2.4279999999999999,"15min":2.2370000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":58.274000000000001,"5min":58.274000000000001,"15min":58.798000000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.2849999999999999,"15min":1.8999999999999999},"min":{"1min":0.27900000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.9860000000000002,"5min":24.702000000000002,"15min":56.616},"last":0.30499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:26:36 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.5049999999999999,"15min":2.4380000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":2.9319999999999999,"5min":10.885,"15min":82.030000000000001},"last":0.434},{"interface":"front","average":{"1min":5.4859999999999998,"5min":2.1640000000000001,"15min":2.6749999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":58.125999999999998,"5min":58.125999999999998,"15min":82.016000000000005},"last":0.34300000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:26:36 2023","interfaces":[{"interface":"back","average":{"1min":4.5970000000000004,"5min":1.7869999999999999,"15min":2.1320000000000001},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.308},"max":{"1min":58.469999999999999,"5min":58.469999999999999,"15min":58.469999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":4.9500000000000002,"5min":2.3809999999999998,"15min":2.2530000000000001},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.26800000000000002},"max":{"1min":58.067999999999998,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.39800000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:26:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.95799999999999996,"15min":2.343},"min":{"1min":0.41799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.052,"5min":4.1959999999999997,"15min":101.675},"last":0.41899999999999998},{"interface":"front","average":{"1min":4.6529999999999996,"5min":2.1829999999999998,"15min":2.75},"min":{"1min":0.42599999999999999,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":58.207999999999998,"5min":58.207999999999998,"15min":101.72499999999999},"last":0.372}]}]},{"osd":3,"up_from":99,"seq":425201762594,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1498376,"kb_used_data":503744,"kb_used_omap":7,"kb_used_meta":994616,"kb_avail":92246776,"statfs":{"total":95995035648,"available":94460698624,"internally_reserved":0,"allocated":515833856,"data_stored":513555904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1018487400},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":3.1739999999999999,"15min":3.9649999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":3.2999999999999998,"5min":65.810000000000002,"15min":125.765},"last":0.628},{"interface":"front","average":{"1min":1.1359999999999999,"5min":3.0409999999999999,"15min":4.4379999999999997},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":6.2709999999999999,"5min":65.924999999999997,"15min":125.46899999999999},"last":4.1369999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":2.8210000000000002,"15min":4.6440000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":6.3079999999999998,"5min":170.196,"15min":186.21299999999999},"last":3.3540000000000001},{"interface":"front","average":{"1min":0.88100000000000001,"5min":2.778,"15min":4.524},"min":{"1min":0.23300000000000001,"5min":0.23000000000000001,"15min":0.193},"max":{"1min":5.1319999999999997,"5min":166.953,"15min":191.78},"last":2.0169999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.3620000000000001,"15min":2.5790000000000002},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.187},"max":{"1min":3.702,"5min":9.2720000000000002,"15min":127.599},"last":2.7120000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.127,"15min":3.448},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":3.327,"5min":12.032999999999999,"15min":129.98599999999999},"last":4.056}]},{"osd":4,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":3.7229999999999999,"15min":2.6070000000000002},"min":{"1min":0.33800000000000002,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":7.7560000000000002,"5min":211.84700000000001,"15min":211.84700000000001},"last":4.0209999999999999},{"interface":"front","average":{"1min":1.488,"5min":3.9159999999999999,"15min":2.7829999999999999},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.111000000000001,"5min":211.17500000000001,"15min":211.17500000000001},"last":3.472}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.2070000000000001,"15min":2.052},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":3.476,"5min":9.6709999999999994,"15min":17.901},"last":3.294},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.1539999999999999,"15min":1.6100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":7.6289999999999996,"5min":7.7530000000000001,"15min":18.068000000000001},"last":0.66400000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.8120000000000001,"15min":1.71},"min":{"1min":0.40200000000000002,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":5.5369999999999999,"5min":64.378,"15min":64.378},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.5629999999999999,"15min":1.976},"min":{"1min":0.377,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":5.4870000000000001,"5min":64.353999999999999,"15min":64.353999999999999},"last":4.1109999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":3.5830000000000002,"15min":2.7879999999999998},"min":{"1min":0.41299999999999998,"5min":0.34799999999999998,"15min":0.34599999999999997},"max":{"1min":3.6259999999999999,"5min":212.62700000000001,"15min":212.62700000000001},"last":4.085},{"interface":"front","average":{"1min":1.1759999999999999,"5min":3.9079999999999999,"15min":3.0179999999999998},"min":{"1min":0.41199999999999998,"5min":0.376,"15min":0.33300000000000002},"max":{"1min":5.633,"5min":212.04599999999999,"15min":212.04599999999999},"last":3.5619999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986609,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":995668,"kb_used_data":97932,"kb_used_omap":0,"kb_used_meta":897728,"kb_avail":92749484,"statfs":{"total":95995035648,"available":94975471616,"internally_reserved":0,"allocated":100282368,"data_stored":98792124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919273472},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:38 2023","interfaces":[{"interface":"back","average":{"1min":12.137,"5min":5.5570000000000004,"15min":6.8680000000000003},"min":{"1min":0.22700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":207.45699999999999,"5min":207.45699999999999,"15min":207.45699999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":12.303000000000001,"5min":5.9370000000000003,"15min":7.0060000000000002},"min":{"1min":0.26400000000000001,"5min":0.251,"15min":0.23699999999999999},"max":{"1min":207.114,"5min":207.114,"15min":207.114},"last":1.0329999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":2.2309999999999999,"15min":4.9459999999999997},"min":{"1min":0.248,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":6.6959999999999997,"5min":90.900000000000006,"15min":135.25399999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.93899999999999995,"5min":2.3130000000000002,"15min":4.8600000000000003},"min":{"1min":0.28999999999999998,"5min":0.252,"15min":0.251},"max":{"1min":3.2389999999999999,"5min":90.787000000000006,"15min":134.44300000000001},"last":1.0760000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:26:54 2023","interfaces":[{"interface":"back","average":{"1min":22.367000000000001,"5min":7.3140000000000001,"15min":4.4880000000000004},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":203.61500000000001,"5min":203.61500000000001,"15min":203.61500000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":21.995000000000001,"5min":7.194,"15min":4.4630000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":203.601,"5min":203.601,"15min":203.601},"last":0.86499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.5469999999999999,"15min":2.7050000000000001},"min":{"1min":0.31,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":6.7199999999999998,"5min":30.789999999999999,"15min":154.13200000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.784,"5min":1.2869999999999999,"15min":2.7559999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":8.4329999999999998,"5min":9.2710000000000008,"15min":154.077},"last":0.83899999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.748,"15min":1.651},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.9020000000000001,"5min":5.9020000000000001,"15min":111.809},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.96799999999999997,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.252},"max":{"1min":6.6559999999999997,"5min":7.2400000000000002,"15min":112.002},"last":0.65000000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.635,"15min":1.6339999999999999},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.039999999999999,"5min":47.801000000000002,"15min":76.307000000000002},"last":1.282},{"interface":"front","average":{"1min":1.091,"5min":1.3200000000000001,"15min":1.4630000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1230000000000002,"5min":47.753,"15min":78.421999999999997},"last":1.3009999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:26:38 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.80900000000000005,"15min":1.073},"min":{"1min":0.317,"5min":0.309,"15min":0.26800000000000002},"max":{"1min":4.9279999999999999,"5min":6.0839999999999996,"15min":34.039000000000001},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.379,"5min":1.087,"15min":1.242},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.9190000000000005,"5min":9.9190000000000005,"15min":33.433999999999997},"last":1.0529999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724973,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":952584,"kb_used_data":68224,"kb_used_omap":0,"kb_used_meta":884352,"kb_avail":92792568,"statfs":{"total":95995035648,"available":95019589632,"internally_reserved":0,"allocated":69861376,"data_stored":68379650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905576448},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":2.9409999999999998,"5min":4.577,"15min":5.7640000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.193},"max":{"1min":43.402000000000001,"5min":139.86799999999999,"15min":151.23599999999999},"last":0.379},{"interface":"front","average":{"1min":3.2389999999999999,"5min":4.5899999999999999,"15min":6.0739999999999998},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":43.104999999999997,"5min":139.84899999999999,"15min":151.566},"last":0.57499999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":6.9960000000000004,"15min":5.3419999999999996},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":46.649000000000001,"5min":296.23099999999999,"15min":296.23099999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":4.1580000000000004,"5min":7.2859999999999996,"15min":5.6269999999999998},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":46.100999999999999,"5min":295.065,"15min":295.065},"last":0.40699999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2949999999999999,"5min":4.7869999999999999,"15min":3.8849999999999998},"min":{"1min":0.307,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":41.573999999999998,"5min":168.779,"15min":168.779},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.1760000000000002,"5min":5.1360000000000001,"15min":3.7250000000000001},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26200000000000001},"max":{"1min":41.676000000000002,"5min":168.03100000000001,"15min":168.03100000000001},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8570000000000002,"5min":1.478,"15min":2.1400000000000001},"min":{"1min":0.35699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":44.088999999999999,"5min":44.088999999999999,"15min":44.088999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.3719999999999999,"5min":1.3620000000000001,"15min":2.1640000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23499999999999999},"max":{"1min":44.103000000000002,"5min":44.103000000000002,"15min":44.103000000000002},"last":0.59899999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":1.8049999999999999,"15min":1.3200000000000001},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":95.668000000000006,"5min":95.668000000000006,"15min":95.668000000000006},"last":0.50600000000000001},{"interface":"front","average":{"1min":6.1790000000000003,"5min":1.802,"15min":1.274},"min":{"1min":0.32000000000000001,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":98.703999999999994,"5min":98.703999999999994,"15min":98.703999999999994},"last":0.52500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.65500000000000003,"15min":0.88200000000000001},"min":{"1min":0.33400000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":1.8400000000000001,"5min":8.5350000000000001,"15min":12.068},"last":0.63},{"interface":"front","average":{"1min":1.605,"5min":0.81599999999999995,"15min":1.042},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":11.577,"5min":11.577,"15min":11.577},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.78500000000000003,"15min":0.99399999999999999},"min":{"1min":0.373,"5min":0.33300000000000002,"15min":0.25900000000000001},"max":{"1min":3.7490000000000001,"5min":5.7699999999999996,"15min":12.096},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.7729999999999999,"5min":0.91900000000000004,"15min":1.1539999999999999},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":11.522,"5min":11.522,"15min":11.522},"last":0.61799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":136118272,"data_stored":136118272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":419561472,"data_stored":419561472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":359727104,"data_stored":359727104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":461176832,"data_stored":461176832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":142868480,"data_stored":142868480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":76152832,"data_stored":76152832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":312737792,"data_stored":312737792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":120586240,"data_stored":120586240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:27:39.128 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:27:39.181 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:39.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:39.288 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:39.370 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:39.439 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:39.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:39.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 10 16 23292 23276 145.432 103.75 0.00200174 0.00683978 2023-10-09T04:27:39.544 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:39.580 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:39.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:27:39.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:39.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:39.994 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:40.029 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:40.087 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:40.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:40.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:40.187 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:40.194 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:40.224 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:40.276 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:40.470 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 11 15 24863 24848 141.143 98.25 0.0112355 0.0070794 2023-10-09T04:27:40.481 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:40.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:40.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:40.631 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:40.705 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:40.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:40.785 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:40.814 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:40.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:27:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:27:40] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2023-10-09T04:27:40.864 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:40.943 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:40.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:40.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:41.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:41.112 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:41 smithi181 ceph-mon[196979]: pgmap v1330: 119 pgs: 119 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T04:27:41.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:41 smithi163 ceph-mon[221067]: pgmap v1330: 119 pgs: 119 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T04:27:41.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:41 smithi163 ceph-mon[213841]: pgmap v1330: 119 pgs: 119 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T04:27:41.474 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 12 16 26778 26762 139.337 119.625 0.00297899 0.00717016 2023-10-09T04:27:42.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:27:42.472 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 13 16 27912 27896 134.071 70.875 0.00179806 0.00744684 2023-10-09T04:27:42.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:42.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:42.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:42.703 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:42.777 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:42.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:42.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:42.884 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:42.950 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:43.068 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:43.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:43.179 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:43.190 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:43 smithi163 ceph-mon[221067]: pgmap v1331: 119 pgs: 119 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T04:27:43.190 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:43 smithi163 ceph-mon[213841]: pgmap v1331: 119 pgs: 119 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T04:27:43.257 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:43.473 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 14 16 29368 29352 130.982 91 0.00589917 0.00762971 2023-10-09T04:27:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:43 smithi181 ceph-mon[196979]: pgmap v1331: 119 pgs: 119 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T04:27:43.520 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:43.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:43.627 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:43.719 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:44.091 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:27:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3001820393' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:44.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:44 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:44.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3001820393' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:44.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:44 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:44.474 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 15 16 30922 30906 128.716 97.125 0.0106685 0.00776421 2023-10-09T04:27:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3001820393' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:44 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:27:44.599 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:44.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:44.630 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:44.708 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:44.779 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:45.104 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:27:45.106 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:27:45.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[221067]: pgmap v1332: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T04:27:45.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:45.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[221067]: failed to encode map e513 with expected crc 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[221067]: failed to encode map e513 with expected crc 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[221067]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[213841]: pgmap v1332: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[213841]: failed to encode map e513 with expected crc 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[213841]: failed to encode map e513 with expected crc 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:45.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:45 smithi163 ceph-mon[213841]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T04:27:45.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:45.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:45.477 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 16 16 32406 32390 126.451 92.75 0.00623255 0.00790275 2023-10-09T04:27:45.517 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:45 smithi181 ceph-mon[196979]: pgmap v1332: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T04:27:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:45 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:27:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:45 smithi181 ceph-mon[196979]: failed to encode map e513 with expected crc 2023-10-09T04:27:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:45 smithi181 ceph-mon[196979]: failed to encode map e513 with expected crc 2023-10-09T04:27:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:45 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:27:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:45 smithi181 ceph-mon[196979]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T04:27:45.584 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:45.887 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:45.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:45.996 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:46.023 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:46.095 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:46.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:46.202 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:46.242 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:46.347 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:46 smithi181 ceph-mon[196979]: from='client.56386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:46 smithi181 ceph-mon[196979]: 1.14 scrub starts 2023-10-09T04:27:46.355 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:46 smithi181 ceph-mon[196979]: 1.14 scrub ok 2023-10-09T04:27:46.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:46.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:46.457 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:46.477 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 17 16 33832 33816 124.256 89.125 0.00194113 0.00801427 2023-10-09T04:27:46.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:46 smithi163 ceph-mon[221067]: from='client.56386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:46.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:46 smithi163 ceph-mon[221067]: 1.14 scrub starts 2023-10-09T04:27:46.483 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:46 smithi163 ceph-mon[221067]: 1.14 scrub ok 2023-10-09T04:27:46.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:46 smithi163 ceph-mon[213841]: from='client.56386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:27:46.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:46 smithi163 ceph-mon[213841]: 1.14 scrub starts 2023-10-09T04:27:46.483 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:46 smithi163 ceph-mon[213841]: 1.14 scrub ok 2023-10-09T04:27:46.486 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:46.891 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1333,"stamp":"2023-10-09T04:27:44.083280+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1747148162,"num_objects":26674,"num_object_clones":0,"num_object_copies":80022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26674,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":26718,"num_write_kb":1705702,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":1901380,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":31696,"ondisk_log_size":31696,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11652548,"kb_used_data":5084420,"kb_used_omap":126,"kb_used_meta":6567937,"kb_avail":738308668,"statfs":{"total":767960285184,"available":756028076032,"internally_reserved":0,"allocated":5206446080,"data_stored":5191255412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6725568123},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,2,9,3,0,0,8,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1328807936,"num_objects":20276,"num_object_clones":0,"num_object_copies":60828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20276,"num_write_kb":1297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.079898"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":684,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982512+0000","last_change":"2023-10-09T04:26:24.679564+0000","last_active":"2023-10-09T04:27:26.982512+0000","last_peered":"2023-10-09T04:27:26.982512+0000","last_clean":"2023-10-09T04:27:26.982512+0000","last_became_active":"2023-10-09T04:26:24.679142+0000","last_became_peered":"2023-10-09T04:26:24.679142+0000","last_unstale":"2023-10-09T04:27:26.982512+0000","last_undegraded":"2023-10-09T04:27:26.982512+0000","last_fullsized":"2023-10-09T04:27:26.982512+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:37.095828+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":97,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200721+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:27:28.200721+0000","last_peered":"2023-10-09T04:27:28.200721+0000","last_clean":"2023-10-09T04:27:28.200721+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:27:28.200721+0000","last_undegraded":"2023-10-09T04:27:28.200721+0000","last_fullsized":"2023-10-09T04:27:28.200721+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201242+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:27:28.201242+0000","last_peered":"2023-10-09T04:27:28.201242+0000","last_clean":"2023-10-09T04:27:28.201242+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:27:28.201242+0000","last_undegraded":"2023-10-09T04:27:28.201242+0000","last_fullsized":"2023-10-09T04:27:28.201242+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":709,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982390+0000","last_change":"2023-10-09T04:24:58.734627+0000","last_active":"2023-10-09T04:27:26.982390+0000","last_peered":"2023-10-09T04:27:26.982390+0000","last_clean":"2023-10-09T04:27:26.982390+0000","last_became_active":"2023-10-09T04:24:58.734248+0000","last_became_peered":"2023-10-09T04:24:58.734248+0000","last_unstale":"2023-10-09T04:27:26.982390+0000","last_undegraded":"2023-10-09T04:27:26.982390+0000","last_fullsized":"2023-10-09T04:27:26.982390+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:59.934563+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":519,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176469+0000","last_change":"2023-10-09T04:24:56.158228+0000","last_active":"2023-10-09T04:27:28.176469+0000","last_peered":"2023-10-09T04:27:28.176469+0000","last_clean":"2023-10-09T04:27:28.176469+0000","last_became_active":"2023-10-09T04:24:56.157946+0000","last_became_peered":"2023-10-09T04:24:56.157946+0000","last_unstale":"2023-10-09T04:27:28.176469+0000","last_undegraded":"2023-10-09T04:27:28.176469+0000","last_fullsized":"2023-10-09T04:27:28.176469+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:37.279836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.743977+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:27:28.743977+0000","last_peered":"2023-10-09T04:27:28.743977+0000","last_clean":"2023-10-09T04:27:28.743977+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:27:28.743977+0000","last_undegraded":"2023-10-09T04:27:28.743977+0000","last_fullsized":"2023-10-09T04:27:28.743977+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":504,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200681+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:27:28.200681+0000","last_peered":"2023-10-09T04:27:28.200681+0000","last_clean":"2023-10-09T04:27:28.200681+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:27:28.200681+0000","last_undegraded":"2023-10-09T04:27:28.200681+0000","last_fullsized":"2023-10-09T04:27:28.200681+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":116,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627105+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:27:28.627105+0000","last_peered":"2023-10-09T04:27:28.627105+0000","last_clean":"2023-10-09T04:27:28.627105+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:27:28.627105+0000","last_undegraded":"2023-10-09T04:27:28.627105+0000","last_fullsized":"2023-10-09T04:27:28.627105+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":475,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982426+0000","last_change":"2023-10-09T04:26:09.747631+0000","last_active":"2023-10-09T04:27:26.982426+0000","last_peered":"2023-10-09T04:27:26.982426+0000","last_clean":"2023-10-09T04:27:26.982426+0000","last_became_active":"2023-10-09T04:26:09.747326+0000","last_became_peered":"2023-10-09T04:26:09.747326+0000","last_unstale":"2023-10-09T04:27:26.982426+0000","last_undegraded":"2023-10-09T04:27:26.982426+0000","last_fullsized":"2023-10-09T04:27:26.982426+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.315346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":911,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774454+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:27:27.774454+0000","last_peered":"2023-10-09T04:27:27.774454+0000","last_clean":"2023-10-09T04:27:27.774454+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:27:27.774454+0000","last_undegraded":"2023-10-09T04:27:27.774454+0000","last_fullsized":"2023-10-09T04:27:27.774454+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.743954+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:27:28.743954+0000","last_peered":"2023-10-09T04:27:28.743954+0000","last_clean":"2023-10-09T04:27:28.743954+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:27:28.743954+0000","last_undegraded":"2023-10-09T04:27:28.743954+0000","last_fullsized":"2023-10-09T04:27:28.743954+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.139772+0000","last_change":"2023-10-09T04:26:09.741543+0000","last_active":"2023-10-09T04:27:32.139772+0000","last_peered":"2023-10-09T04:27:32.139772+0000","last_clean":"2023-10-09T04:27:32.139772+0000","last_became_active":"2023-10-09T04:26:09.741193+0000","last_became_peered":"2023-10-09T04:26:09.741193+0000","last_unstale":"2023-10-09T04:27:32.139772+0000","last_undegraded":"2023-10-09T04:27:32.139772+0000","last_fullsized":"2023-10-09T04:27:32.139772+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:52:43.560332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":808,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774653+0000","last_change":"2023-10-09T04:26:30.562322+0000","last_active":"2023-10-09T04:27:27.774653+0000","last_peered":"2023-10-09T04:27:27.774653+0000","last_clean":"2023-10-09T04:27:27.774653+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:27:27.774653+0000","last_undegraded":"2023-10-09T04:27:27.774653+0000","last_fullsized":"2023-10-09T04:27:27.774653+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.363626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.363626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":471,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.981983+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:27:26.981983+0000","last_peered":"2023-10-09T04:27:26.981983+0000","last_clean":"2023-10-09T04:27:26.981983+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:27:26.981983+0000","last_undegraded":"2023-10-09T04:27:26.981983+0000","last_fullsized":"2023-10-09T04:27:26.981983+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176460+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:27:28.176460+0000","last_peered":"2023-10-09T04:27:28.176460+0000","last_clean":"2023-10-09T04:27:28.176460+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:27:28.176460+0000","last_undegraded":"2023-10-09T04:27:28.176460+0000","last_fullsized":"2023-10-09T04:27:28.176460+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":716,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982296+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:27:26.982296+0000","last_peered":"2023-10-09T04:27:26.982296+0000","last_clean":"2023-10-09T04:27:26.982296+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:27:26.982296+0000","last_undegraded":"2023-10-09T04:27:26.982296+0000","last_fullsized":"2023-10-09T04:27:26.982296+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":586,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774696+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:27:27.774696+0000","last_peered":"2023-10-09T04:27:27.774696+0000","last_clean":"2023-10-09T04:27:27.774696+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:27:27.774696+0000","last_undegraded":"2023-10-09T04:27:27.774696+0000","last_fullsized":"2023-10-09T04:27:27.774696+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":158,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983491+0000","last_change":"2023-10-09T04:26:21.351148+0000","last_active":"2023-10-09T04:27:26.983491+0000","last_peered":"2023-10-09T04:27:26.983491+0000","last_clean":"2023-10-09T04:27:26.983491+0000","last_became_active":"2023-10-09T04:26:21.349344+0000","last_became_peered":"2023-10-09T04:26:21.349344+0000","last_unstale":"2023-10-09T04:27:26.983491+0000","last_undegraded":"2023-10-09T04:27:26.983491+0000","last_fullsized":"2023-10-09T04:27:26.983491+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:41.495268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":690,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:29.080555+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:27:29.080555+0000","last_peered":"2023-10-09T04:27:29.080555+0000","last_clean":"2023-10-09T04:27:29.080555+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:27:29.080555+0000","last_undegraded":"2023-10-09T04:27:29.080555+0000","last_fullsized":"2023-10-09T04:27:29.080555+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":87,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201006+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:27:28.201006+0000","last_peered":"2023-10-09T04:27:28.201006+0000","last_clean":"2023-10-09T04:27:28.201006+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:27:28.201006+0000","last_undegraded":"2023-10-09T04:27:28.201006+0000","last_fullsized":"2023-10-09T04:27:28.201006+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201011+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:27:28.201011+0000","last_peered":"2023-10-09T04:27:28.201011+0000","last_clean":"2023-10-09T04:27:28.201011+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:27:28.201011+0000","last_undegraded":"2023-10-09T04:27:28.201011+0000","last_fullsized":"2023-10-09T04:27:28.201011+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":543,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627277+0000","last_change":"2023-10-09T04:27:02.906226+0000","last_active":"2023-10-09T04:27:28.627277+0000","last_peered":"2023-10-09T04:27:28.627277+0000","last_clean":"2023-10-09T04:27:28.627277+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:27:28.627277+0000","last_undegraded":"2023-10-09T04:27:28.627277+0000","last_fullsized":"2023-10-09T04:27:28.627277+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:21.516499+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":555,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774320+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:27:27.774320+0000","last_peered":"2023-10-09T04:27:27.774320+0000","last_clean":"2023-10-09T04:27:27.774320+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:27:27.774320+0000","last_undegraded":"2023-10-09T04:27:27.774320+0000","last_fullsized":"2023-10-09T04:27:27.774320+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":287,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774495+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:27:27.774495+0000","last_peered":"2023-10-09T04:27:27.774495+0000","last_clean":"2023-10-09T04:27:27.774495+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:27:27.774495+0000","last_undegraded":"2023-10-09T04:27:27.774495+0000","last_fullsized":"2023-10-09T04:27:27.774495+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":676,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983494+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:27:26.983494+0000","last_peered":"2023-10-09T04:27:26.983494+0000","last_clean":"2023-10-09T04:27:26.983494+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:27:26.983494+0000","last_undegraded":"2023-10-09T04:27:26.983494+0000","last_fullsized":"2023-10-09T04:27:26.983494+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":116,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627396+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:27:28.627396+0000","last_peered":"2023-10-09T04:27:28.627396+0000","last_clean":"2023-10-09T04:27:28.627396+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:27:28.627396+0000","last_undegraded":"2023-10-09T04:27:28.627396+0000","last_fullsized":"2023-10-09T04:27:28.627396+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":490,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982244+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:27:26.982244+0000","last_peered":"2023-10-09T04:27:26.982244+0000","last_clean":"2023-10-09T04:27:26.982244+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:27:26.982244+0000","last_undegraded":"2023-10-09T04:27:26.982244+0000","last_fullsized":"2023-10-09T04:27:26.982244+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":657,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176277+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:27:28.176277+0000","last_peered":"2023-10-09T04:27:28.176277+0000","last_clean":"2023-10-09T04:27:28.176277+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:27:28.176277+0000","last_undegraded":"2023-10-09T04:27:28.176277+0000","last_fullsized":"2023-10-09T04:27:28.176277+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200477+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:27:28.200477+0000","last_peered":"2023-10-09T04:27:28.200477+0000","last_clean":"2023-10-09T04:27:28.200477+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:27:28.200477+0000","last_undegraded":"2023-10-09T04:27:28.200477+0000","last_fullsized":"2023-10-09T04:27:28.200477+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.743895+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:27:28.743895+0000","last_peered":"2023-10-09T04:27:28.743895+0000","last_clean":"2023-10-09T04:27:28.743895+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:27:28.743895+0000","last_undegraded":"2023-10-09T04:27:28.743895+0000","last_fullsized":"2023-10-09T04:27:28.743895+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":676,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982862+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:27:26.982862+0000","last_peered":"2023-10-09T04:27:26.982862+0000","last_clean":"2023-10-09T04:27:26.982862+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:27:26.982862+0000","last_undegraded":"2023-10-09T04:27:26.982862+0000","last_fullsized":"2023-10-09T04:27:26.982862+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":126,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627205+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:27:28.627205+0000","last_peered":"2023-10-09T04:27:28.627205+0000","last_clean":"2023-10-09T04:27:28.627205+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:27:28.627205+0000","last_undegraded":"2023-10-09T04:27:28.627205+0000","last_fullsized":"2023-10-09T04:27:28.627205+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":177,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176758+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:27:28.176758+0000","last_peered":"2023-10-09T04:27:28.176758+0000","last_clean":"2023-10-09T04:27:28.176758+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:27:28.176758+0000","last_undegraded":"2023-10-09T04:27:28.176758+0000","last_fullsized":"2023-10-09T04:27:28.176758+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"512'1626","reported_seq":1639,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.033650+0000","last_change":"2023-10-09T04:27:20.720040+0000","last_active":"2023-10-09T04:27:41.033650+0000","last_peered":"2023-10-09T04:27:41.033650+0000","last_clean":"2023-10-09T04:27:41.033650+0000","last_became_active":"2023-10-09T04:27:20.719715+0000","last_became_peered":"2023-10-09T04:27:20.719715+0000","last_unstale":"2023-10-09T04:27:41.033650+0000","last_undegraded":"2023-10-09T04:27:41.033650+0000","last_fullsized":"2023-10-09T04:27:41.033650+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1626,"log_dups_size":0,"ondisk_log_size":1626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:37.979204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106561536,"num_objects":1626,"num_object_clones":0,"num_object_copies":4878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1626,"num_write_kb":104064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":954,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774476+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:27:27.774476+0000","last_peered":"2023-10-09T04:27:27.774476+0000","last_clean":"2023-10-09T04:27:27.774476+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:27:27.774476+0000","last_undegraded":"2023-10-09T04:27:27.774476+0000","last_fullsized":"2023-10-09T04:27:27.774476+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":556,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774655+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:27:27.774655+0000","last_peered":"2023-10-09T04:27:27.774655+0000","last_clean":"2023-10-09T04:27:27.774655+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:27:27.774655+0000","last_undegraded":"2023-10-09T04:27:27.774655+0000","last_fullsized":"2023-10-09T04:27:27.774655+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":473,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982389+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:27:26.982389+0000","last_peered":"2023-10-09T04:27:26.982389+0000","last_clean":"2023-10-09T04:27:26.982389+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:27:26.982389+0000","last_undegraded":"2023-10-09T04:27:26.982389+0000","last_fullsized":"2023-10-09T04:27:26.982389+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"512'1657","reported_seq":1671,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.499277+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:27:41.499277+0000","last_peered":"2023-10-09T04:27:41.499277+0000","last_clean":"2023-10-09T04:27:41.499277+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:27:41.499277+0000","last_undegraded":"2023-10-09T04:27:41.499277+0000","last_fullsized":"2023-10-09T04:27:41.499277+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1657,"log_dups_size":0,"ondisk_log_size":1657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108593152,"num_objects":1657,"num_object_clones":0,"num_object_copies":4971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1657,"num_write_kb":106048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":745,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176530+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:27:28.176530+0000","last_peered":"2023-10-09T04:27:28.176530+0000","last_clean":"2023-10-09T04:27:28.176530+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:27:28.176530+0000","last_undegraded":"2023-10-09T04:27:28.176530+0000","last_fullsized":"2023-10-09T04:27:28.176530+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":532,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774736+0000","last_change":"2023-10-09T04:24:56.212579+0000","last_active":"2023-10-09T04:27:27.774736+0000","last_peered":"2023-10-09T04:27:27.774736+0000","last_clean":"2023-10-09T04:27:27.774736+0000","last_became_active":"2023-10-09T04:24:56.184403+0000","last_became_peered":"2023-10-09T04:24:56.184403+0000","last_unstale":"2023-10-09T04:27:27.774736+0000","last_undegraded":"2023-10-09T04:27:27.774736+0000","last_fullsized":"2023-10-09T04:27:27.774736+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:10.486526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200537+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:27:28.200537+0000","last_peered":"2023-10-09T04:27:28.200537+0000","last_clean":"2023-10-09T04:27:28.200537+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:27:28.200537+0000","last_undegraded":"2023-10-09T04:27:28.200537+0000","last_fullsized":"2023-10-09T04:27:28.200537+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"512'1625","reported_seq":1638,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.044240+0000","last_change":"2023-10-09T04:27:20.720676+0000","last_active":"2023-10-09T04:27:41.044240+0000","last_peered":"2023-10-09T04:27:41.044240+0000","last_clean":"2023-10-09T04:27:41.044240+0000","last_became_active":"2023-10-09T04:27:20.720342+0000","last_became_peered":"2023-10-09T04:27:20.720342+0000","last_unstale":"2023-10-09T04:27:41.044240+0000","last_undegraded":"2023-10-09T04:27:41.044240+0000","last_fullsized":"2023-10-09T04:27:41.044240+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1625,"log_dups_size":0,"ondisk_log_size":1625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:37.485070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106496000,"num_objects":1625,"num_object_clones":0,"num_object_copies":4875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1625,"num_write_kb":104000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":716,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982204+0000","last_change":"2023-10-09T04:24:58.648512+0000","last_active":"2023-10-09T04:27:26.982204+0000","last_peered":"2023-10-09T04:27:26.982204+0000","last_clean":"2023-10-09T04:27:26.982204+0000","last_became_active":"2023-10-09T04:24:58.648143+0000","last_became_peered":"2023-10-09T04:24:58.648143+0000","last_unstale":"2023-10-09T04:27:26.982204+0000","last_undegraded":"2023-10-09T04:27:26.982204+0000","last_fullsized":"2023-10-09T04:27:26.982204+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:26.096956+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":483,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774589+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:27:27.774589+0000","last_peered":"2023-10-09T04:27:27.774589+0000","last_clean":"2023-10-09T04:27:27.774589+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:27:27.774589+0000","last_undegraded":"2023-10-09T04:27:27.774589+0000","last_fullsized":"2023-10-09T04:27:27.774589+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":138,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983367+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:27:26.983367+0000","last_peered":"2023-10-09T04:27:26.983367+0000","last_clean":"2023-10-09T04:27:26.983367+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:27:26.983367+0000","last_undegraded":"2023-10-09T04:27:26.983367+0000","last_fullsized":"2023-10-09T04:27:26.983367+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"512'1613","reported_seq":1626,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.028198+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:27:41.028198+0000","last_peered":"2023-10-09T04:27:41.028198+0000","last_clean":"2023-10-09T04:27:41.028198+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:27:41.028198+0000","last_undegraded":"2023-10-09T04:27:41.028198+0000","last_fullsized":"2023-10-09T04:27:41.028198+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1613,"log_dups_size":0,"ondisk_log_size":1613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105709568,"num_objects":1613,"num_object_clones":0,"num_object_copies":4839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1613,"num_write_kb":103232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":768,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201366+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:27:28.201366+0000","last_peered":"2023-10-09T04:27:28.201366+0000","last_clean":"2023-10-09T04:27:28.201366+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:27:28.201366+0000","last_undegraded":"2023-10-09T04:27:28.201366+0000","last_fullsized":"2023-10-09T04:27:28.201366+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":511,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982584+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:27:26.982584+0000","last_peered":"2023-10-09T04:27:26.982584+0000","last_clean":"2023-10-09T04:27:26.982584+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:27:26.982584+0000","last_undegraded":"2023-10-09T04:27:26.982584+0000","last_fullsized":"2023-10-09T04:27:26.982584+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":135,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.141265+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:27:32.141265+0000","last_peered":"2023-10-09T04:27:32.141265+0000","last_clean":"2023-10-09T04:27:32.141265+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:27:32.141265+0000","last_undegraded":"2023-10-09T04:27:32.141265+0000","last_fullsized":"2023-10-09T04:27:32.141265+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"512'1627","reported_seq":1638,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.045275+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:27:41.045275+0000","last_peered":"2023-10-09T04:27:41.045275+0000","last_clean":"2023-10-09T04:27:41.045275+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:27:41.045275+0000","last_undegraded":"2023-10-09T04:27:41.045275+0000","last_fullsized":"2023-10-09T04:27:41.045275+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1627,"log_dups_size":0,"ondisk_log_size":1627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106627072,"num_objects":1627,"num_object_clones":0,"num_object_copies":4881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1627,"num_write_kb":104128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":975,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774048+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:27:27.774048+0000","last_peered":"2023-10-09T04:27:27.774048+0000","last_clean":"2023-10-09T04:27:27.774048+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:27:27.774048+0000","last_undegraded":"2023-10-09T04:27:27.774048+0000","last_fullsized":"2023-10-09T04:27:27.774048+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":147,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.141230+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:27:32.141230+0000","last_peered":"2023-10-09T04:27:32.141230+0000","last_clean":"2023-10-09T04:27:32.141230+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:27:32.141230+0000","last_undegraded":"2023-10-09T04:27:32.141230+0000","last_fullsized":"2023-10-09T04:27:32.141230+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200994+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:27:28.200994+0000","last_peered":"2023-10-09T04:27:28.200994+0000","last_clean":"2023-10-09T04:27:28.200994+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:27:28.200994+0000","last_undegraded":"2023-10-09T04:27:28.200994+0000","last_fullsized":"2023-10-09T04:27:28.200994+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"512'1733","reported_seq":1747,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.505148+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:27:41.505148+0000","last_peered":"2023-10-09T04:27:41.505148+0000","last_clean":"2023-10-09T04:27:41.505148+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:27:41.505148+0000","last_undegraded":"2023-10-09T04:27:41.505148+0000","last_fullsized":"2023-10-09T04:27:41.505148+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1733,"log_dups_size":0,"ondisk_log_size":1733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113573888,"num_objects":1733,"num_object_clones":0,"num_object_copies":5199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1733,"num_write_kb":110912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":721,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982624+0000","last_change":"2023-10-09T04:26:23.679008+0000","last_active":"2023-10-09T04:27:26.982624+0000","last_peered":"2023-10-09T04:27:26.982624+0000","last_clean":"2023-10-09T04:27:26.982624+0000","last_became_active":"2023-10-09T04:26:23.678629+0000","last_became_peered":"2023-10-09T04:26:23.678629+0000","last_unstale":"2023-10-09T04:27:26.982624+0000","last_undegraded":"2023-10-09T04:27:26.982624+0000","last_fullsized":"2023-10-09T04:27:26.982624+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:41.534992+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.199931+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:27:28.199931+0000","last_peered":"2023-10-09T04:27:28.199931+0000","last_clean":"2023-10-09T04:27:28.199931+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:27:28.199931+0000","last_undegraded":"2023-10-09T04:27:28.199931+0000","last_fullsized":"2023-10-09T04:27:28.199931+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":94,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201237+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:27:28.201237+0000","last_peered":"2023-10-09T04:27:28.201237+0000","last_clean":"2023-10-09T04:27:28.201237+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:27:28.201237+0000","last_undegraded":"2023-10-09T04:27:28.201237+0000","last_fullsized":"2023-10-09T04:27:28.201237+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"512'1594","reported_seq":1606,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:40.573087+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:27:40.573087+0000","last_peered":"2023-10-09T04:27:40.573087+0000","last_clean":"2023-10-09T04:27:40.573087+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:27:40.573087+0000","last_undegraded":"2023-10-09T04:27:40.573087+0000","last_fullsized":"2023-10-09T04:27:40.573087+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1594,"log_dups_size":0,"ondisk_log_size":1594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104464384,"num_objects":1594,"num_object_clones":0,"num_object_copies":4782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1594,"num_write_kb":102016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":618,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:32.141342+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:27:32.141342+0000","last_peered":"2023-10-09T04:27:32.141342+0000","last_clean":"2023-10-09T04:27:32.141342+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:27:32.141342+0000","last_undegraded":"2023-10-09T04:27:32.141342+0000","last_fullsized":"2023-10-09T04:27:32.141342+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200599+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:27:28.200599+0000","last_peered":"2023-10-09T04:27:28.200599+0000","last_clean":"2023-10-09T04:27:28.200599+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:27:28.200599+0000","last_undegraded":"2023-10-09T04:27:28.200599+0000","last_fullsized":"2023-10-09T04:27:28.200599+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200046+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:27:28.200046+0000","last_peered":"2023-10-09T04:27:28.200046+0000","last_clean":"2023-10-09T04:27:28.200046+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:27:28.200046+0000","last_undegraded":"2023-10-09T04:27:28.200046+0000","last_fullsized":"2023-10-09T04:27:28.200046+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"512'1860","reported_seq":1874,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:43.830281+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:27:43.830281+0000","last_peered":"2023-10-09T04:27:43.830281+0000","last_clean":"2023-10-09T04:27:43.830281+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:27:43.830281+0000","last_undegraded":"2023-10-09T04:27:43.830281+0000","last_fullsized":"2023-10-09T04:27:43.830281+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1860,"log_dups_size":0,"ondisk_log_size":1860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1860,"num_write_kb":119040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":716,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982731+0000","last_change":"2023-10-09T04:26:23.680981+0000","last_active":"2023-10-09T04:27:26.982731+0000","last_peered":"2023-10-09T04:27:26.982731+0000","last_clean":"2023-10-09T04:27:26.982731+0000","last_became_active":"2023-10-09T04:26:23.680615+0000","last_became_peered":"2023-10-09T04:26:23.680615+0000","last_unstale":"2023-10-09T04:27:26.982731+0000","last_undegraded":"2023-10-09T04:27:26.982731+0000","last_fullsized":"2023-10-09T04:27:26.982731+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:36.804345+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":177,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176107+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:27:28.176107+0000","last_peered":"2023-10-09T04:27:28.176107+0000","last_clean":"2023-10-09T04:27:28.176107+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:27:28.176107+0000","last_undegraded":"2023-10-09T04:27:28.176107+0000","last_fullsized":"2023-10-09T04:27:28.176107+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":87,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201238+0000","last_change":"2023-10-09T04:26:09.746900+0000","last_active":"2023-10-09T04:27:28.201238+0000","last_peered":"2023-10-09T04:27:28.201238+0000","last_clean":"2023-10-09T04:27:28.201238+0000","last_became_active":"2023-10-09T04:26:09.746607+0000","last_became_peered":"2023-10-09T04:26:09.746607+0000","last_unstale":"2023-10-09T04:27:28.201238+0000","last_undegraded":"2023-10-09T04:27:28.201238+0000","last_fullsized":"2023-10-09T04:27:28.201238+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:07.695043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"512'1686","reported_seq":1699,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:42.160667+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:27:42.160667+0000","last_peered":"2023-10-09T04:27:42.160667+0000","last_clean":"2023-10-09T04:27:42.160667+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:27:42.160667+0000","last_undegraded":"2023-10-09T04:27:42.160667+0000","last_fullsized":"2023-10-09T04:27:42.160667+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1686,"log_dups_size":0,"ondisk_log_size":1686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110493696,"num_objects":1686,"num_object_clones":0,"num_object_copies":5058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1686,"num_write_kb":107904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":638,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983192+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:27:26.983192+0000","last_peered":"2023-10-09T04:27:26.983192+0000","last_clean":"2023-10-09T04:27:26.983192+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:27:26.983192+0000","last_undegraded":"2023-10-09T04:27:26.983192+0000","last_fullsized":"2023-10-09T04:27:26.983192+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":115,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627491+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:27:28.627491+0000","last_peered":"2023-10-09T04:27:28.627491+0000","last_clean":"2023-10-09T04:27:28.627491+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:27:28.627491+0000","last_undegraded":"2023-10-09T04:27:28.627491+0000","last_fullsized":"2023-10-09T04:27:28.627491+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":528,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982156+0000","last_change":"2023-10-09T04:26:21.351382+0000","last_active":"2023-10-09T04:27:26.982156+0000","last_peered":"2023-10-09T04:27:26.982156+0000","last_clean":"2023-10-09T04:27:26.982156+0000","last_became_active":"2023-10-09T04:26:21.348023+0000","last_became_peered":"2023-10-09T04:26:21.348023+0000","last_unstale":"2023-10-09T04:27:26.982156+0000","last_undegraded":"2023-10-09T04:27:26.982156+0000","last_fullsized":"2023-10-09T04:27:26.982156+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:45.177721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":391,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982819+0000","last_change":"2023-10-09T04:26:09.749151+0000","last_active":"2023-10-09T04:27:26.982819+0000","last_peered":"2023-10-09T04:27:26.982819+0000","last_clean":"2023-10-09T04:27:26.982819+0000","last_became_active":"2023-10-09T04:26:09.748857+0000","last_became_peered":"2023-10-09T04:26:09.748857+0000","last_unstale":"2023-10-09T04:27:26.982819+0000","last_undegraded":"2023-10-09T04:27:26.982819+0000","last_fullsized":"2023-10-09T04:27:26.982819+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:38.228161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"512'1740","reported_seq":1754,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.474102+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:27:41.474102+0000","last_peered":"2023-10-09T04:27:41.474102+0000","last_clean":"2023-10-09T04:27:41.474102+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:27:41.474102+0000","last_undegraded":"2023-10-09T04:27:41.474102+0000","last_fullsized":"2023-10-09T04:27:41.474102+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1740,"log_dups_size":0,"ondisk_log_size":1740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114032640,"num_objects":1740,"num_object_clones":0,"num_object_copies":5220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1740,"num_write_kb":111360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":719,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982004+0000","last_change":"2023-10-09T04:26:23.684553+0000","last_active":"2023-10-09T04:27:26.982004+0000","last_peered":"2023-10-09T04:27:26.982004+0000","last_clean":"2023-10-09T04:27:26.982004+0000","last_became_active":"2023-10-09T04:26:23.684125+0000","last_became_peered":"2023-10-09T04:26:23.684125+0000","last_unstale":"2023-10-09T04:27:26.982004+0000","last_undegraded":"2023-10-09T04:27:26.982004+0000","last_fullsized":"2023-10-09T04:27:26.982004+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:01.715214+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":495,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982537+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:27:26.982537+0000","last_peered":"2023-10-09T04:27:26.982537+0000","last_clean":"2023-10-09T04:27:26.982537+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:27:26.982537+0000","last_undegraded":"2023-10-09T04:27:26.982537+0000","last_fullsized":"2023-10-09T04:27:26.982537+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.199966+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:27:28.199966+0000","last_peered":"2023-10-09T04:27:28.199966+0000","last_clean":"2023-10-09T04:27:28.199966+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:27:28.199966+0000","last_undegraded":"2023-10-09T04:27:28.199966+0000","last_fullsized":"2023-10-09T04:27:28.199966+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":476,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:29.080477+0000","last_change":"2023-10-09T04:26:21.345296+0000","last_active":"2023-10-09T04:27:29.080477+0000","last_peered":"2023-10-09T04:27:29.080477+0000","last_clean":"2023-10-09T04:27:29.080477+0000","last_became_active":"2023-10-09T04:26:21.345003+0000","last_became_peered":"2023-10-09T04:26:21.345003+0000","last_unstale":"2023-10-09T04:27:29.080477+0000","last_undegraded":"2023-10-09T04:27:29.080477+0000","last_fullsized":"2023-10-09T04:27:29.080477+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:42:33.938068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"512'1559","reported_seq":1572,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.027554+0000","last_change":"2023-10-09T04:27:20.717622+0000","last_active":"2023-10-09T04:27:41.027554+0000","last_peered":"2023-10-09T04:27:41.027554+0000","last_clean":"2023-10-09T04:27:41.027554+0000","last_became_active":"2023-10-09T04:27:20.717291+0000","last_became_peered":"2023-10-09T04:27:20.717291+0000","last_unstale":"2023-10-09T04:27:41.027554+0000","last_undegraded":"2023-10-09T04:27:41.027554+0000","last_fullsized":"2023-10-09T04:27:41.027554+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1559,"log_dups_size":0,"ondisk_log_size":1559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:34.329924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102170624,"num_objects":1559,"num_object_clones":0,"num_object_copies":4677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1559,"num_write_kb":99776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":686,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:31.578630+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:27:31.578630+0000","last_peered":"2023-10-09T04:27:31.578630+0000","last_clean":"2023-10-09T04:27:31.578630+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:27:31.578630+0000","last_undegraded":"2023-10-09T04:27:31.578630+0000","last_fullsized":"2023-10-09T04:27:31.578630+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":362,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627184+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:27:28.627184+0000","last_peered":"2023-10-09T04:27:28.627184+0000","last_clean":"2023-10-09T04:27:28.627184+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:27:28.627184+0000","last_undegraded":"2023-10-09T04:27:28.627184+0000","last_fullsized":"2023-10-09T04:27:28.627184+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":451,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:29.080117+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:27:29.080117+0000","last_peered":"2023-10-09T04:27:29.080117+0000","last_clean":"2023-10-09T04:27:29.080117+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:27:29.080117+0000","last_undegraded":"2023-10-09T04:27:29.080117+0000","last_fullsized":"2023-10-09T04:27:29.080117+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":217,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982558+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:27:26.982558+0000","last_peered":"2023-10-09T04:27:26.982558+0000","last_clean":"2023-10-09T04:27:26.982558+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:27:26.982558+0000","last_undegraded":"2023-10-09T04:27:26.982558+0000","last_fullsized":"2023-10-09T04:27:26.982558+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"512'1538","reported_seq":1551,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.024446+0000","last_change":"2023-10-09T04:27:20.717505+0000","last_active":"2023-10-09T04:27:41.024446+0000","last_peered":"2023-10-09T04:27:41.024446+0000","last_clean":"2023-10-09T04:27:41.024446+0000","last_became_active":"2023-10-09T04:27:20.717202+0000","last_became_peered":"2023-10-09T04:27:20.717202+0000","last_unstale":"2023-10-09T04:27:41.024446+0000","last_undegraded":"2023-10-09T04:27:41.024446+0000","last_fullsized":"2023-10-09T04:27:41.024446+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1538,"log_dups_size":0,"ondisk_log_size":1538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:51.034299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100794368,"num_objects":1538,"num_object_clones":0,"num_object_copies":4614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1538,"num_write_kb":98432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":515,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982296+0000","last_change":"2023-10-09T04:26:21.429887+0000","last_active":"2023-10-09T04:27:26.982296+0000","last_peered":"2023-10-09T04:27:26.982296+0000","last_clean":"2023-10-09T04:27:26.982296+0000","last_became_active":"2023-10-09T04:26:21.392380+0000","last_became_peered":"2023-10-09T04:26:21.392380+0000","last_unstale":"2023-10-09T04:27:26.982296+0000","last_undegraded":"2023-10-09T04:27:26.982296+0000","last_fullsized":"2023-10-09T04:27:26.982296+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:05:55.650685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":966,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774028+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:27:27.774028+0000","last_peered":"2023-10-09T04:27:27.774028+0000","last_clean":"2023-10-09T04:27:27.774028+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:27:27.774028+0000","last_undegraded":"2023-10-09T04:27:27.774028+0000","last_fullsized":"2023-10-09T04:27:27.774028+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":448,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982768+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:27:26.982768+0000","last_peered":"2023-10-09T04:27:26.982768+0000","last_clean":"2023-10-09T04:27:26.982768+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:27:26.982768+0000","last_undegraded":"2023-10-09T04:27:26.982768+0000","last_fullsized":"2023-10-09T04:27:26.982768+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":74,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201124+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:27:28.201124+0000","last_peered":"2023-10-09T04:27:28.201124+0000","last_clean":"2023-10-09T04:27:28.201124+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:27:28.201124+0000","last_undegraded":"2023-10-09T04:27:28.201124+0000","last_fullsized":"2023-10-09T04:27:28.201124+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"512'1663","reported_seq":1677,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.493350+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:27:41.493350+0000","last_peered":"2023-10-09T04:27:41.493350+0000","last_clean":"2023-10-09T04:27:41.493350+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:27:41.493350+0000","last_undegraded":"2023-10-09T04:27:41.493350+0000","last_fullsized":"2023-10-09T04:27:41.493350+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1663,"log_dups_size":0,"ondisk_log_size":1663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108986368,"num_objects":1663,"num_object_clones":0,"num_object_copies":4989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1663,"num_write_kb":106432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":607,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201342+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:27:28.201342+0000","last_peered":"2023-10-09T04:27:28.201342+0000","last_clean":"2023-10-09T04:27:28.201342+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:27:28.201342+0000","last_undegraded":"2023-10-09T04:27:28.201342+0000","last_fullsized":"2023-10-09T04:27:28.201342+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":512,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774027+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:27:27.774027+0000","last_peered":"2023-10-09T04:27:27.774027+0000","last_clean":"2023-10-09T04:27:27.774027+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:27:27.774027+0000","last_undegraded":"2023-10-09T04:27:27.774027+0000","last_fullsized":"2023-10-09T04:27:27.774027+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":153,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627380+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:27:28.627380+0000","last_peered":"2023-10-09T04:27:28.627380+0000","last_clean":"2023-10-09T04:27:28.627380+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:27:28.627380+0000","last_undegraded":"2023-10-09T04:27:28.627380+0000","last_fullsized":"2023-10-09T04:27:28.627380+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":135,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.743813+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:27:28.743813+0000","last_peered":"2023-10-09T04:27:28.743813+0000","last_clean":"2023-10-09T04:27:28.743813+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:27:28.743813+0000","last_undegraded":"2023-10-09T04:27:28.743813+0000","last_fullsized":"2023-10-09T04:27:28.743813+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"512'1770","reported_seq":1783,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:42.165479+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:27:42.165479+0000","last_peered":"2023-10-09T04:27:42.165479+0000","last_clean":"2023-10-09T04:27:42.165479+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:27:42.165479+0000","last_undegraded":"2023-10-09T04:27:42.165479+0000","last_fullsized":"2023-10-09T04:27:42.165479+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1770,"log_dups_size":0,"ondisk_log_size":1770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1770,"num_write_kb":113280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":630,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983334+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:27:26.983334+0000","last_peered":"2023-10-09T04:27:26.983334+0000","last_clean":"2023-10-09T04:27:26.983334+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:27:26.983334+0000","last_undegraded":"2023-10-09T04:27:26.983334+0000","last_fullsized":"2023-10-09T04:27:26.983334+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":535,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774673+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:27:27.774673+0000","last_peered":"2023-10-09T04:27:27.774673+0000","last_clean":"2023-10-09T04:27:27.774673+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:27:27.774673+0000","last_undegraded":"2023-10-09T04:27:27.774673+0000","last_fullsized":"2023-10-09T04:27:27.774673+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":270,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982248+0000","last_change":"2023-10-09T04:26:21.355315+0000","last_active":"2023-10-09T04:27:26.982248+0000","last_peered":"2023-10-09T04:27:26.982248+0000","last_clean":"2023-10-09T04:27:26.982248+0000","last_became_active":"2023-10-09T04:26:21.355038+0000","last_became_peered":"2023-10-09T04:26:21.355038+0000","last_unstale":"2023-10-09T04:27:26.982248+0000","last_undegraded":"2023-10-09T04:27:26.982248+0000","last_fullsized":"2023-10-09T04:27:26.982248+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:45.819056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":231,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982098+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:27:26.982098+0000","last_peered":"2023-10-09T04:27:26.982098+0000","last_clean":"2023-10-09T04:27:26.982098+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:27:26.982098+0000","last_undegraded":"2023-10-09T04:27:26.982098+0000","last_fullsized":"2023-10-09T04:27:26.982098+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"512'1654","reported_seq":1667,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:41.036979+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:27:41.036979+0000","last_peered":"2023-10-09T04:27:41.036979+0000","last_clean":"2023-10-09T04:27:41.036979+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:27:41.036979+0000","last_undegraded":"2023-10-09T04:27:41.036979+0000","last_fullsized":"2023-10-09T04:27:41.036979+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1654,"log_dups_size":0,"ondisk_log_size":1654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108396544,"num_objects":1654,"num_object_clones":0,"num_object_copies":4962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1654,"num_write_kb":105856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":569,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200098+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:27:28.200098+0000","last_peered":"2023-10-09T04:27:28.200098+0000","last_clean":"2023-10-09T04:27:28.200098+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:27:28.200098+0000","last_undegraded":"2023-10-09T04:27:28.200098+0000","last_fullsized":"2023-10-09T04:27:28.200098+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":472,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774814+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:27:27.774814+0000","last_peered":"2023-10-09T04:27:27.774814+0000","last_clean":"2023-10-09T04:27:27.774814+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:27:27.774814+0000","last_undegraded":"2023-10-09T04:27:27.774814+0000","last_fullsized":"2023-10-09T04:27:27.774814+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200033+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:27:28.200033+0000","last_peered":"2023-10-09T04:27:28.200033+0000","last_clean":"2023-10-09T04:27:28.200033+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:27:28.200033+0000","last_undegraded":"2023-10-09T04:27:28.200033+0000","last_fullsized":"2023-10-09T04:27:28.200033+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":420,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774206+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:27:27.774206+0000","last_peered":"2023-10-09T04:27:27.774206+0000","last_clean":"2023-10-09T04:27:27.774206+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:27:27.774206+0000","last_undegraded":"2023-10-09T04:27:27.774206+0000","last_fullsized":"2023-10-09T04:27:27.774206+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"512'1705","reported_seq":1718,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:27:42.173982+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:27:42.173982+0000","last_peered":"2023-10-09T04:27:42.173982+0000","last_clean":"2023-10-09T04:27:42.173982+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:27:42.173982+0000","last_undegraded":"2023-10-09T04:27:42.173982+0000","last_fullsized":"2023-10-09T04:27:42.173982+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":1705,"log_dups_size":0,"ondisk_log_size":1705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111738880,"num_objects":1705,"num_object_clones":0,"num_object_copies":5115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1705,"num_write_kb":109120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":687,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176488+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:27:28.176488+0000","last_peered":"2023-10-09T04:27:28.176488+0000","last_clean":"2023-10-09T04:27:28.176488+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:27:28.176488+0000","last_undegraded":"2023-10-09T04:27:28.176488+0000","last_fullsized":"2023-10-09T04:27:28.176488+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":311,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983491+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:27:26.983491+0000","last_peered":"2023-10-09T04:27:26.983491+0000","last_clean":"2023-10-09T04:27:26.983491+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:27:26.983491+0000","last_undegraded":"2023-10-09T04:27:26.983491+0000","last_fullsized":"2023-10-09T04:27:26.983491+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":147,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.744086+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:27:28.744086+0000","last_peered":"2023-10-09T04:27:28.744086+0000","last_clean":"2023-10-09T04:27:28.744086+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:27:28.744086+0000","last_undegraded":"2023-10-09T04:27:28.744086+0000","last_fullsized":"2023-10-09T04:27:28.744086+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.199931+0000","last_change":"2023-10-09T04:24:56.169203+0000","last_active":"2023-10-09T04:27:28.199931+0000","last_peered":"2023-10-09T04:27:28.199931+0000","last_clean":"2023-10-09T04:27:28.199931+0000","last_became_active":"2023-10-09T04:24:56.168737+0000","last_became_peered":"2023-10-09T04:24:56.168737+0000","last_unstale":"2023-10-09T04:27:28.199931+0000","last_undegraded":"2023-10-09T04:27:28.199931+0000","last_fullsized":"2023-10-09T04:27:28.199931+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:21.515886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":554,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627270+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:27:28.627270+0000","last_peered":"2023-10-09T04:27:28.627270+0000","last_clean":"2023-10-09T04:27:28.627270+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:27:28.627270+0000","last_undegraded":"2023-10-09T04:27:28.627270+0000","last_fullsized":"2023-10-09T04:27:28.627270+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":134,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.626960+0000","last_change":"2023-10-09T04:24:56.170052+0000","last_active":"2023-10-09T04:27:28.626960+0000","last_peered":"2023-10-09T04:27:28.626960+0000","last_clean":"2023-10-09T04:27:28.626960+0000","last_became_active":"2023-10-09T04:24:56.169607+0000","last_became_peered":"2023-10-09T04:24:56.169607+0000","last_unstale":"2023-10-09T04:27:28.626960+0000","last_undegraded":"2023-10-09T04:27:28.626960+0000","last_fullsized":"2023-10-09T04:27:28.626960+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.849923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":51,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.615097+0000","last_change":"2023-10-09T04:26:21.345230+0000","last_active":"2023-10-09T04:27:28.615097+0000","last_peered":"2023-10-09T04:27:28.615097+0000","last_clean":"2023-10-09T04:27:28.615097+0000","last_became_active":"2023-10-09T04:26:21.344939+0000","last_became_peered":"2023-10-09T04:26:21.344939+0000","last_unstale":"2023-10-09T04:27:28.615097+0000","last_undegraded":"2023-10-09T04:27:28.615097+0000","last_fullsized":"2023-10-09T04:27:28.615097+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.495703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":586,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.983310+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:27:26.983310+0000","last_peered":"2023-10-09T04:27:26.983310+0000","last_clean":"2023-10-09T04:27:26.983310+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:27:26.983310+0000","last_undegraded":"2023-10-09T04:27:26.983310+0000","last_fullsized":"2023-10-09T04:27:26.983310+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":529,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.982691+0000","last_change":"2023-10-09T04:24:56.181978+0000","last_active":"2023-10-09T04:27:26.982691+0000","last_peered":"2023-10-09T04:27:26.982691+0000","last_clean":"2023-10-09T04:27:26.982691+0000","last_became_active":"2023-10-09T04:24:56.164297+0000","last_became_peered":"2023-10-09T04:24:56.164297+0000","last_unstale":"2023-10-09T04:27:26.982691+0000","last_undegraded":"2023-10-09T04:27:26.982691+0000","last_fullsized":"2023-10-09T04:27:26.982691+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:15.703672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200960+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:27:28.200960+0000","last_peered":"2023-10-09T04:27:28.200960+0000","last_clean":"2023-10-09T04:27:28.200960+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:27:28.200960+0000","last_undegraded":"2023-10-09T04:27:28.200960+0000","last_fullsized":"2023-10-09T04:27:28.200960+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":540,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:26.981978+0000","last_change":"2023-10-09T04:24:56.202659+0000","last_active":"2023-10-09T04:27:26.981978+0000","last_peered":"2023-10-09T04:27:26.981978+0000","last_clean":"2023-10-09T04:27:26.981978+0000","last_became_active":"2023-10-09T04:24:56.166522+0000","last_became_peered":"2023-10-09T04:24:56.166522+0000","last_unstale":"2023-10-09T04:27:26.981978+0000","last_undegraded":"2023-10-09T04:27:26.981978+0000","last_fullsized":"2023-10-09T04:27:26.981978+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:15.858204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200451+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:27:28.200451+0000","last_peered":"2023-10-09T04:27:28.200451+0000","last_clean":"2023-10-09T04:27:28.200451+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:27:28.200451+0000","last_undegraded":"2023-10-09T04:27:28.200451+0000","last_fullsized":"2023-10-09T04:27:28.200451+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":765,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:27.774393+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:27:27.774393+0000","last_peered":"2023-10-09T04:27:27.774393+0000","last_clean":"2023-10-09T04:27:27.774393+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:27:27.774393+0000","last_undegraded":"2023-10-09T04:27:27.774393+0000","last_fullsized":"2023-10-09T04:27:27.774393+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":245,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.176776+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:27:28.176776+0000","last_peered":"2023-10-09T04:27:28.176776+0000","last_clean":"2023-10-09T04:27:28.176776+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:27:28.176776+0000","last_undegraded":"2023-10-09T04:27:28.176776+0000","last_fullsized":"2023-10-09T04:27:28.176776+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":362,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.627183+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:27:28.627183+0000","last_peered":"2023-10-09T04:27:28.627183+0000","last_clean":"2023-10-09T04:27:28.627183+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:27:28.627183+0000","last_undegraded":"2023-10-09T04:27:28.627183+0000","last_fullsized":"2023-10-09T04:27:28.627183+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":623,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.201126+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:27:28.201126+0000","last_peered":"2023-10-09T04:27:28.201126+0000","last_clean":"2023-10-09T04:27:28.201126+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:27:28.201126+0000","last_undegraded":"2023-10-09T04:27:28.201126+0000","last_fullsized":"2023-10-09T04:27:28.201126+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":139,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.744042+0000","last_change":"2023-10-09T04:26:21.347607+0000","last_active":"2023-10-09T04:27:28.744042+0000","last_peered":"2023-10-09T04:27:28.744042+0000","last_clean":"2023-10-09T04:27:28.744042+0000","last_became_active":"2023-10-09T04:26:21.347136+0000","last_became_peered":"2023-10-09T04:26:21.347136+0000","last_unstale":"2023-10-09T04:27:28.744042+0000","last_undegraded":"2023-10-09T04:27:28.744042+0000","last_fullsized":"2023-10-09T04:27:28.744042+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:52.721389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:27:28.200343+0000","last_change":"2023-10-09T04:24:56.183973+0000","last_active":"2023-10-09T04:27:28.200343+0000","last_peered":"2023-10-09T04:27:28.200343+0000","last_clean":"2023-10-09T04:27:28.200343+0000","last_became_active":"2023-10-09T04:24:56.181453+0000","last_became_peered":"2023-10-09T04:24:56.181453+0000","last_unstale":"2023-10-09T04:27:28.200343+0000","last_undegraded":"2023-10-09T04:27:28.200343+0000","last_fullsized":"2023-10-09T04:27:28.200343+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:35.829983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1746534400,"num_objects":26650,"num_object_clones":0,"num_object_copies":79950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26650,"num_write_kb":1705600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5264113664,"data_stored":5264113664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":26650,"ondisk_log_size":26650,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":11614,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":608258,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1889766,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201702,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1522536,"kb_used_data":618528,"kb_used_omap":0,"kb_used_meta":904000,"kb_avail":92222616,"statfs":{"total":95995035648,"available":94435958784,"internally_reserved":0,"allocated":633372672,"data_stored":631870870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925696000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,5,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.77,"15min":3.0190000000000001},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.20799999999999999},"max":{"1min":4.4580000000000002,"5min":70.614000000000004,"15min":148.65899999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.24,"5min":2.0489999999999999,"15min":3.0670000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.221},"max":{"1min":6.0389999999999997,"5min":70.721999999999994,"15min":148.642},"last":1.216}]},{"osd":1,"last update":"Mon Oct 9 04:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":4.4359999999999999,"15min":5.5069999999999997},"min":{"1min":0.316,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.4870000000000001,"5min":261.935,"15min":261.935},"last":1.948},{"interface":"front","average":{"1min":1.5569999999999999,"5min":4.4740000000000002,"15min":5.3600000000000003},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":6.0229999999999997,"5min":262.07600000000002,"15min":262.07600000000002},"last":1.454}]},{"osd":2,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":4.383,"15min":2.778},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.194,"5min":253.41200000000001,"15min":253.41200000000001},"last":1.482},{"interface":"front","average":{"1min":1.3049999999999999,"5min":4.5090000000000003,"15min":3.1720000000000002},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.246},"max":{"1min":5.9900000000000002,"5min":253.39400000000001,"15min":253.39400000000001},"last":1.397}]},{"osd":3,"last update":"Mon Oct 9 04:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":3.758,"15min":2.8599999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.9219999999999997,"5min":255.97300000000001,"15min":255.97300000000001},"last":1.5840000000000001},{"interface":"front","average":{"1min":1.5169999999999999,"5min":3.9319999999999999,"15min":3.1219999999999999},"min":{"1min":0.38500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.0069999999999997,"5min":256.06099999999998,"15min":256.06099999999998},"last":1.429}]},{"osd":4,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":0.76400000000000001,"15min":0.76500000000000001},"min":{"1min":0.30299999999999999,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":5.8689999999999998,"5min":5.8689999999999998,"15min":8.9359999999999999},"last":1.611},{"interface":"front","average":{"1min":1.319,"5min":0.81399999999999995,"15min":1.131},"min":{"1min":0.27600000000000002,"5min":0.251,"15min":0.251},"max":{"1min":6.0670000000000002,"5min":7.2549999999999999,"15min":12.566000000000001},"last":1.768}]},{"osd":5,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.1599999999999999,"15min":1.665},"min":{"1min":0.32300000000000001,"5min":0.254,"15min":0.254},"max":{"1min":7.4820000000000002,"5min":17.882999999999999,"15min":120.066},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.202,"5min":1.2270000000000001,"15min":1.4730000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":5.9539999999999997,"5min":17.568999999999999,"15min":120.09699999999999},"last":1.7829999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.121,"15min":1.2230000000000001},"min":{"1min":0.378,"5min":0.311,"15min":0.307},"max":{"1min":7.5170000000000003,"5min":7.5170000000000003,"15min":8.4550000000000001},"last":1.917},{"interface":"front","average":{"1min":1.5940000000000001,"5min":0.995,"15min":0.93799999999999994},"min":{"1min":0.40300000000000002,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":9.4179999999999993,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":1.8100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463338,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1443540,"kb_used_data":853772,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":92301612,"statfs":{"total":95995035648,"available":94516850688,"internally_reserved":0,"allocated":874262528,"data_stored":872780193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.528,"15min":3.6160000000000001},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.193},"max":{"1min":4.5460000000000003,"5min":65.439999999999998,"15min":133.78999999999999},"last":0.372},{"interface":"front","average":{"1min":1.23,"5min":1.6259999999999999,"15min":3.7469999999999999},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.218},"max":{"1min":5.8019999999999996,"5min":65.427999999999997,"15min":133.69300000000001},"last":0.29699999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":5.0700000000000003,"15min":5.1040000000000001},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":10.083,"5min":315.73599999999999,"15min":315.73599999999999},"last":2.1520000000000001},{"interface":"front","average":{"1min":1.9199999999999999,"5min":5.335,"15min":5.1909999999999998},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":15.159000000000001,"5min":316.91500000000002,"15min":316.91500000000002},"last":2.1019999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:26:47 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":3.3100000000000001,"15min":2.7879999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.23899999999999999},"max":{"1min":5.1040000000000001,"5min":70.358999999999995,"15min":139.06899999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.456,"5min":3.359,"15min":2.8340000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1360000000000001,"5min":70.391000000000005,"15min":139.173},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.397,"15min":2.0600000000000001},"min":{"1min":0.35999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.048,"5min":14.083,"15min":51.082000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.997,"5min":1.3149999999999999,"15min":1.9970000000000001},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":3.677,"5min":14.052,"15min":51.031999999999996},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.4350000000000001,"15min":1.427},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.24399999999999999},"max":{"1min":3.117,"5min":161.27500000000001,"15min":161.27500000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.3780000000000001,"15min":1.496},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":5.1189999999999998,"5min":160.32599999999999,"15min":160.32599999999999},"last":0.35699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":4.883,"5min":1.6120000000000001,"15min":1.2649999999999999},"min":{"1min":0.40799999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":74.043000000000006,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.42099999999999999},{"interface":"front","average":{"1min":5.0579999999999998,"5min":1.532,"15min":1.268},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":74.073999999999998,"5min":74.073999999999998,"15min":74.073999999999998},"last":0.39900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.95499999999999996,"15min":1.6459999999999999},"min":{"1min":0.40400000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.3040000000000003,"5min":5.9050000000000002,"15min":161.39400000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.81000000000000005,"15min":1.6339999999999999},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.27800000000000002},"max":{"1min":4.242,"5min":5.1520000000000001,"15min":161.41200000000001},"last":0.38100000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089648,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667956,"kb_used_data":930860,"kb_used_omap":9,"kb_used_meta":737078,"kb_avail":92077196,"statfs":{"total":95995035648,"available":94287048704,"internally_reserved":0,"allocated":953200640,"data_stored":950940063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":754768873},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":5.6790000000000003,"5min":6.21,"15min":4.7789999999999999},"min":{"1min":0.188,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":86.963999999999999,"5min":259.26799999999997,"15min":259.26799999999997},"last":1.296},{"interface":"front","average":{"1min":5.798,"5min":6.2130000000000001,"15min":5.4530000000000003},"min":{"1min":0.222,"5min":0.222,"15min":0.182},"max":{"1min":83.213999999999999,"5min":258.46899999999999,"15min":258.46899999999999},"last":1.3280000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:26:56 2023","interfaces":[{"interface":"back","average":{"1min":4.5830000000000002,"5min":5.2039999999999997,"15min":3.2440000000000002},"min":{"1min":0.22900000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":74.474000000000004,"5min":259.315,"15min":259.315},"last":1.675},{"interface":"front","average":{"1min":4.4340000000000002,"5min":5.3739999999999997,"15min":3.6080000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":75.653000000000006,"5min":259.733,"15min":259.733},"last":0.68600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.7989999999999999,"15min":2.798},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8680000000000001,"5min":71.745999999999995,"15min":71.745999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.665,"15min":3.2189999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8520000000000003,"5min":75.613,"15min":75.613},"last":0.51400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":4.3529999999999998,"15min":3.008},"min":{"1min":0.318,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":6.9370000000000003,"5min":257.83300000000003,"15min":257.83300000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.1439999999999999,"5min":4.2830000000000004,"15min":2.8420000000000001},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.4989999999999997,"5min":257.863,"15min":257.863},"last":0.76800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.97399999999999998,"15min":2.177},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":6.9649999999999999,"5min":8.2300000000000004,"15min":38.848999999999997},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.018,"15min":2.2839999999999998},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":7.452,"5min":7.452,"15min":39.593000000000004},"last":0.82499999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.4179999999999999,"15min":2.028},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5590000000000002,"5min":35.045999999999999,"15min":35.045999999999999},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.21,"5min":1.502,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0259999999999998,"5min":34.960999999999999,"15min":34.960999999999999},"last":0.625}]},{"osd":7,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":4.577,"15min":3.9039999999999999},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.5289999999999999,"5min":259.08499999999998,"15min":259.08499999999998},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.119,"5min":4.6970000000000001,"15min":3.5489999999999999},"min":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":4.9790000000000001,"5min":259.13299999999998,"15min":259.13299999999998},"last":0.65000000000000002}]}]},{"osd":0,"up_from":84,"seq":360777253173,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1306684,"kb_used_data":515252,"kb_used_omap":28,"kb_used_meta":791395,"kb_avail":92438468,"statfs":{"total":95995035648,"available":94656991232,"internally_reserved":0,"allocated":527618048,"data_stored":525229650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":810389328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,8],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.849,"15min":4.4390000000000001},"min":{"1min":0.20499999999999999,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":4.2549999999999999,"5min":96.305000000000007,"15min":209.215},"last":0.879},{"interface":"front","average":{"1min":1.655,"5min":2.2149999999999999,"15min":4.4859999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":16.131,"5min":95.861000000000004,"15min":120.934},"last":0.28399999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.546,"15min":2.8340000000000001},"min":{"1min":0.23899999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":17.562999999999999,"5min":17.562999999999999,"15min":88.088999999999999},"last":0.25},{"interface":"front","average":{"1min":1.544,"5min":1.5009999999999999,"15min":3.1110000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.4499999999999993,"5min":17.655000000000001,"15min":88.004999999999995},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.95199999999999996,"15min":2.9740000000000002},"min":{"1min":0.27100000000000002,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":10.853999999999999,"5min":10.853999999999999,"15min":99.709000000000003},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.1539999999999999,"15min":3.036},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.292,"5min":12.292,"15min":99.560000000000002},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":1.786,"15min":2.5840000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":86.906999999999996,"5min":86.906999999999996,"15min":88.058000000000007},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.7919999999999998,"5min":1.78,"15min":2.4689999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":86.891000000000005,"5min":86.891000000000005,"15min":86.891000000000005},"last":0.45400000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":6.8380000000000001,"5min":2.194,"15min":3.4670000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":134.93000000000001,"5min":134.93000000000001,"15min":156.96100000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":6.7880000000000003,"5min":2.0779999999999998,"15min":2.972},"min":{"1min":0.29499999999999998,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":135.04599999999999,"5min":135.04599999999999,"15min":156.97300000000001},"last":1.0209999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.97699999999999998,"15min":1.766},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":4.7160000000000002,"5min":4.7160000000000002,"15min":63.468000000000004},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.851,"5min":1.1719999999999999,"15min":2.29},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.34100000000000003},"max":{"1min":12.26,"5min":12.26,"15min":88.180000000000007},"last":0.54300000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.119,"15min":1.915},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":11.962,"5min":11.962,"15min":63.509},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.242,"15min":2.4390000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":12.321,"5min":12.321,"15min":88.120000000000005},"last":0.60599999999999998}]}]},{"osd":2,"up_from":94,"seq":403726926121,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1326372,"kb_used_data":573660,"kb_used_omap":81,"kb_used_meta":752622,"kb_avail":92418780,"statfs":{"total":95995035648,"available":94636830720,"internally_reserved":0,"allocated":587427840,"data_stored":585120504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":770685146},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.1059999999999999,"15min":3.677},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17699999999999999},"max":{"1min":19.873000000000001,"5min":60.192,"15min":81.385999999999996},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.0569999999999999,"5min":2.3839999999999999,"15min":3.5630000000000002},"min":{"1min":0.218,"5min":0.216,"15min":0.20799999999999999},"max":{"1min":19.827999999999999,"5min":58.289999999999999,"15min":79.453999999999994},"last":1.165}]},{"osd":1,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":2.226,"15min":4.335},"min":{"1min":0.224,"5min":0.224,"15min":0.191},"max":{"1min":3.2370000000000001,"5min":115.184,"15min":158.13399999999999},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.75900000000000001,"5min":2.2519999999999998,"15min":4.4199999999999999},"min":{"1min":0.219,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":3.8940000000000001,"5min":113.94,"15min":161.12},"last":1.7589999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:06 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.0089999999999999,"15min":2.3879999999999999},"min":{"1min":0.23699999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":15.201000000000001,"5min":15.201000000000001,"15min":58.997},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.369,"15min":2.5920000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":15.102,"5min":15.102,"15min":66.135000000000005},"last":1.2250000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":2.1499999999999999,"15min":2.1389999999999998},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":3.8639999999999999,"5min":58.274000000000001,"15min":58.798000000000002},"last":1.798},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.079,"15min":1.7230000000000001},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.3049999999999997,"5min":24.702000000000002,"15min":56.616},"last":0.73399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.3560000000000001,"15min":2.3519999999999999},"min":{"1min":0.40699999999999997,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":4.4459999999999997,"5min":7.984,"15min":82.030000000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.9670000000000001,"15min":2.581},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.28599999999999998},"max":{"1min":3.923,"5min":58.125999999999998,"15min":82.016000000000005},"last":0.745}]},{"osd":6,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.736,"15min":1.952},"min":{"1min":0.36499999999999999,"5min":0.34399999999999997,"15min":0.308},"max":{"1min":5.2720000000000002,"5min":58.469999999999999,"15min":58.469999999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.2189999999999999,"15min":2.1579999999999999},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.26800000000000002},"max":{"1min":3.198,"5min":58.067999999999998,"15min":58.067999999999998},"last":1.615}]},{"osd":7,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.99399999999999999,"15min":2.149},"min":{"1min":0.41899999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.2290000000000001,"5min":5.2290000000000001,"15min":101.675},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.179,"5min":2.2639999999999998,"15min":2.653},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":4.6239999999999997,"5min":58.207999999999998,"15min":101.72499999999999},"last":1.2749999999999999}]}]},{"osd":3,"up_from":99,"seq":425201762595,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1745736,"kb_used_data":738816,"kb_used_omap":7,"kb_used_meta":1006904,"kb_avail":91999416,"statfs":{"total":95995035648,"available":94207401984,"internally_reserved":0,"allocated":756547584,"data_stored":754269632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1031070312},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":3.1739999999999999,"15min":3.9649999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":3.2999999999999998,"5min":65.810000000000002,"15min":125.765},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":3.0409999999999999,"15min":4.4379999999999997},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":6.2709999999999999,"5min":65.924999999999997,"15min":125.46899999999999},"last":0.28000000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":2.8210000000000002,"15min":4.6440000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":6.3079999999999998,"5min":170.196,"15min":186.21299999999999},"last":2.3479999999999999},{"interface":"front","average":{"1min":0.88100000000000001,"5min":2.778,"15min":4.524},"min":{"1min":0.23300000000000001,"5min":0.23000000000000001,"15min":0.193},"max":{"1min":5.1319999999999997,"5min":166.953,"15min":191.78},"last":0.82699999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.3620000000000001,"15min":2.5790000000000002},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.187},"max":{"1min":3.702,"5min":9.2720000000000002,"15min":127.599},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.127,"15min":3.448},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":3.327,"5min":12.032999999999999,"15min":129.98599999999999},"last":0.995}]},{"osd":4,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":3.7229999999999999,"15min":2.6070000000000002},"min":{"1min":0.33800000000000002,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":7.7560000000000002,"5min":211.84700000000001,"15min":211.84700000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.488,"5min":3.9159999999999999,"15min":2.7829999999999999},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.111000000000001,"5min":211.17500000000001,"15min":211.17500000000001},"last":1.3169999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.2070000000000001,"15min":2.052},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":3.476,"5min":9.6709999999999994,"15min":17.901},"last":1.244},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.1539999999999999,"15min":1.6100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":7.6289999999999996,"5min":7.7530000000000001,"15min":18.068000000000001},"last":1.0569999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.8120000000000001,"15min":1.71},"min":{"1min":0.40200000000000002,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":5.5369999999999999,"5min":64.378,"15min":64.378},"last":3.952},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.5629999999999999,"15min":1.976},"min":{"1min":0.377,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":5.4870000000000001,"5min":64.353999999999999,"15min":64.353999999999999},"last":0.51700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":3.5830000000000002,"15min":2.7879999999999998},"min":{"1min":0.41299999999999998,"5min":0.34799999999999998,"15min":0.34599999999999997},"max":{"1min":3.6259999999999999,"5min":212.62700000000001,"15min":212.62700000000001},"last":2.7989999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":3.9079999999999999,"15min":3.0179999999999998},"min":{"1min":0.41199999999999998,"5min":0.376,"15min":0.33300000000000002},"max":{"1min":5.633,"5min":212.04599999999999,"15min":212.04599999999999},"last":1.2849999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986611,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1347100,"kb_used_data":445268,"kb_used_omap":0,"kb_used_meta":901824,"kb_avail":92398052,"statfs":{"total":95995035648,"available":94615605248,"internally_reserved":0,"allocated":455954432,"data_stored":454464039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923467776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:38 2023","interfaces":[{"interface":"back","average":{"1min":7.5609999999999999,"5min":6.6189999999999998,"15min":7.0549999999999997},"min":{"1min":0.26500000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":121.652,"5min":207.45699999999999,"15min":207.45699999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":8.4710000000000001,"5min":7.0990000000000002,"15min":7.2510000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":125.911,"5min":207.114,"15min":207.114},"last":1.399}]},{"osd":1,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":2.2309999999999999,"15min":4.9459999999999997},"min":{"1min":0.248,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":6.6959999999999997,"5min":90.900000000000006,"15min":135.25399999999999},"last":2.3959999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":2.3130000000000002,"15min":4.8600000000000003},"min":{"1min":0.28999999999999998,"5min":0.252,"15min":0.251},"max":{"1min":3.2389999999999999,"5min":90.787000000000006,"15min":134.44300000000001},"last":1.0980000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:26:54 2023","interfaces":[{"interface":"back","average":{"1min":22.367000000000001,"5min":7.3140000000000001,"15min":4.4880000000000004},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":203.61500000000001,"5min":203.61500000000001,"15min":203.61500000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":21.995000000000001,"5min":7.194,"15min":4.4630000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":203.601,"5min":203.601,"15min":203.601},"last":0.40699999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.5469999999999999,"15min":2.7050000000000001},"min":{"1min":0.31,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":6.7199999999999998,"5min":30.789999999999999,"15min":154.13200000000001},"last":2.3679999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.2869999999999999,"15min":2.7559999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":8.4329999999999998,"5min":9.2710000000000008,"15min":154.077},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.748,"15min":1.651},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.9020000000000001,"5min":5.9020000000000001,"15min":111.809},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.96799999999999997,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.252},"max":{"1min":6.6559999999999997,"5min":7.2400000000000002,"15min":112.002},"last":2.3380000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.635,"15min":1.6339999999999999},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.039999999999999,"5min":47.801000000000002,"15min":76.307000000000002},"last":2.6160000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.3200000000000001,"15min":1.4630000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1230000000000002,"5min":47.753,"15min":78.421999999999997},"last":3.1779999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":0.88700000000000001,"15min":1.1140000000000001},"min":{"1min":0.41099999999999998,"5min":0.309,"15min":0.26800000000000002},"max":{"1min":5.0510000000000002,"5min":6.0839999999999996,"15min":34.039000000000001},"last":0.621},{"interface":"front","average":{"1min":1.732,"5min":1.23,"15min":1.3100000000000001},"min":{"1min":0.40100000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":12.061,"5min":12.061,"15min":33.433999999999997},"last":2.294}]}]},{"osd":5,"up_from":39,"seq":167503724975,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292624,"kb_used_data":408264,"kb_used_omap":0,"kb_used_meta":884352,"kb_avail":92452528,"statfs":{"total":95995035648,"available":94671388672,"internally_reserved":0,"allocated":418062336,"data_stored":416580461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":905576448},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":2.9409999999999998,"5min":4.577,"15min":5.7640000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.193},"max":{"1min":43.402000000000001,"5min":139.86799999999999,"15min":151.23599999999999},"last":0.23200000000000001},{"interface":"front","average":{"1min":3.2389999999999999,"5min":4.5899999999999999,"15min":6.0739999999999998},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":43.104999999999997,"5min":139.84899999999999,"15min":151.566},"last":0.72499999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":6.9960000000000004,"15min":5.3419999999999996},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":46.649000000000001,"5min":296.23099999999999,"15min":296.23099999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":4.1580000000000004,"5min":7.2859999999999996,"15min":5.6269999999999998},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":46.100999999999999,"5min":295.065,"15min":295.065},"last":1.655}]},{"osd":2,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":3.2949999999999999,"5min":4.7869999999999999,"15min":3.8849999999999998},"min":{"1min":0.307,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":41.573999999999998,"5min":168.779,"15min":168.779},"last":1.5549999999999999},{"interface":"front","average":{"1min":3.1760000000000002,"5min":5.1360000000000001,"15min":3.7250000000000001},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.26200000000000001},"max":{"1min":41.676000000000002,"5min":168.03100000000001,"15min":168.03100000000001},"last":0.33200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8570000000000002,"5min":1.478,"15min":2.1400000000000001},"min":{"1min":0.35699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":44.088999999999999,"5min":44.088999999999999,"15min":44.088999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":3.3719999999999999,"5min":1.3620000000000001,"15min":2.1640000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23499999999999999},"max":{"1min":44.103000000000002,"5min":44.103000000000002,"15min":44.103000000000002},"last":0.70699999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":6.2380000000000004,"5min":1.8049999999999999,"15min":1.3200000000000001},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":95.668000000000006,"5min":95.668000000000006,"15min":95.668000000000006},"last":0.40400000000000003},{"interface":"front","average":{"1min":6.1790000000000003,"5min":1.802,"15min":1.274},"min":{"1min":0.32000000000000001,"5min":0.27900000000000003,"15min":0.26500000000000001},"max":{"1min":98.703999999999994,"5min":98.703999999999994,"15min":98.703999999999994},"last":0.36099999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.65500000000000003,"15min":0.88200000000000001},"min":{"1min":0.33400000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":1.8400000000000001,"5min":8.5350000000000001,"15min":12.068},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.605,"5min":0.81599999999999995,"15min":1.042},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":11.577,"5min":11.577,"15min":11.577},"last":0.441}]},{"osd":7,"last update":"Mon Oct 9 04:26:53 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.78500000000000003,"15min":0.99399999999999999},"min":{"1min":0.373,"5min":0.33300000000000002,"15min":0.25900000000000001},"max":{"1min":3.7490000000000001,"5min":5.7699999999999996,"15min":12.096},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.7729999999999999,"5min":0.91900000000000004,"15min":1.1539999999999999},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":11.522,"5min":11.522,"15min":11.522},"last":0.69099999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":556466176,"data_stored":556466176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":991625216,"data_stored":991625216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":589037568,"data_stored":589037568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":717422592,"data_stored":717422592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":451280896,"data_stored":451280896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":415825920,"data_stored":415825920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":902955008,"data_stored":902955008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":639500288,"data_stored":639500288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:27:46.899 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:27:46.899 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:27:46.900 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:27:46.900 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2023-10-09T04:27:46.900 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T04:27:46.942 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:46.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:47.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:47.098 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:47.107 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18926 10.5 10:a0371204:::benchmark_data_smithi163_523108_object18925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.12623154099998, 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0791226709999999, 2023-10-09T04:27:47.108 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18926 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.109 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977985+0000", 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.110 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978010+0000", 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5281000000000001e-05 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978022+0000", 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1691e-05 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.111 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978029+0000", 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6140000000000003e-06 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978125+0000", 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5649000000000005e-05 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978259+0000", 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013442700000000001 2023-10-09T04:27:47.112 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978399+0000", 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013955200000000001 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.979678+0000", 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001279362 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.113 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.980540+0000", 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086218799999999995 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.007748+0000", 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.027207293 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.007761+0000", 2023-10-09T04:27:47.114 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3307e-05 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979759+0000", 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97199803900000004 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979772+0000", 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2935999999999999e-05 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.115 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057024+0000", 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.07725206 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057109+0000", 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4544999999999998e-05 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.116 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18936 10.8 10:1447a75e:::benchmark_data_smithi163_523108_object18935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.105131766, 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0663146870000002, 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.117 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18936 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.118 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999084+0000", 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999108+0000", 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4119000000000001e-05 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.119 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999119+0000", 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0609e-05 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999129+0000", 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0173999999999999e-05 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.005396+0000", 2023-10-09T04:27:47.120 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062664510000000001 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.005408+0000", 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2614e-05 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.978908+0000", 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97349999099999995 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.121 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.978922+0000", 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3569e-05 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058726+0000", 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.079804481 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.122 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.059470+0000", 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00074404699999999996 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.059625+0000", 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015495099999999999 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060888+0000", 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012629589999999999 2023-10-09T04:27:47.123 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.061724+0000", 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083528300000000003 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.064344+0000", 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026200139999999999 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.124 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.064384+0000", 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0055999999999997e-05 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.065400+0000", 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010168359999999999 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.125 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18951 10.5 10:a56ce1b4:::benchmark_data_smithi163_523108_object18950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 321.08602575700002, 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0484555979999999, 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.126 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18951 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.127 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018190+0000", 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018218+0000", 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7906e-05 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.128 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018235+0000", 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7246999999999999e-05 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018245+0000", 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.465e-06 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018303+0000", 2023-10-09T04:27:47.129 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8554999999999998e-05 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018315+0000", 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2088e-05 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979816+0000", 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96150034200000001 2023-10-09T04:27:47.130 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979836+0000", 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0537e-05 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057168+0000", 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.077331303 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.131 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057311+0000", 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014343400000000001 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057491+0000", 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000179817 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060562+0000", 2023-10-09T04:27:47.132 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030713649999999999 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060602+0000", 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9830999999999999e-05 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.065968+0000", 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053659409999999999 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.133 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.066023+0000", 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5339e-05 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.066647+0000", 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062404699999999997 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.134 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38490 10.5 10:a06fa733:::benchmark_data_smithi163_523108_object38489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 302.03549124900002, 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0132891260000001, 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.135 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38490 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.136 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068746+0000", 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0443000000000001e-05 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.137 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068753+0000", 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9829999999999999e-06 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068759+0000", 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5550000000000002e-06 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.138 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.077390+0000", 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086309060000000007 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.077400+0000", 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0161e-05 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.139 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075149+0000", 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99774961600000001 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075307+0000", 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015781200000000001 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075470+0000", 2023-10-09T04:27:47.140 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016307 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080366+0000", 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048961869999999998 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080409+0000", 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2290999999999999e-05 2023-10-09T04:27:47.141 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081931+0000", 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015225309999999999 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081989+0000", 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7714999999999999e-05 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.142 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.082015+0000", 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6398999999999999e-05 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57312 10.8 10:143e0015:::benchmark_data_smithi163_523108_object57311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 280.76583226399998, 2023-10-09T04:27:47.143 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.136295909, 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57312 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.144 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338384+0000", 2023-10-09T04:27:47.145 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338414+0000", 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0681e-05 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338431+0000", 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6909999999999999e-05 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.146 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338443+0000", 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1537e-05 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342061+0000", 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036182689999999999 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.147 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342072+0000", 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1219999999999999e-05 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.464257+0000", 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.122184904 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465066+0000", 2023-10-09T04:27:47.148 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080893800000000004 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474628+0000", 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095617159999999996 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474681+0000", 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3440000000000003e-05 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.149 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57318 10.8 10:1fa756a5:::benchmark_data_smithi163_523108_object57317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 280.76575905099997, 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1361408630000001, 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.150 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57318 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:27:47.151 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338484+0000", 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6522000000000001e-05 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.152 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338493+0000", 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6600000000000001e-06 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338499+0000", 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.285e-06 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.153 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342095+0000", 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035956619999999999 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342105+0000", 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5959999999999993e-06 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465152+0000", 2023-10-09T04:27:47.154 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.123046999 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465958+0000", 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080678800000000001 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474519+0000", 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085601770000000004 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.155 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474599+0000", 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0765000000000003e-05 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57319 10.a 10:5ee00c90:::benchmark_data_smithi163_523108_object57318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:27:47.156 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 280.765712882, 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1486770740000001, 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57319 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.157 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338504+0000", 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:47.158 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338532+0000", 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7617999999999999e-05 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338540+0000", 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0949999999999996e-06 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.159 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338545+0000", 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4469999999999997e-06 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.339505+0000", 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00095972899999999999 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.160 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.339519+0000", 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3448000000000001e-05 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470780+0000", 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.131261257 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470931+0000", 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015102200000000001 2023-10-09T04:27:47.161 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.471097+0000", 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016623699999999999 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.478095+0000", 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069980989999999998 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.162 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.478878+0000", 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078291299999999999 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.481861+0000", 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029827709999999999 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.163 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486996+0000", 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051355569999999998 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.487059+0000", 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2792999999999993e-05 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.487182+0000", 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012257699999999999 2023-10-09T04:27:47.164 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57338 10.5 10:a63760e3:::benchmark_data_smithi163_523108_object57337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 280.61899743800001, 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0231315560000001, 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.165 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57338 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.166 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485218+0000", 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485241+0000", 2023-10-09T04:27:47.167 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2555e-05 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485260+0000", 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814e-05 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485267+0000", 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5599999999999996e-06 2023-10-09T04:27:47.168 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485316+0000", 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8528e-05 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485332+0000", 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6419e-05 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.169 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.495704+0000", 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.010371613 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.495862+0000", 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000157911 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:27:47.170 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.496027+0000", 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016513000000000001 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.505438+0000", 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0094115150000000005 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.507674+0000", 2023-10-09T04:27:47.171 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022358320000000001 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.507710+0000", 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5670999999999998e-05 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508174+0000", 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046372900000000002 2023-10-09T04:27:47.172 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508224+0000", 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0794000000000002e-05 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508352+0000", 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127293 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.173 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83363 10.5 10:a7ea1084:::benchmark_data_smithi163_523108_object9347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.361999+0000", 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 205.742218393, 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1263849750000001, 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.174 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83363 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.361999+0000", 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.175 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.361999+0000", 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.361997+0000", 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362009+0000", 2023-10-09T04:27:47.176 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1888e-05 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362032+0000", 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3439999999999999e-05 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362043+0000", 2023-10-09T04:27:47.177 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0873000000000001e-05 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362103+0000", 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9784999999999998e-05 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362285+0000", 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018241500000000001 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.178 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362474+0000", 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188996 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.364221+0000", 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001746538 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.179 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.364375+0000", 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015463299999999999 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.366914+0000", 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025380670000000002 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430708+0000", 2023-10-09T04:27:47.180 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.063794540999999996 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430727+0000", 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9128e-05 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488297+0000", 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.057569824 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.181 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488384+0000", 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7097999999999999e-05 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83370 10.15 10:a9c25298:::benchmark_data_smithi163_523108_object9354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.383272+0000", 2023-10-09T04:27:47.182 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 205.720945699, 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.097316261, 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83370 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.183 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383272+0000", 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383272+0000", 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383271+0000", 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:27:47.184 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383276+0000", 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9679999999999997e-06 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383285+0000", 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8589999999999994e-06 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.185 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383290+0000", 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2809999999999999e-06 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383337+0000", 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7315999999999999e-05 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383467+0000", 2023-10-09T04:27:47.186 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012938199999999999 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383564+0000", 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7003999999999995e-05 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.385231+0000", 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0016668589999999999 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.187 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.385448+0000", 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021762999999999999 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.385614+0000", 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165353 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.188 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430659+0000", 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.045045307 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430673+0000", 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4246999999999999e-05 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.480490+0000", 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0498168329999999 2023-10-09T04:27:47.189 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.480588+0000", 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8188000000000002e-05 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83385 10.15 10:ac2c79ee:::benchmark_data_smithi163_523108_object9369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2023-10-09T04:27:47.190 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.444437+0000", 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 205.659780716, 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.099660825, 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83385 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.191 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444437+0000", 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444437+0000", 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444435+0000", 2023-10-09T04:27:47.192 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444444+0000", 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8049999999999996e-06 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444464+0000", 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0163999999999999e-05 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.193 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444475+0000", 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0655999999999999e-05 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444536+0000", 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1309000000000005e-05 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.194 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444550+0000", 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4188e-05 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.480981+0000", 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.036431117 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.481113+0000", 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013171600000000001 2023-10-09T04:27:47.195 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.481247+0000", 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013369599999999999 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488039+0000", 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067923580000000001 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.196 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488120+0000", 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0903999999999997e-05 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488205+0000", 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5637999999999996e-05 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.543917+0000", 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.055711636000000002 2023-10-09T04:27:47.197 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.543976+0000", 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8547e-05 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.544098+0000", 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012199199999999999 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.198 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83399 10.15 10:a8a18d93:::benchmark_data_smithi163_523108_object9383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.515179+0000", 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 205.58903838399999, 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.032424445, 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.199 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83399 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515179+0000", 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515179+0000", 2023-10-09T04:27:47.200 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515178+0000", 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515184+0000", 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6899999999999997e-06 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.201 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515192+0000", 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6840000000000002e-06 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515199+0000", 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6610000000000003e-06 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.202 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515251+0000", 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2537999999999997e-05 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515260+0000", 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4149999999999999e-06 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.203 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.482834+0000", 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96757368700000002 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.482968+0000", 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134244 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.483092+0000", 2023-10-09T04:27:47.204 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000124311 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488047+0000", 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049550740000000003 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488168+0000", 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120597 2023-10-09T04:27:47.205 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488256+0000", 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8319999999999995e-05 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.547435+0000", 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.059178808999999999 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.206 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.547493+0000", 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8366000000000003e-05 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.547604+0000", 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110287 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.207 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116378 10.e 10:7321e9ff:::benchmark_data_smithi163_523108_object42362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 98.437770491999999, 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.099838952, 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.208 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116378 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:27:47.209 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666450+0000", 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.14e-06 2023-10-09T04:27:47.210 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666456+0000", 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9830000000000001e-06 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666460+0000", 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.668e-06 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.211 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.672764+0000", 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006303594 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.672913+0000", 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014891500000000001 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T04:27:47.212 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.673054+0000", 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014072499999999999 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.680949+0000", 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078951679999999993 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.685946+0000", 2023-10-09T04:27:47.213 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049966689999999996 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701182+0000", 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015236826 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701198+0000", 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5494000000000001e-05 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.214 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766190+0000", 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0649925410000001 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766286+0000", 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5639999999999999e-05 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.215 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116382 10.e 10:746dc59f:::benchmark_data_smithi163_523108_object42366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.670407+0000", 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 98.433810778999998, 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0964046039999999, 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.216 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116382 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670407+0000", 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670407+0000", 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.217 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670405+0000", 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670415+0000", 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0526000000000001e-05 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.218 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670432+0000", 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6578999999999998e-05 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670441+0000", 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4690000000000003e-06 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.219 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676668+0000", 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062271150000000001 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676858+0000", 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018957699999999999 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.677005+0000", 2023-10-09T04:27:47.220 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014689999999999999 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.681628+0000", 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046230899999999998 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.685998+0000", 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00437039 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.221 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701238+0000", 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015239571 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701247+0000", 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4429999999999992e-06 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.222 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766739+0000", 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0654917580000001 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766811+0000", 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2170999999999997e-05 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.223 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116384 10.8 10:10a2b316:::benchmark_data_smithi163_523108_object42368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.676857+0000", 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 98.427360973999996, 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0946857999999999, 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116384 2023-10-09T04:27:47.224 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676857+0000", 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676857+0000", 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.225 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676855+0000", 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676860+0000", 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.51e-06 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.226 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676868+0000", 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5439999999999999e-06 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676875+0000", 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4760000000000003e-06 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676963+0000", 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7641999999999999e-05 2023-10-09T04:27:47.227 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.677085+0000", 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012232700000000001 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.677206+0000", 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012082999999999999 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.228 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.679648+0000", 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024424469999999999 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.682902+0000", 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003254112 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701396+0000", 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01849375 2023-10-09T04:27:47.229 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701407+0000", 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0718e-05 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.771443+0000", 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0700367690000001 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.230 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.771542+0000", 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8873e-05 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116395 10.18 10:1a07f0f6:::benchmark_data_smithi163_523108_object42379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.718986+0000", 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 98.385231473999994, 2023-10-09T04:27:47.231 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0974267150000001, 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116395 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718986+0000", 2023-10-09T04:27:47.232 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718986+0000", 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718984+0000", 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.233 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718995+0000", 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0963e-05 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719018+0000", 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3388999999999999e-05 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.234 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719029+0000", 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0668e-05 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719106+0000", 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7255999999999994e-05 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719121+0000", 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4649e-05 2023-10-09T04:27:47.235 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779188+0000", 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0600670510000001 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779314+0000", 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012573400000000001 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.236 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779443+0000", 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012905499999999999 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.815890+0000", 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.036447307999999998 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.815901+0000", 2023-10-09T04:27:47.237 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0958000000000001e-05 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816203+0000", 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00030209200000000001 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816232+0000", 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8938999999999999e-05 2023-10-09T04:27:47.238 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816272+0000", 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0194000000000002e-05 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816413+0000", 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000140457 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.239 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116396 10.18 10:1aeefe62:::benchmark_data_smithi163_523108_object42380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.721996+0000", 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 98.382221798000003, 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0999008619999999, 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.240 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116396 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.721996+0000", 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.721996+0000", 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.241 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.721994+0000", 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722004+0000", 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0215e-05 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.242 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722026+0000", 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1446000000000001e-05 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722034+0000", 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4430000000000003e-06 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.243 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722094+0000", 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9790999999999997e-05 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722108+0000", 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.416e-05 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.781401+0000", 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0592935349999999 2023-10-09T04:27:47.244 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.781533+0000", 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013104699999999999 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.781660+0000", 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012787000000000001 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.245 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.815940+0000", 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034279653 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816453+0000", 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00051292099999999995 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816487+0000", 2023-10-09T04:27:47.246 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3652000000000002e-05 2023-10-09T04:27:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.247 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.821795+0000", 2023-10-09T04:27:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053085459999999999 2023-10-09T04:27:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.249 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.821830+0000", 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4980000000000001e-05 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.821897+0000", 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6478999999999998e-05 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.250 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116399 10.8 10:127a1cac:::benchmark_data_smithi163_523108_object42383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.729156+0000", 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 98.375061651999999, 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.05177231, 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116399 2023-10-09T04:27:47.251 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729156+0000", 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729156+0000", 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.252 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729153+0000", 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999971 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729166+0000", 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.29e-05 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.253 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729191+0000", 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5306000000000001e-05 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729203+0000", 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1882000000000001e-05 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729281+0000", 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7956999999999999e-05 2023-10-09T04:27:47.254 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729299+0000", 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8009e-05 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.771878+0000", 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0425782130000001 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.255 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.772010+0000", 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013244900000000001 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.772131+0000", 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121487 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.256 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779101+0000", 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069697759999999996 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779135+0000", 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4143999999999997e-05 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779167+0000", 2023-10-09T04:27:47.257 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1587000000000001e-05 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.780705+0000", 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015378309999999999 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.780772+0000", 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7465000000000005e-05 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.258 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.780928+0000", 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015596100000000001 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:120723 10.8 10:160fe9bf:::benchmark_data_smithi163_523108_object46707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:20.337171+0000", 2023-10-09T04:27:47.259 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 86.767046773000004, 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0261010859999999, 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 120723 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.260 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337171+0000", 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337171+0000", 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337169+0000", 2023-10-09T04:27:47.261 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337177+0000", 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5129999999999999e-06 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337190+0000", 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3644e-05 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.262 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337199+0000", 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1110000000000008e-06 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337249+0000", 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0102000000000002e-05 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.263 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337420+0000", 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017113799999999999 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337547+0000", 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127572 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.339814+0000", 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00226677 2023-10-09T04:27:47.264 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.340029+0000", 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021489799999999999 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553248+0000", 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.213219086 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.265 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553264+0000", 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5498999999999998e-05 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.363189+0000", 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.80992576299999997 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.266 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.363272+0000", 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2527000000000005e-05 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:120726 10.15 10:a8ac2898:::benchmark_data_smithi163_523108_object46710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:20.352749+0000", 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 86.751468603000006, 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0461038250000001, 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:27:47.267 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 120726 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352749+0000", 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.268 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352749+0000", 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352747+0000", 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.269 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352755+0000", 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9699999999999999e-06 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553214+0000", 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20045837799999999 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553225+0000", 2023-10-09T04:27:47.270 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1096e-05 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.557694+0000", 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044695400000000001 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.557706+0000", 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.207e-05 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.271 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.378627+0000", 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.82092094299999996 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.378725+0000", 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7714000000000007e-05 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:27:47.272 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.378826+0000", 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101026 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.382312+0000", 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034861559999999998 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.383401+0000", 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001088889 2023-10-09T04:27:47.273 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.384130+0000", 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072926299999999996 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.398699+0000", 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014568129000000001 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.274 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.398758+0000", 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9379000000000002e-05 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.398853+0000", 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4778999999999995e-05 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.275 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:27:47.276 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:27:47.276 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:27:47.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:47 smithi163 ceph-mon[213841]: pgmap v1334: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-10-09T04:27:47.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:47 smithi163 ceph-mon[221067]: pgmap v1334: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-10-09T04:27:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:47.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:47.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:47.378 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:47.469 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:47.480 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 18 16 35391 35375 122.747 97.4375 0.0102273 0.00814204 2023-10-09T04:27:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:47 smithi181 ceph-mon[196979]: pgmap v1334: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-10-09T04:27:47.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:27:48.370 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:48.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:48.478 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:48.480 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 19 16 36755 36739 120.774 85.25 0.00992495 0.00827464 2023-10-09T04:27:48.528 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:48.617 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:48.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:48.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:48.714 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:48 smithi163 ceph-mon[213841]: pgmap v1335: 119 pgs: 119 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T04:27:48.714 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:48 smithi163 ceph-mon[221067]: pgmap v1335: 119 pgs: 119 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T04:27:48.723 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:48.781 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:48.853 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:27:49.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:48 smithi181 ceph-mon[196979]: pgmap v1335: 119 pgs: 119 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-09T04:27:49.098 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:49.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:49.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:49.215 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:49.269 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:49.310 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:49.424 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:49.430 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:49.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:49.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:27:49.480559+0000 min lat: 0.00118268 max lat: 0.360884 avg lat: 0.00837902 2023-10-09T04:27:49.482 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:27:49.482 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 20 16 38197 38181 119.242 90.125 0.0111785 0.00837902 2023-10-09T04:27:49.534 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:49.614 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:49.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:49.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:49.916 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:49.917 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:49.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:49.974 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:49.980 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:50.046 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:50.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:50.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:50.152 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:27:50.222 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:50.480 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 21 16 39773 39757 118.254 98.5 0.00482369 0.00845086 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[213841]: pgmap v1336: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[221067]: pgmap v1336: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T04:27:50.916 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T04:27:50.917 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T04:27:50.917 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:27:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:27:50] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2023-10-09T04:27:51.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:50 smithi181 ceph-mon[196979]: pgmap v1336: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T04:27:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T04:27:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T04:27:51.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-10-09T04:27:51.291 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:51.480 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 22 16 41202 41186 116.939 89.3125 0.00559926 0.00854539 2023-10-09T04:27:51.984 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:27:51.984 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:27:51.984 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:27:51.990 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: failed to encode map e514 with expected crc 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: failed to encode map e514 with expected crc 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T04:27:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1621625310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:52.026 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:27:52.063 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:27:52.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: failed to encode map e514 with expected crc 2023-10-09T04:27:52.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: failed to encode map e514 with expected crc 2023-10-09T04:27:52.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T04:27:52.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T04:27:52.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T04:27:52.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1621625310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: failed to encode map e514 with expected crc 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: failed to encode map e514 with expected crc 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-10-09T04:27:52.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T04:27:52.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1621625310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:27:52.094 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:27:52.131 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.141 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:27:52.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 23 16 42404 42388 115.119 75.125 0.00834887 0.00868157 2023-10-09T04:27:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:52 smithi181 ceph-mon[196979]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:52 smithi181 ceph-mon[196979]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T04:27:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:52 smithi181 ceph-mon[196979]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:52 smithi181 ceph-mon[196979]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:52 smithi181 ceph-mon[196979]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:52 smithi181 ceph-mon[196979]: pgmap v1339: 119 pgs: 119 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T04:27:53.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[213841]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[213841]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T04:27:53.085 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[213841]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[213841]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[213841]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[213841]: pgmap v1339: 119 pgs: 119 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[221067]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[221067]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[221067]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[221067]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[221067]: failed to encode map e515 with expected crc 2023-10-09T04:27:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:52 smithi163 ceph-mon[221067]: pgmap v1339: 119 pgs: 119 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T04:27:53.189 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:27:53.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:53.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:27:53.301 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:53.374 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:53.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 24 16 43853 43837 114.096 90.5625 0.00865108 0.00875978 2023-10-09T04:27:53.642 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:53.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:53.745 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:54.009 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:54.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:54.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:54.209 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:27:54.263 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:54.345 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:27:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:54.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:27:54.454 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:54.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 25 16 45240 45224 112.999 86.6875 0.0103971 0.00884302 2023-10-09T04:27:54.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:54 smithi181 ceph-mon[196979]: pgmap v1340: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[213841]: pgmap v1340: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.525 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:54.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:54.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:27:54.526 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:54 smithi163 ceph-mon[221067]: pgmap v1340: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T04:27:54.538 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:54.756 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:54.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:54.867 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:54.930 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:55.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:55.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:55.283 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:27:55.305 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T04:27:55.317 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:55.383 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:55.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 26 16 46590 46574 111.898 84.375 0.00738293 0.00893178 2023-10-09T04:27:55.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3272206771' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:27:55.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:55 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:27:55.485 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/855821013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:55.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3272206771' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:27:55.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:55 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:27:55.485 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/855821013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:55.541 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:55.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:27:55.601 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:27:55.653 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:27:55.700 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3272206771' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:27:55.700 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:55 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:27:55.700 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/855821013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:27:55.705 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:55.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:27:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:55.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:27:55.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:27:55.918 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:55.996 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:27:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:56.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:27:56.108 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:56.158 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:56.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:56.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:56.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:56.350 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:56.420 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:56.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 27 16 47892 47876 110.768 81.375 0.0035143 0.00900497 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/855821013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: failed to encode map e516 with expected crc 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: failed to encode map e516 with expected crc 2023-10-09T04:27:56.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[221067]: pgmap v1342: 119 pgs: 119 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/855821013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: failed to encode map e516 with expected crc 2023-10-09T04:27:56.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: failed to encode map e516 with expected crc 2023-10-09T04:27:56.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:56 smithi163 ceph-mon[213841]: pgmap v1342: 119 pgs: 119 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/855821013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: failed to encode map e516 with expected crc 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: failed to encode map e516 with expected crc 2023-10-09T04:27:56.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:56 smithi181 ceph-mon[196979]: pgmap v1342: 119 pgs: 119 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T04:27:57.210 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:57.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:57.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:57.367 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:57.437 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:57.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:57.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 28 16 49221 49205 109.779 83.0625 0.0135987 0.00910397 2023-10-09T04:27:57.529 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:57.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:27:57.599 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:57.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:27:57.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:57.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:27:57.684 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:27:57.684 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:27:57.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:27:57.846 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:58.481 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 29 16 50678 50662 109.133 91.0625 0.00594712 0.00915881 2023-10-09T04:27:58.493 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:27:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:58.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:27:58.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:27:58.667 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:59.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:27:58 smithi181 ceph-mon[196979]: pgmap v1343: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:27:59.060 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:27:58 smithi163 ceph-mon[213841]: pgmap v1343: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:27:59.061 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:27:58 smithi163 ceph-mon[221067]: pgmap v1343: 119 pgs: 119 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:27:59.086 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:27:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:59.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:27:59.196 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:27:59.226 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:27:59.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:27:59.482 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 30 16 51720 51704 107.666 65.125 0.00471929 0.00925868 2023-10-09T04:27:59.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:27:59.957 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:27:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:27:59.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:00.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:00.130 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:00.204 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:00.205 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:00.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:00.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:00.312 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:00.388 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:00.482 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 31 16 52993 52977 106.759 79.5625 0.0290755 0.00936166 2023-10-09T04:28:00.484 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:00.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:00.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:00.632 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:00.639 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:28:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:28:00] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2023-10-09T04:28:00.702 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:00.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:00.810 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:00.870 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:00.956 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:00.981 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:00.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:01.018 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:01.058 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:01.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:01.102 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:01.137 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:01.143 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:01.187 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:01.220 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:01.223 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:01.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:01.265 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:01.276 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.277 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:28:01.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:01 smithi163 ceph-mon[221067]: pgmap v1344: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T04:28:01.306 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2436464734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:01.306 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:01 smithi163 ceph-mon[213841]: pgmap v1344: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T04:28:01.306 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2436464734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:01.325 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:01.390 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:01.482 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 32 16 54289 54273 105.954 81 0.00472485 0.00943307 2023-10-09T04:28:01.487 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:01.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:01.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:01 smithi181 ceph-mon[196979]: pgmap v1344: 119 pgs: 119 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T04:28:01.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2436464734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:01.593 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:01.653 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:02.326 INFO:tasks.thrashosds.thrasher:in_osds: [1, 5, 7, 3, 6, 0, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:28:02.326 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:28:02.326 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:28:02.327 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:28:02.487 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 33 16 55582 55566 105.177 80.8125 0.00657402 0.0095023 2023-10-09T04:28:02.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:02.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:02.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:02.663 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:02.707 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:02.750 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:02.822 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:02.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:02.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:03.010 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:03.264 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:03.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:03.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:03 smithi163 ceph-mon[221067]: pgmap v1345: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T04:28:03.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:03 smithi163 ceph-mon[213841]: pgmap v1345: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T04:28:03.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:03.466 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:03.487 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 34 16 56773 56757 104.274 74.4375 0.0186852 0.00958351 2023-10-09T04:28:03.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:03 smithi181 ceph-mon[196979]: pgmap v1345: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T04:28:03.680 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:03.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:03.789 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:03.816 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:04.042 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:04.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:04.135 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:28:04.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:04.203 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:04.296 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:04.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2320265166' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2320265166' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:04.407 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:04.486 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:04.487 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 35 16 57726 57710 102.996 59.5625 0.00950322 0.00970366 2023-10-09T04:28:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2320265166' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:04.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:04.681 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:28:04.834 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:05.008 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:05.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:05.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:05.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:05.182 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:05.297 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:05.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:05.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: pgmap v1346: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:28:05.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:05.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: failed to encode map e517 with expected crc 2023-10-09T04:28:05.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: failed to encode map e517 with expected crc 2023-10-09T04:28:05.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:05.379 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T04:28:05.380 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:05.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: pgmap v1346: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:28:05.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:05.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: failed to encode map e517 with expected crc 2023-10-09T04:28:05.382 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: failed to encode map e517 with expected crc 2023-10-09T04:28:05.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:05.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T04:28:05.383 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:05.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:05.450 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: pgmap v1346: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:28:05.451 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:05.451 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: failed to encode map e517 with expected crc 2023-10-09T04:28:05.451 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: failed to encode map e517 with expected crc 2023-10-09T04:28:05.451 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:05.451 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T04:28:05.451 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:05.454 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:05.488 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 36 16 58712 58696 101.846 61.625 0.0110767 0.00981361 2023-10-09T04:28:05.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:05.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:05.630 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:05.696 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:28:05.696 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:28:05.696 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:28:05.696 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:28:05.696 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:28:05.702 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:05.775 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:06.488 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 37 16 59593 59577 100.582 55.0625 0.00591222 0.00993722 2023-10-09T04:28:06.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:06.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:06.705 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:06.753 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:06.795 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:06.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:06.902 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:06.949 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:06.980 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:28:07.030 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:07.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:07.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:07.106 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:28:07.137 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:07.180 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:07.244 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:07.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:07.300 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:07 smithi163 ceph-mon[213841]: pgmap v1348: 119 pgs: 119 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:28:07.300 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:07 smithi163 ceph-mon[221067]: pgmap v1348: 119 pgs: 119 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:28:07.352 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:07.488 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 38 16 60594 60578 99.5814 62.5625 0.00475962 0.0100362 2023-10-09T04:28:07.513 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:07.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:07 smithi181 ceph-mon[196979]: pgmap v1348: 119 pgs: 119 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:28:07.771 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:07.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:07.883 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:07.964 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:08.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:08.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:08.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:08.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:08.444 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:08.489 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 39 16 61845 61829 99.0299 78.1875 0.0138709 0.0100934 2023-10-09T04:28:08.516 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:08.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:08.623 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:08.716 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:08.717 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:08.832 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:08.934 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:08.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:09.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:08 smithi181 ceph-mon[196979]: pgmap v1349: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:28:09.051 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:09.052 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:08 smithi163 ceph-mon[221067]: pgmap v1349: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:28:09.053 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:08 smithi163 ceph-mon[213841]: pgmap v1349: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:28:09.149 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:09.205 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:28:09.227 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:28:09.489 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:28:09.489913+0000 min lat: 0.00118268 max lat: 0.360884 avg lat: 0.0101873 2023-10-09T04:28:09.489 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:28:09.489 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 40 16 62839 62823 98.1076 62.125 0.0201032 0.0101873 2023-10-09T04:28:09.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:09.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:09.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:09.652 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:09.653 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:09.669 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:09.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:09.736 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:09.743 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:28:09.744 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:28:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1817191064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:10.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1817191064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:10.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1817191064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:10.489 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 41 16 64063 64047 97.5806 76.5 0.00763351 0.0102441 2023-10-09T04:28:10.760 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1349,"stamp":"2023-10-09T04:28:08.016168+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3845003168,"num_objects":58685,"num_object_clones":0,"num_object_copies":176055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":58722,"num_write_kb":3754333,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":1847710,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63707,"ondisk_log_size":63707,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17766156,"kb_used_data":11147980,"kb_used_omap":126,"kb_used_meta":6617985,"kb_avail":732195060,"statfs":{"total":767960285184,"available":749767741440,"internally_reserved":0,"allocated":11415531520,"data_stored":11400322726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6776817275},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,11,4,0,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":792395776,"num_objects":12091,"num_object_clones":0,"num_object_copies":36273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12091,"num_write_kb":773824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002868"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":689,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284583+0000","last_change":"2023-10-09T04:26:24.679564+0000","last_active":"2023-10-09T04:27:55.284583+0000","last_peered":"2023-10-09T04:27:55.284583+0000","last_clean":"2023-10-09T04:27:55.284583+0000","last_became_active":"2023-10-09T04:26:24.679142+0000","last_became_peered":"2023-10-09T04:26:24.679142+0000","last_unstale":"2023-10-09T04:27:55.284583+0000","last_undegraded":"2023-10-09T04:27:55.284583+0000","last_fullsized":"2023-10-09T04:27:55.284583+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:37.095828+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":102,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268416+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:27:55.268416+0000","last_peered":"2023-10-09T04:27:55.268416+0000","last_clean":"2023-10-09T04:27:55.268416+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:27:55.268416+0000","last_undegraded":"2023-10-09T04:27:55.268416+0000","last_fullsized":"2023-10-09T04:27:55.268416+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":79,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269873+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:27:55.269873+0000","last_peered":"2023-10-09T04:27:55.269873+0000","last_clean":"2023-10-09T04:27:55.269873+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:27:55.269873+0000","last_undegraded":"2023-10-09T04:27:55.269873+0000","last_fullsized":"2023-10-09T04:27:55.269873+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":714,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284937+0000","last_change":"2023-10-09T04:24:58.734627+0000","last_active":"2023-10-09T04:27:55.284937+0000","last_peered":"2023-10-09T04:27:55.284937+0000","last_clean":"2023-10-09T04:27:55.284937+0000","last_became_active":"2023-10-09T04:24:58.734248+0000","last_became_peered":"2023-10-09T04:24:58.734248+0000","last_unstale":"2023-10-09T04:27:55.284937+0000","last_undegraded":"2023-10-09T04:27:55.284937+0000","last_fullsized":"2023-10-09T04:27:55.284937+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:59.934563+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":524,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.279627+0000","last_change":"2023-10-09T04:24:56.158228+0000","last_active":"2023-10-09T04:27:55.279627+0000","last_peered":"2023-10-09T04:27:55.279627+0000","last_clean":"2023-10-09T04:27:55.279627+0000","last_became_active":"2023-10-09T04:24:56.157946+0000","last_became_peered":"2023-10-09T04:24:56.157946+0000","last_unstale":"2023-10-09T04:27:55.279627+0000","last_undegraded":"2023-10-09T04:27:55.279627+0000","last_fullsized":"2023-10-09T04:27:55.279627+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:37.279836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277310+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:27:55.277310+0000","last_peered":"2023-10-09T04:27:55.277310+0000","last_clean":"2023-10-09T04:27:55.277310+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:27:55.277310+0000","last_undegraded":"2023-10-09T04:27:55.277310+0000","last_fullsized":"2023-10-09T04:27:55.277310+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":509,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268275+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:27:55.268275+0000","last_peered":"2023-10-09T04:27:55.268275+0000","last_clean":"2023-10-09T04:27:55.268275+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:27:55.268275+0000","last_undegraded":"2023-10-09T04:27:55.268275+0000","last_fullsized":"2023-10-09T04:27:55.268275+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.592966+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:27:56.592966+0000","last_peered":"2023-10-09T04:27:56.592966+0000","last_clean":"2023-10-09T04:27:56.592966+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:27:56.592966+0000","last_undegraded":"2023-10-09T04:27:56.592966+0000","last_fullsized":"2023-10-09T04:27:56.592966+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":480,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285635+0000","last_change":"2023-10-09T04:26:09.747631+0000","last_active":"2023-10-09T04:27:55.285635+0000","last_peered":"2023-10-09T04:27:55.285635+0000","last_clean":"2023-10-09T04:27:55.285635+0000","last_became_active":"2023-10-09T04:26:09.747326+0000","last_became_peered":"2023-10-09T04:26:09.747326+0000","last_unstale":"2023-10-09T04:27:55.285635+0000","last_undegraded":"2023-10-09T04:27:55.285635+0000","last_fullsized":"2023-10-09T04:27:55.285635+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.315346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":916,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269585+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:27:55.269585+0000","last_peered":"2023-10-09T04:27:55.269585+0000","last_clean":"2023-10-09T04:27:55.269585+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:27:55.269585+0000","last_undegraded":"2023-10-09T04:27:55.269585+0000","last_fullsized":"2023-10-09T04:27:55.269585+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.593215+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:27:56.593215+0000","last_peered":"2023-10-09T04:27:56.593215+0000","last_clean":"2023-10-09T04:27:56.593215+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:27:56.593215+0000","last_undegraded":"2023-10-09T04:27:56.593215+0000","last_fullsized":"2023-10-09T04:27:56.593215+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":257,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.276974+0000","last_change":"2023-10-09T04:27:51.852635+0000","last_active":"2023-10-09T04:27:55.276974+0000","last_peered":"2023-10-09T04:27:55.276974+0000","last_clean":"2023-10-09T04:27:55.276974+0000","last_became_active":"2023-10-09T04:27:51.852385+0000","last_became_peered":"2023-10-09T04:27:51.852385+0000","last_unstale":"2023-10-09T04:27:55.276974+0000","last_undegraded":"2023-10-09T04:27:55.276974+0000","last_fullsized":"2023-10-09T04:27:55.276974+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:54.313846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":823,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268870+0000","last_change":"2023-10-09T04:27:45.967966+0000","last_active":"2023-10-09T04:27:55.268870+0000","last_peered":"2023-10-09T04:27:55.268870+0000","last_clean":"2023-10-09T04:27:55.268870+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:27:55.268870+0000","last_undegraded":"2023-10-09T04:27:55.268870+0000","last_fullsized":"2023-10-09T04:27:55.268870+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:45.967906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:45.967906+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:06.528314+0000","scrub_duration":0.0012842089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":476,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285417+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:27:55.285417+0000","last_peered":"2023-10-09T04:27:55.285417+0000","last_clean":"2023-10-09T04:27:55.285417+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:27:55.285417+0000","last_undegraded":"2023-10-09T04:27:55.285417+0000","last_fullsized":"2023-10-09T04:27:55.285417+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":160,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.279585+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:27:55.279585+0000","last_peered":"2023-10-09T04:27:55.279585+0000","last_clean":"2023-10-09T04:27:55.279585+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:27:55.279585+0000","last_undegraded":"2023-10-09T04:27:55.279585+0000","last_fullsized":"2023-10-09T04:27:55.279585+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":721,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285567+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:27:55.285567+0000","last_peered":"2023-10-09T04:27:55.285567+0000","last_clean":"2023-10-09T04:27:55.285567+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:27:55.285567+0000","last_undegraded":"2023-10-09T04:27:55.285567+0000","last_fullsized":"2023-10-09T04:27:55.285567+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":591,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269945+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:27:55.269945+0000","last_peered":"2023-10-09T04:27:55.269945+0000","last_clean":"2023-10-09T04:27:55.269945+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:27:55.269945+0000","last_undegraded":"2023-10-09T04:27:55.269945+0000","last_fullsized":"2023-10-09T04:27:55.269945+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284706+0000","last_change":"2023-10-09T04:27:51.852273+0000","last_active":"2023-10-09T04:27:55.284706+0000","last_peered":"2023-10-09T04:27:55.284706+0000","last_clean":"2023-10-09T04:27:55.284706+0000","last_became_active":"2023-10-09T04:27:51.851992+0000","last_became_peered":"2023-10-09T04:27:51.851992+0000","last_unstale":"2023-10-09T04:27:55.284706+0000","last_undegraded":"2023-10-09T04:27:55.284706+0000","last_fullsized":"2023-10-09T04:27:55.284706+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:49.300670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":695,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T04:27:51.882479+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:27:51.882479+0000","last_peered":"2023-10-09T04:27:51.882479+0000","last_clean":"2023-10-09T04:27:51.882479+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:27:51.882479+0000","last_undegraded":"2023-10-09T04:27:51.882479+0000","last_fullsized":"2023-10-09T04:27:51.882479+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":92,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269374+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:27:55.269374+0000","last_peered":"2023-10-09T04:27:55.269374+0000","last_clean":"2023-10-09T04:27:55.269374+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:27:55.269374+0000","last_undegraded":"2023-10-09T04:27:55.269374+0000","last_fullsized":"2023-10-09T04:27:55.269374+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268822+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:27:55.268822+0000","last_peered":"2023-10-09T04:27:55.268822+0000","last_clean":"2023-10-09T04:27:55.268822+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:27:55.268822+0000","last_undegraded":"2023-10-09T04:27:55.268822+0000","last_fullsized":"2023-10-09T04:27:55.268822+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":549,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.279154+0000","last_change":"2023-10-09T04:27:02.906226+0000","last_active":"2023-10-09T04:27:55.279154+0000","last_peered":"2023-10-09T04:27:55.279154+0000","last_clean":"2023-10-09T04:27:55.279154+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:27:55.279154+0000","last_undegraded":"2023-10-09T04:27:55.279154+0000","last_fullsized":"2023-10-09T04:27:55.279154+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:21.516499+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":560,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267968+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:27:55.267968+0000","last_peered":"2023-10-09T04:27:55.267968+0000","last_clean":"2023-10-09T04:27:55.267968+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:27:55.267968+0000","last_undegraded":"2023-10-09T04:27:55.267968+0000","last_fullsized":"2023-10-09T04:27:55.267968+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":292,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268682+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:27:55.268682+0000","last_peered":"2023-10-09T04:27:55.268682+0000","last_clean":"2023-10-09T04:27:55.268682+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:27:55.268682+0000","last_undegraded":"2023-10-09T04:27:55.268682+0000","last_fullsized":"2023-10-09T04:27:55.268682+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":681,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284635+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:27:55.284635+0000","last_peered":"2023-10-09T04:27:55.284635+0000","last_clean":"2023-10-09T04:27:55.284635+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:27:55.284635+0000","last_undegraded":"2023-10-09T04:27:55.284635+0000","last_fullsized":"2023-10-09T04:27:55.284635+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277689+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:27:55.277689+0000","last_peered":"2023-10-09T04:27:55.277689+0000","last_clean":"2023-10-09T04:27:55.277689+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:27:55.277689+0000","last_undegraded":"2023-10-09T04:27:55.277689+0000","last_fullsized":"2023-10-09T04:27:55.277689+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":495,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285403+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:27:55.285403+0000","last_peered":"2023-10-09T04:27:55.285403+0000","last_clean":"2023-10-09T04:27:55.285403+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:27:55.285403+0000","last_undegraded":"2023-10-09T04:27:55.285403+0000","last_fullsized":"2023-10-09T04:27:55.285403+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":662,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.278467+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:27:55.278467+0000","last_peered":"2023-10-09T04:27:55.278467+0000","last_clean":"2023-10-09T04:27:55.278467+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:27:55.278467+0000","last_undegraded":"2023-10-09T04:27:55.278467+0000","last_fullsized":"2023-10-09T04:27:55.278467+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267794+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:27:55.267794+0000","last_peered":"2023-10-09T04:27:55.267794+0000","last_clean":"2023-10-09T04:27:55.267794+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:27:55.267794+0000","last_undegraded":"2023-10-09T04:27:55.267794+0000","last_fullsized":"2023-10-09T04:27:55.267794+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.593174+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:27:56.593174+0000","last_peered":"2023-10-09T04:27:56.593174+0000","last_clean":"2023-10-09T04:27:56.593174+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:27:56.593174+0000","last_undegraded":"2023-10-09T04:27:56.593174+0000","last_fullsized":"2023-10-09T04:27:56.593174+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":681,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285115+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:27:55.285115+0000","last_peered":"2023-10-09T04:27:55.285115+0000","last_clean":"2023-10-09T04:27:55.285115+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:27:55.285115+0000","last_undegraded":"2023-10-09T04:27:55.285115+0000","last_fullsized":"2023-10-09T04:27:55.285115+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":132,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277844+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:27:55.277844+0000","last_peered":"2023-10-09T04:27:55.277844+0000","last_clean":"2023-10-09T04:27:55.277844+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:27:55.277844+0000","last_undegraded":"2023-10-09T04:27:55.277844+0000","last_fullsized":"2023-10-09T04:27:55.277844+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.278036+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:27:55.278036+0000","last_peered":"2023-10-09T04:27:55.278036+0000","last_clean":"2023-10-09T04:27:55.278036+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:27:55.278036+0000","last_undegraded":"2023-10-09T04:27:55.278036+0000","last_fullsized":"2023-10-09T04:27:55.278036+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"517'3563","reported_seq":3580,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.145669+0000","last_change":"2023-10-09T04:27:20.720040+0000","last_active":"2023-10-09T04:28:05.145669+0000","last_peered":"2023-10-09T04:28:05.145669+0000","last_clean":"2023-10-09T04:28:05.145669+0000","last_became_active":"2023-10-09T04:27:20.719715+0000","last_became_peered":"2023-10-09T04:27:20.719715+0000","last_unstale":"2023-10-09T04:28:05.145669+0000","last_undegraded":"2023-10-09T04:28:05.145669+0000","last_fullsized":"2023-10-09T04:28:05.145669+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3563,"log_dups_size":0,"ondisk_log_size":3563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:37.979204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233504768,"num_objects":3563,"num_object_clones":0,"num_object_copies":10689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3563,"num_write_kb":228032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":959,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269657+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:27:55.269657+0000","last_peered":"2023-10-09T04:27:55.269657+0000","last_clean":"2023-10-09T04:27:55.269657+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:27:55.269657+0000","last_undegraded":"2023-10-09T04:27:55.269657+0000","last_fullsized":"2023-10-09T04:27:55.269657+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":561,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269209+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:27:55.269209+0000","last_peered":"2023-10-09T04:27:55.269209+0000","last_clean":"2023-10-09T04:27:55.269209+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:27:55.269209+0000","last_undegraded":"2023-10-09T04:27:55.269209+0000","last_fullsized":"2023-10-09T04:27:55.269209+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":478,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.283226+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:27:55.283226+0000","last_peered":"2023-10-09T04:27:55.283226+0000","last_clean":"2023-10-09T04:27:55.283226+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:27:55.283226+0000","last_undegraded":"2023-10-09T04:27:55.283226+0000","last_fullsized":"2023-10-09T04:27:55.283226+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"517'3740","reported_seq":3759,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:06.590516+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:28:06.590516+0000","last_peered":"2023-10-09T04:28:06.590516+0000","last_clean":"2023-10-09T04:28:06.590516+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:28:06.590516+0000","last_undegraded":"2023-10-09T04:28:06.590516+0000","last_fullsized":"2023-10-09T04:28:06.590516+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3740,"log_dups_size":0,"ondisk_log_size":3740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245104640,"num_objects":3740,"num_object_clones":0,"num_object_copies":11220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3740,"num_write_kb":239360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":750,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277092+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:27:55.277092+0000","last_peered":"2023-10-09T04:27:55.277092+0000","last_clean":"2023-10-09T04:27:55.277092+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:27:55.277092+0000","last_undegraded":"2023-10-09T04:27:55.277092+0000","last_fullsized":"2023-10-09T04:27:55.277092+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":537,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268764+0000","last_change":"2023-10-09T04:24:56.212579+0000","last_active":"2023-10-09T04:27:55.268764+0000","last_peered":"2023-10-09T04:27:55.268764+0000","last_clean":"2023-10-09T04:27:55.268764+0000","last_became_active":"2023-10-09T04:24:56.184403+0000","last_became_peered":"2023-10-09T04:24:56.184403+0000","last_unstale":"2023-10-09T04:27:55.268764+0000","last_undegraded":"2023-10-09T04:27:55.268764+0000","last_fullsized":"2023-10-09T04:27:55.268764+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:10.486526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267941+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:27:55.267941+0000","last_peered":"2023-10-09T04:27:55.267941+0000","last_clean":"2023-10-09T04:27:55.267941+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:27:55.267941+0000","last_undegraded":"2023-10-09T04:27:55.267941+0000","last_fullsized":"2023-10-09T04:27:55.267941+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"517'3614","reported_seq":3632,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.174115+0000","last_change":"2023-10-09T04:27:20.720676+0000","last_active":"2023-10-09T04:28:05.174115+0000","last_peered":"2023-10-09T04:28:05.174115+0000","last_clean":"2023-10-09T04:28:05.174115+0000","last_became_active":"2023-10-09T04:27:20.720342+0000","last_became_peered":"2023-10-09T04:27:20.720342+0000","last_unstale":"2023-10-09T04:28:05.174115+0000","last_undegraded":"2023-10-09T04:28:05.174115+0000","last_fullsized":"2023-10-09T04:28:05.174115+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3614,"log_dups_size":0,"ondisk_log_size":3614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:37.485070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236847104,"num_objects":3614,"num_object_clones":0,"num_object_copies":10842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3614,"num_write_kb":231296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":721,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285520+0000","last_change":"2023-10-09T04:24:58.648512+0000","last_active":"2023-10-09T04:27:55.285520+0000","last_peered":"2023-10-09T04:27:55.285520+0000","last_clean":"2023-10-09T04:27:55.285520+0000","last_became_active":"2023-10-09T04:24:58.648143+0000","last_became_peered":"2023-10-09T04:24:58.648143+0000","last_unstale":"2023-10-09T04:27:55.285520+0000","last_undegraded":"2023-10-09T04:27:55.285520+0000","last_fullsized":"2023-10-09T04:27:55.285520+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:26.096956+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":488,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269806+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:27:55.269806+0000","last_peered":"2023-10-09T04:27:55.269806+0000","last_clean":"2023-10-09T04:27:55.269806+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:27:55.269806+0000","last_undegraded":"2023-10-09T04:27:55.269806+0000","last_fullsized":"2023-10-09T04:27:55.269806+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":143,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.265695+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:27:55.265695+0000","last_peered":"2023-10-09T04:27:55.265695+0000","last_clean":"2023-10-09T04:27:55.265695+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:27:55.265695+0000","last_undegraded":"2023-10-09T04:27:55.265695+0000","last_fullsized":"2023-10-09T04:27:55.265695+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"517'3602","reported_seq":3620,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.158312+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:28:05.158312+0000","last_peered":"2023-10-09T04:28:05.158312+0000","last_clean":"2023-10-09T04:28:05.158312+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:28:05.158312+0000","last_undegraded":"2023-10-09T04:28:05.158312+0000","last_fullsized":"2023-10-09T04:28:05.158312+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3602,"log_dups_size":0,"ondisk_log_size":3602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236060672,"num_objects":3602,"num_object_clones":0,"num_object_copies":10806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3602,"num_write_kb":230528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":773,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.271240+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:27:55.271240+0000","last_peered":"2023-10-09T04:27:55.271240+0000","last_clean":"2023-10-09T04:27:55.271240+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:27:55.271240+0000","last_undegraded":"2023-10-09T04:27:55.271240+0000","last_fullsized":"2023-10-09T04:27:55.271240+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":516,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285196+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:27:55.285196+0000","last_peered":"2023-10-09T04:27:55.285196+0000","last_clean":"2023-10-09T04:27:55.285196+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:27:55.285196+0000","last_undegraded":"2023-10-09T04:27:55.285196+0000","last_fullsized":"2023-10-09T04:27:55.285196+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277967+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:27:55.277967+0000","last_peered":"2023-10-09T04:27:55.277967+0000","last_clean":"2023-10-09T04:27:55.277967+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:27:55.277967+0000","last_undegraded":"2023-10-09T04:27:55.277967+0000","last_fullsized":"2023-10-09T04:27:55.277967+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"517'3685","reported_seq":3703,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.149973+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:28:05.149973+0000","last_peered":"2023-10-09T04:28:05.149973+0000","last_clean":"2023-10-09T04:28:05.149973+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:28:05.149973+0000","last_undegraded":"2023-10-09T04:28:05.149973+0000","last_fullsized":"2023-10-09T04:28:05.149973+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3685,"log_dups_size":0,"ondisk_log_size":3685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241500160,"num_objects":3685,"num_object_clones":0,"num_object_copies":11055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3685,"num_write_kb":235840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":980,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267604+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:27:55.267604+0000","last_peered":"2023-10-09T04:27:55.267604+0000","last_clean":"2023-10-09T04:27:55.267604+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:27:55.267604+0000","last_undegraded":"2023-10-09T04:27:55.267604+0000","last_fullsized":"2023-10-09T04:27:55.267604+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":152,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277889+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:27:55.277889+0000","last_peered":"2023-10-09T04:27:55.277889+0000","last_clean":"2023-10-09T04:27:55.277889+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:27:55.277889+0000","last_undegraded":"2023-10-09T04:27:55.277889+0000","last_fullsized":"2023-10-09T04:27:55.277889+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269284+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:27:55.269284+0000","last_peered":"2023-10-09T04:27:55.269284+0000","last_clean":"2023-10-09T04:27:55.269284+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:27:55.269284+0000","last_undegraded":"2023-10-09T04:27:55.269284+0000","last_fullsized":"2023-10-09T04:27:55.269284+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"517'3765","reported_seq":3784,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:06.568054+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:28:06.568054+0000","last_peered":"2023-10-09T04:28:06.568054+0000","last_clean":"2023-10-09T04:28:06.568054+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:28:06.568054+0000","last_undegraded":"2023-10-09T04:28:06.568054+0000","last_fullsized":"2023-10-09T04:28:06.568054+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3765,"log_dups_size":0,"ondisk_log_size":3765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246743040,"num_objects":3765,"num_object_clones":0,"num_object_copies":11295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3765,"num_write_kb":240960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":726,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284729+0000","last_change":"2023-10-09T04:26:23.679008+0000","last_active":"2023-10-09T04:27:55.284729+0000","last_peered":"2023-10-09T04:27:55.284729+0000","last_clean":"2023-10-09T04:27:55.284729+0000","last_became_active":"2023-10-09T04:26:23.678629+0000","last_became_peered":"2023-10-09T04:26:23.678629+0000","last_unstale":"2023-10-09T04:27:55.284729+0000","last_undegraded":"2023-10-09T04:27:55.284729+0000","last_fullsized":"2023-10-09T04:27:55.284729+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:41.534992+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268343+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:27:55.268343+0000","last_peered":"2023-10-09T04:27:55.268343+0000","last_clean":"2023-10-09T04:27:55.268343+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:27:55.268343+0000","last_undegraded":"2023-10-09T04:27:55.268343+0000","last_fullsized":"2023-10-09T04:27:55.268343+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":99,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269808+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:27:55.269808+0000","last_peered":"2023-10-09T04:27:55.269808+0000","last_clean":"2023-10-09T04:27:55.269808+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:27:55.269808+0000","last_undegraded":"2023-10-09T04:27:55.269808+0000","last_fullsized":"2023-10-09T04:27:55.269808+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"517'3756","reported_seq":3772,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.704222+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:28:05.704222+0000","last_peered":"2023-10-09T04:28:05.704222+0000","last_clean":"2023-10-09T04:28:05.704222+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:28:05.704222+0000","last_undegraded":"2023-10-09T04:28:05.704222+0000","last_fullsized":"2023-10-09T04:28:05.704222+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3756,"log_dups_size":0,"ondisk_log_size":3756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246153216,"num_objects":3756,"num_object_clones":0,"num_object_copies":11268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3756,"num_write_kb":240384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":623,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277975+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:27:55.277975+0000","last_peered":"2023-10-09T04:27:55.277975+0000","last_clean":"2023-10-09T04:27:55.277975+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:27:55.277975+0000","last_undegraded":"2023-10-09T04:27:55.277975+0000","last_fullsized":"2023-10-09T04:27:55.277975+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268019+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:27:55.268019+0000","last_peered":"2023-10-09T04:27:55.268019+0000","last_clean":"2023-10-09T04:27:55.268019+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:27:55.268019+0000","last_undegraded":"2023-10-09T04:27:55.268019+0000","last_fullsized":"2023-10-09T04:27:55.268019+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.271327+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:27:55.271327+0000","last_peered":"2023-10-09T04:27:55.271327+0000","last_clean":"2023-10-09T04:27:55.271327+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:27:55.271327+0000","last_undegraded":"2023-10-09T04:27:55.271327+0000","last_fullsized":"2023-10-09T04:27:55.271327+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"516'3614","reported_seq":3632,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:28:03.763541+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:28:03.763541+0000","last_peered":"2023-10-09T04:28:03.763541+0000","last_clean":"2023-10-09T04:28:03.763541+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:28:03.763541+0000","last_undegraded":"2023-10-09T04:28:03.763541+0000","last_fullsized":"2023-10-09T04:28:03.763541+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3614,"log_dups_size":0,"ondisk_log_size":3614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236847104,"num_objects":3614,"num_object_clones":0,"num_object_copies":10842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3614,"num_write_kb":231296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":721,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284798+0000","last_change":"2023-10-09T04:26:23.680981+0000","last_active":"2023-10-09T04:27:55.284798+0000","last_peered":"2023-10-09T04:27:55.284798+0000","last_clean":"2023-10-09T04:27:55.284798+0000","last_became_active":"2023-10-09T04:26:23.680615+0000","last_became_peered":"2023-10-09T04:26:23.680615+0000","last_unstale":"2023-10-09T04:27:55.284798+0000","last_undegraded":"2023-10-09T04:27:55.284798+0000","last_fullsized":"2023-10-09T04:27:55.284798+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:36.804345+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":182,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.276656+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:27:55.276656+0000","last_peered":"2023-10-09T04:27:55.276656+0000","last_clean":"2023-10-09T04:27:55.276656+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:27:55.276656+0000","last_undegraded":"2023-10-09T04:27:55.276656+0000","last_fullsized":"2023-10-09T04:27:55.276656+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269256+0000","last_change":"2023-10-09T04:26:09.746900+0000","last_active":"2023-10-09T04:27:55.269256+0000","last_peered":"2023-10-09T04:27:55.269256+0000","last_clean":"2023-10-09T04:27:55.269256+0000","last_became_active":"2023-10-09T04:26:09.746607+0000","last_became_peered":"2023-10-09T04:26:09.746607+0000","last_unstale":"2023-10-09T04:27:55.269256+0000","last_undegraded":"2023-10-09T04:27:55.269256+0000","last_fullsized":"2023-10-09T04:27:55.269256+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:07.695043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"517'3640","reported_seq":3658,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.147358+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:28:05.147358+0000","last_peered":"2023-10-09T04:28:05.147358+0000","last_clean":"2023-10-09T04:28:05.147358+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:28:05.147358+0000","last_undegraded":"2023-10-09T04:28:05.147358+0000","last_fullsized":"2023-10-09T04:28:05.147358+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3640,"log_dups_size":0,"ondisk_log_size":3640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238551040,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3640,"num_write_kb":232960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":643,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.266254+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:27:55.266254+0000","last_peered":"2023-10-09T04:27:55.266254+0000","last_clean":"2023-10-09T04:27:55.266254+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:27:55.266254+0000","last_undegraded":"2023-10-09T04:27:55.266254+0000","last_fullsized":"2023-10-09T04:27:55.266254+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277580+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:27:55.277580+0000","last_peered":"2023-10-09T04:27:55.277580+0000","last_clean":"2023-10-09T04:27:55.277580+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:27:55.277580+0000","last_undegraded":"2023-10-09T04:27:55.277580+0000","last_fullsized":"2023-10-09T04:27:55.277580+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":533,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285332+0000","last_change":"2023-10-09T04:26:21.351382+0000","last_active":"2023-10-09T04:27:55.285332+0000","last_peered":"2023-10-09T04:27:55.285332+0000","last_clean":"2023-10-09T04:27:55.285332+0000","last_became_active":"2023-10-09T04:26:21.348023+0000","last_became_peered":"2023-10-09T04:26:21.348023+0000","last_unstale":"2023-10-09T04:27:55.285332+0000","last_undegraded":"2023-10-09T04:27:55.285332+0000","last_fullsized":"2023-10-09T04:27:55.285332+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:45.177721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":396,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285068+0000","last_change":"2023-10-09T04:26:09.749151+0000","last_active":"2023-10-09T04:27:55.285068+0000","last_peered":"2023-10-09T04:27:55.285068+0000","last_clean":"2023-10-09T04:27:55.285068+0000","last_became_active":"2023-10-09T04:26:09.748857+0000","last_became_peered":"2023-10-09T04:26:09.748857+0000","last_unstale":"2023-10-09T04:27:55.285068+0000","last_undegraded":"2023-10-09T04:27:55.285068+0000","last_fullsized":"2023-10-09T04:27:55.285068+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:38.228161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"517'3834","reported_seq":3853,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:06.584019+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:28:06.584019+0000","last_peered":"2023-10-09T04:28:06.584019+0000","last_clean":"2023-10-09T04:28:06.584019+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:28:06.584019+0000","last_undegraded":"2023-10-09T04:28:06.584019+0000","last_fullsized":"2023-10-09T04:28:06.584019+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3834,"log_dups_size":0,"ondisk_log_size":3834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251265024,"num_objects":3834,"num_object_clones":0,"num_object_copies":11502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3834,"num_write_kb":245376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":724,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284704+0000","last_change":"2023-10-09T04:26:23.684553+0000","last_active":"2023-10-09T04:27:55.284704+0000","last_peered":"2023-10-09T04:27:55.284704+0000","last_clean":"2023-10-09T04:27:55.284704+0000","last_became_active":"2023-10-09T04:26:23.684125+0000","last_became_peered":"2023-10-09T04:26:23.684125+0000","last_unstale":"2023-10-09T04:27:55.284704+0000","last_undegraded":"2023-10-09T04:27:55.284704+0000","last_fullsized":"2023-10-09T04:27:55.284704+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:01.715214+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":500,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284904+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:27:55.284904+0000","last_peered":"2023-10-09T04:27:55.284904+0000","last_clean":"2023-10-09T04:27:55.284904+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:27:55.284904+0000","last_undegraded":"2023-10-09T04:27:55.284904+0000","last_fullsized":"2023-10-09T04:27:55.284904+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268889+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:27:55.268889+0000","last_peered":"2023-10-09T04:27:55.268889+0000","last_clean":"2023-10-09T04:27:55.268889+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:27:55.268889+0000","last_undegraded":"2023-10-09T04:27:55.268889+0000","last_fullsized":"2023-10-09T04:27:55.268889+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285104+0000","last_change":"2023-10-09T04:27:51.852322+0000","last_active":"2023-10-09T04:27:55.285104+0000","last_peered":"2023-10-09T04:27:55.285104+0000","last_clean":"2023-10-09T04:27:55.285104+0000","last_became_active":"2023-10-09T04:27:51.851990+0000","last_became_peered":"2023-10-09T04:27:51.851990+0000","last_unstale":"2023-10-09T04:27:55.285104+0000","last_undegraded":"2023-10-09T04:27:55.285104+0000","last_fullsized":"2023-10-09T04:27:55.285104+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:19.671405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"517'3547","reported_seq":3564,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.155706+0000","last_change":"2023-10-09T04:27:20.717622+0000","last_active":"2023-10-09T04:28:05.155706+0000","last_peered":"2023-10-09T04:28:05.155706+0000","last_clean":"2023-10-09T04:28:05.155706+0000","last_became_active":"2023-10-09T04:27:20.717291+0000","last_became_peered":"2023-10-09T04:27:20.717291+0000","last_unstale":"2023-10-09T04:28:05.155706+0000","last_undegraded":"2023-10-09T04:28:05.155706+0000","last_fullsized":"2023-10-09T04:28:05.155706+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3547,"log_dups_size":0,"ondisk_log_size":3547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:34.329924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232456192,"num_objects":3547,"num_object_clones":0,"num_object_copies":10641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3547,"num_write_kb":227008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":691,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277613+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:27:55.277613+0000","last_peered":"2023-10-09T04:27:55.277613+0000","last_clean":"2023-10-09T04:27:55.277613+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:27:55.277613+0000","last_undegraded":"2023-10-09T04:27:55.277613+0000","last_fullsized":"2023-10-09T04:27:55.277613+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":367,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.278991+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:27:55.278991+0000","last_peered":"2023-10-09T04:27:55.278991+0000","last_clean":"2023-10-09T04:27:55.278991+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:27:55.278991+0000","last_undegraded":"2023-10-09T04:27:55.278991+0000","last_fullsized":"2023-10-09T04:27:55.278991+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":455,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-09T04:27:51.775882+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:27:51.775882+0000","last_peered":"2023-10-09T04:27:51.775882+0000","last_clean":"2023-10-09T04:27:51.775882+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:27:51.775882+0000","last_undegraded":"2023-10-09T04:27:51.775882+0000","last_fullsized":"2023-10-09T04:27:51.775882+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":222,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285701+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:27:55.285701+0000","last_peered":"2023-10-09T04:27:55.285701+0000","last_clean":"2023-10-09T04:27:55.285701+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:27:55.285701+0000","last_undegraded":"2023-10-09T04:27:55.285701+0000","last_fullsized":"2023-10-09T04:27:55.285701+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"517'3599","reported_seq":3617,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.149557+0000","last_change":"2023-10-09T04:27:20.717505+0000","last_active":"2023-10-09T04:28:05.149557+0000","last_peered":"2023-10-09T04:28:05.149557+0000","last_clean":"2023-10-09T04:28:05.149557+0000","last_became_active":"2023-10-09T04:27:20.717202+0000","last_became_peered":"2023-10-09T04:27:20.717202+0000","last_unstale":"2023-10-09T04:28:05.149557+0000","last_undegraded":"2023-10-09T04:28:05.149557+0000","last_fullsized":"2023-10-09T04:28:05.149557+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3599,"log_dups_size":0,"ondisk_log_size":3599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:51.034299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235864064,"num_objects":3599,"num_object_clones":0,"num_object_copies":10797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3599,"num_write_kb":230336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":520,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284884+0000","last_change":"2023-10-09T04:26:21.429887+0000","last_active":"2023-10-09T04:27:55.284884+0000","last_peered":"2023-10-09T04:27:55.284884+0000","last_clean":"2023-10-09T04:27:55.284884+0000","last_became_active":"2023-10-09T04:26:21.392380+0000","last_became_peered":"2023-10-09T04:26:21.392380+0000","last_unstale":"2023-10-09T04:27:55.284884+0000","last_undegraded":"2023-10-09T04:27:55.284884+0000","last_fullsized":"2023-10-09T04:27:55.284884+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:05:55.650685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":971,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267725+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:27:55.267725+0000","last_peered":"2023-10-09T04:27:55.267725+0000","last_clean":"2023-10-09T04:27:55.267725+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:27:55.267725+0000","last_undegraded":"2023-10-09T04:27:55.267725+0000","last_fullsized":"2023-10-09T04:27:55.267725+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":453,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285021+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:27:55.285021+0000","last_peered":"2023-10-09T04:27:55.285021+0000","last_clean":"2023-10-09T04:27:55.285021+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:27:55.285021+0000","last_undegraded":"2023-10-09T04:27:55.285021+0000","last_fullsized":"2023-10-09T04:27:55.285021+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":79,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269657+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:27:55.269657+0000","last_peered":"2023-10-09T04:27:55.269657+0000","last_clean":"2023-10-09T04:27:55.269657+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:27:55.269657+0000","last_undegraded":"2023-10-09T04:27:55.269657+0000","last_fullsized":"2023-10-09T04:27:55.269657+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"517'3707","reported_seq":3726,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:06.581040+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:28:06.581040+0000","last_peered":"2023-10-09T04:28:06.581040+0000","last_clean":"2023-10-09T04:28:06.581040+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:28:06.581040+0000","last_undegraded":"2023-10-09T04:28:06.581040+0000","last_fullsized":"2023-10-09T04:28:06.581040+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3707,"log_dups_size":0,"ondisk_log_size":3707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242941952,"num_objects":3707,"num_object_clones":0,"num_object_copies":11121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3707,"num_write_kb":237248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":612,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268960+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:27:55.268960+0000","last_peered":"2023-10-09T04:27:55.268960+0000","last_clean":"2023-10-09T04:27:55.268960+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:27:55.268960+0000","last_undegraded":"2023-10-09T04:27:55.268960+0000","last_fullsized":"2023-10-09T04:27:55.268960+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":517,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267643+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:27:55.267643+0000","last_peered":"2023-10-09T04:27:55.267643+0000","last_clean":"2023-10-09T04:27:55.267643+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:27:55.267643+0000","last_undegraded":"2023-10-09T04:27:55.267643+0000","last_fullsized":"2023-10-09T04:27:55.267643+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":158,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.279353+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:27:55.279353+0000","last_peered":"2023-10-09T04:27:55.279353+0000","last_clean":"2023-10-09T04:27:55.279353+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:27:55.279353+0000","last_undegraded":"2023-10-09T04:27:55.279353+0000","last_fullsized":"2023-10-09T04:27:55.279353+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":140,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277184+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:27:55.277184+0000","last_peered":"2023-10-09T04:27:55.277184+0000","last_clean":"2023-10-09T04:27:55.277184+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:27:55.277184+0000","last_undegraded":"2023-10-09T04:27:55.277184+0000","last_fullsized":"2023-10-09T04:27:55.277184+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"517'3709","reported_seq":3727,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.142798+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:28:05.142798+0000","last_peered":"2023-10-09T04:28:05.142798+0000","last_clean":"2023-10-09T04:28:05.142798+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:28:05.142798+0000","last_undegraded":"2023-10-09T04:28:05.142798+0000","last_fullsized":"2023-10-09T04:28:05.142798+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3709,"log_dups_size":0,"ondisk_log_size":3709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243073024,"num_objects":3709,"num_object_clones":0,"num_object_copies":11127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3709,"num_write_kb":237376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":635,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.265291+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:27:55.265291+0000","last_peered":"2023-10-09T04:27:55.265291+0000","last_clean":"2023-10-09T04:27:55.265291+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:27:55.265291+0000","last_undegraded":"2023-10-09T04:27:55.265291+0000","last_fullsized":"2023-10-09T04:27:55.265291+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":540,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269872+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:27:55.269872+0000","last_peered":"2023-10-09T04:27:55.269872+0000","last_clean":"2023-10-09T04:27:55.269872+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:27:55.269872+0000","last_undegraded":"2023-10-09T04:27:55.269872+0000","last_fullsized":"2023-10-09T04:27:55.269872+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":275,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285150+0000","last_change":"2023-10-09T04:26:21.355315+0000","last_active":"2023-10-09T04:27:55.285150+0000","last_peered":"2023-10-09T04:27:55.285150+0000","last_clean":"2023-10-09T04:27:55.285150+0000","last_became_active":"2023-10-09T04:26:21.355038+0000","last_became_peered":"2023-10-09T04:26:21.355038+0000","last_unstale":"2023-10-09T04:27:55.285150+0000","last_undegraded":"2023-10-09T04:27:55.285150+0000","last_fullsized":"2023-10-09T04:27:55.285150+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:45.819056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":236,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284824+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:27:55.284824+0000","last_peered":"2023-10-09T04:27:55.284824+0000","last_clean":"2023-10-09T04:27:55.284824+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:27:55.284824+0000","last_undegraded":"2023-10-09T04:27:55.284824+0000","last_fullsized":"2023-10-09T04:27:55.284824+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"517'3694","reported_seq":3712,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.169375+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:28:05.169375+0000","last_peered":"2023-10-09T04:28:05.169375+0000","last_clean":"2023-10-09T04:28:05.169375+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:28:05.169375+0000","last_undegraded":"2023-10-09T04:28:05.169375+0000","last_fullsized":"2023-10-09T04:28:05.169375+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3694,"log_dups_size":0,"ondisk_log_size":3694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242089984,"num_objects":3694,"num_object_clones":0,"num_object_copies":11082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3694,"num_write_kb":236416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":574,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269733+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:27:55.269733+0000","last_peered":"2023-10-09T04:27:55.269733+0000","last_clean":"2023-10-09T04:27:55.269733+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:27:55.269733+0000","last_undegraded":"2023-10-09T04:27:55.269733+0000","last_fullsized":"2023-10-09T04:27:55.269733+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":477,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269448+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:27:55.269448+0000","last_peered":"2023-10-09T04:27:55.269448+0000","last_clean":"2023-10-09T04:27:55.269448+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:27:55.269448+0000","last_undegraded":"2023-10-09T04:27:55.269448+0000","last_fullsized":"2023-10-09T04:27:55.269448+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.271364+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:27:55.271364+0000","last_peered":"2023-10-09T04:27:55.271364+0000","last_clean":"2023-10-09T04:27:55.271364+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:27:55.271364+0000","last_undegraded":"2023-10-09T04:27:55.271364+0000","last_fullsized":"2023-10-09T04:27:55.271364+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":425,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268775+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:27:55.268775+0000","last_peered":"2023-10-09T04:27:55.268775+0000","last_clean":"2023-10-09T04:27:55.268775+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:27:55.268775+0000","last_undegraded":"2023-10-09T04:27:55.268775+0000","last_fullsized":"2023-10-09T04:27:55.268775+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"517'3592","reported_seq":3610,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:05.158583+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:28:05.158583+0000","last_peered":"2023-10-09T04:28:05.158583+0000","last_clean":"2023-10-09T04:28:05.158583+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:28:05.158583+0000","last_undegraded":"2023-10-09T04:28:05.158583+0000","last_fullsized":"2023-10-09T04:28:05.158583+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":3592,"log_dups_size":0,"ondisk_log_size":3592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235405312,"num_objects":3592,"num_object_clones":0,"num_object_copies":10776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3592,"num_write_kb":229888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":692,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.276969+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:27:55.276969+0000","last_peered":"2023-10-09T04:27:55.276969+0000","last_clean":"2023-10-09T04:27:55.276969+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:27:55.276969+0000","last_undegraded":"2023-10-09T04:27:55.276969+0000","last_fullsized":"2023-10-09T04:27:55.276969+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":316,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.266329+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:27:55.266329+0000","last_peered":"2023-10-09T04:27:55.266329+0000","last_clean":"2023-10-09T04:27:55.266329+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:27:55.266329+0000","last_undegraded":"2023-10-09T04:27:55.266329+0000","last_fullsized":"2023-10-09T04:27:55.266329+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":152,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277391+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:27:55.277391+0000","last_peered":"2023-10-09T04:27:55.277391+0000","last_clean":"2023-10-09T04:27:55.277391+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:27:55.277391+0000","last_undegraded":"2023-10-09T04:27:55.277391+0000","last_fullsized":"2023-10-09T04:27:55.277391+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267869+0000","last_change":"2023-10-09T04:24:56.169203+0000","last_active":"2023-10-09T04:27:55.267869+0000","last_peered":"2023-10-09T04:27:55.267869+0000","last_clean":"2023-10-09T04:27:55.267869+0000","last_became_active":"2023-10-09T04:24:56.168737+0000","last_became_peered":"2023-10-09T04:24:56.168737+0000","last_unstale":"2023-10-09T04:27:55.267869+0000","last_undegraded":"2023-10-09T04:27:55.267869+0000","last_fullsized":"2023-10-09T04:27:55.267869+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:21.515886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":560,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:04.121563+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:28:04.121563+0000","last_peered":"2023-10-09T04:28:04.121563+0000","last_clean":"2023-10-09T04:28:04.121563+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:28:04.121563+0000","last_undegraded":"2023-10-09T04:28:04.121563+0000","last_fullsized":"2023-10-09T04:28:04.121563+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":140,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:28:04.124718+0000","last_change":"2023-10-09T04:24:56.170052+0000","last_active":"2023-10-09T04:28:04.124718+0000","last_peered":"2023-10-09T04:28:04.124718+0000","last_clean":"2023-10-09T04:28:04.124718+0000","last_became_active":"2023-10-09T04:24:56.169607+0000","last_became_peered":"2023-10-09T04:24:56.169607+0000","last_unstale":"2023-10-09T04:28:04.124718+0000","last_undegraded":"2023-10-09T04:28:04.124718+0000","last_fullsized":"2023-10-09T04:28:04.124718+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.849923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.282630+0000","last_change":"2023-10-09T04:26:21.345230+0000","last_active":"2023-10-09T04:27:55.282630+0000","last_peered":"2023-10-09T04:27:55.282630+0000","last_clean":"2023-10-09T04:27:55.282630+0000","last_became_active":"2023-10-09T04:26:21.344939+0000","last_became_peered":"2023-10-09T04:26:21.344939+0000","last_unstale":"2023-10-09T04:27:55.282630+0000","last_undegraded":"2023-10-09T04:27:55.282630+0000","last_fullsized":"2023-10-09T04:27:55.282630+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.495703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":591,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.265244+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:27:55.265244+0000","last_peered":"2023-10-09T04:27:55.265244+0000","last_clean":"2023-10-09T04:27:55.265244+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:27:55.265244+0000","last_undegraded":"2023-10-09T04:27:55.265244+0000","last_fullsized":"2023-10-09T04:27:55.265244+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":534,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.285246+0000","last_change":"2023-10-09T04:24:56.181978+0000","last_active":"2023-10-09T04:27:55.285246+0000","last_peered":"2023-10-09T04:27:55.285246+0000","last_clean":"2023-10-09T04:27:55.285246+0000","last_became_active":"2023-10-09T04:24:56.164297+0000","last_became_peered":"2023-10-09T04:24:56.164297+0000","last_unstale":"2023-10-09T04:27:55.285246+0000","last_undegraded":"2023-10-09T04:27:55.285246+0000","last_fullsized":"2023-10-09T04:27:55.285246+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:15.703672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268759+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:27:55.268759+0000","last_peered":"2023-10-09T04:27:55.268759+0000","last_clean":"2023-10-09T04:27:55.268759+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:27:55.268759+0000","last_undegraded":"2023-10-09T04:27:55.268759+0000","last_fullsized":"2023-10-09T04:27:55.268759+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":545,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.284907+0000","last_change":"2023-10-09T04:24:56.202659+0000","last_active":"2023-10-09T04:27:55.284907+0000","last_peered":"2023-10-09T04:27:55.284907+0000","last_clean":"2023-10-09T04:27:55.284907+0000","last_became_active":"2023-10-09T04:24:56.166522+0000","last_became_peered":"2023-10-09T04:24:56.166522+0000","last_unstale":"2023-10-09T04:27:55.284907+0000","last_undegraded":"2023-10-09T04:27:55.284907+0000","last_fullsized":"2023-10-09T04:27:55.284907+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:15.858204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268107+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:27:55.268107+0000","last_peered":"2023-10-09T04:27:55.268107+0000","last_clean":"2023-10-09T04:27:55.268107+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:27:55.268107+0000","last_undegraded":"2023-10-09T04:27:55.268107+0000","last_fullsized":"2023-10-09T04:27:55.268107+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":770,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.268607+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:27:55.268607+0000","last_peered":"2023-10-09T04:27:55.268607+0000","last_clean":"2023-10-09T04:27:55.268607+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:27:55.268607+0000","last_undegraded":"2023-10-09T04:27:55.268607+0000","last_fullsized":"2023-10-09T04:27:55.268607+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":250,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.277880+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:27:55.277880+0000","last_peered":"2023-10-09T04:27:55.277880+0000","last_clean":"2023-10-09T04:27:55.277880+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:27:55.277880+0000","last_undegraded":"2023-10-09T04:27:55.277880+0000","last_fullsized":"2023-10-09T04:27:55.277880+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":367,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.278915+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:27:55.278915+0000","last_peered":"2023-10-09T04:27:55.278915+0000","last_clean":"2023-10-09T04:27:55.278915+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:27:55.278915+0000","last_undegraded":"2023-10-09T04:27:55.278915+0000","last_fullsized":"2023-10-09T04:27:55.278915+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":628,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.269585+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:27:55.269585+0000","last_peered":"2023-10-09T04:27:55.269585+0000","last_clean":"2023-10-09T04:27:55.269585+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:27:55.269585+0000","last_undegraded":"2023-10-09T04:27:55.269585+0000","last_fullsized":"2023-10-09T04:27:55.269585+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":144,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.276924+0000","last_change":"2023-10-09T04:26:21.347607+0000","last_active":"2023-10-09T04:27:55.276924+0000","last_peered":"2023-10-09T04:27:55.276924+0000","last_clean":"2023-10-09T04:27:55.276924+0000","last_became_active":"2023-10-09T04:26:21.347136+0000","last_became_peered":"2023-10-09T04:26:21.347136+0000","last_unstale":"2023-10-09T04:27:55.276924+0000","last_undegraded":"2023-10-09T04:27:55.276924+0000","last_fullsized":"2023-10-09T04:27:55.276924+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:52.721389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":82,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-09T04:27:55.267404+0000","last_change":"2023-10-09T04:24:56.183973+0000","last_active":"2023-10-09T04:27:55.267404+0000","last_peered":"2023-10-09T04:27:55.267404+0000","last_clean":"2023-10-09T04:27:55.267404+0000","last_became_active":"2023-10-09T04:24:56.181453+0000","last_became_peered":"2023-10-09T04:24:56.181453+0000","last_unstale":"2023-10-09T04:27:55.267404+0000","last_undegraded":"2023-10-09T04:27:55.267404+0000","last_fullsized":"2023-10-09T04:27:55.267404+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:35.829983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3844407296,"num_objects":58661,"num_object_clones":0,"num_object_copies":175983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":58661,"num_write_kb":3754304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11466637312,"data_stored":11466637312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58661,"ondisk_log_size":58661,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":11614,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1836096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201707,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2293968,"kb_used_data":1398152,"kb_used_omap":0,"kb_used_meta":895808,"kb_avail":91451184,"statfs":{"total":95995035648,"available":93646012416,"internally_reserved":0,"allocated":1431707648,"data_stored":1430205449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917307392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:52 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.6950000000000001,"15min":2.7730000000000001},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.133,"5min":70.614000000000004,"15min":148.65899999999999},"last":59.780000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":2,"15min":2.827},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.221},"max":{"1min":5.3129999999999997,"5min":70.721999999999994,"15min":148.642},"last":59.709000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":4.4359999999999999,"15min":5.5069999999999997},"min":{"1min":0.316,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.4870000000000001,"5min":261.935,"15min":261.935},"last":65.361999999999995},{"interface":"front","average":{"1min":1.5569999999999999,"5min":4.4740000000000002,"15min":5.3600000000000003},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":6.0229999999999997,"5min":262.07600000000002,"15min":262.07600000000002},"last":62.767000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:27:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.9279999999999999,"15min":2.758},"min":{"1min":0.35899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.8580000000000001,"5min":66.328999999999994,"15min":253.41200000000001},"last":0.505},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.012,"15min":3.1320000000000001},"min":{"1min":0.41899999999999998,"5min":0.29899999999999999,"15min":0.246},"max":{"1min":5.3540000000000001,"5min":66.314999999999998,"15min":253.39400000000001},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":3.758,"15min":2.8599999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.9219999999999997,"5min":255.97300000000001,"15min":255.97300000000001},"last":56.354999999999997},{"interface":"front","average":{"1min":1.5169999999999999,"5min":3.9319999999999999,"15min":3.1219999999999999},"min":{"1min":0.38500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.0069999999999997,"5min":256.06099999999998,"15min":256.06099999999998},"last":56.334000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.81699999999999995,"15min":0.79900000000000004},"min":{"1min":0.38600000000000001,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":6.8019999999999996,"5min":6.8019999999999996,"15min":8.9359999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.113,"5min":0.90200000000000002,"15min":1.1639999999999999},"min":{"1min":0.38300000000000001,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":5.4530000000000003,"5min":7.2549999999999999,"15min":12.566000000000001},"last":0.40000000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.1879999999999999,"15min":1.6779999999999999},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":4.5410000000000004,"5min":17.882999999999999,"15min":120.066},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.248,"15min":1.5049999999999999},"min":{"1min":0.39300000000000002,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":3.625,"5min":17.568999999999999,"15min":120.09699999999999},"last":0.41699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:27:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.091,"15min":1.2350000000000001},"min":{"1min":0.41499999999999998,"5min":0.311,"15min":0.307},"max":{"1min":3.9430000000000001,"5min":7.5170000000000003,"15min":8.4550000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.056,"15min":0.96399999999999997},"min":{"1min":0.504,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":5.1479999999999997,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.45500000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463343,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2441020,"kb_used_data":1847156,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":91304132,"statfs":{"total":95995035648,"available":93495431168,"internally_reserved":0,"allocated":1891487744,"data_stored":1890005012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.5680000000000001,"15min":3.258},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.193},"max":{"1min":5.319,"5min":65.439999999999998,"15min":133.78999999999999},"last":3.0960000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.7070000000000001,"15min":3.3580000000000001},"min":{"1min":0.26300000000000001,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":6.6529999999999996,"5min":65.427999999999997,"15min":133.69300000000001},"last":3.7650000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":5.0700000000000003,"15min":5.1040000000000001},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":10.083,"5min":315.73599999999999,"15min":315.73599999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.9199999999999999,"5min":5.335,"15min":5.1909999999999998},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":15.159000000000001,"5min":316.91500000000002,"15min":316.91500000000002},"last":2.6779999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:27:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":3.3220000000000001,"15min":2.6339999999999999},"min":{"1min":0.25,"5min":0.248,"15min":0.23899999999999999},"max":{"1min":5.7549999999999999,"5min":70.358999999999995,"15min":139.06899999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.1990000000000001,"5min":3.3740000000000001,"15min":2.6760000000000002},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7240000000000002,"5min":70.391000000000005,"15min":139.173},"last":0.58199999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.397,"15min":2.0600000000000001},"min":{"1min":0.35999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.048,"5min":14.083,"15min":51.082000000000001},"last":2.9279999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.3149999999999999,"15min":1.9970000000000001},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":3.677,"5min":14.052,"15min":51.031999999999996},"last":3.5110000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":2.4350000000000001,"15min":1.427},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.24399999999999999},"max":{"1min":3.117,"5min":161.27500000000001,"15min":161.27500000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":2.3780000000000001,"15min":1.496},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":5.1189999999999998,"5min":160.32599999999999,"15min":160.32599999999999},"last":0.39700000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:27:22 2023","interfaces":[{"interface":"back","average":{"1min":4.883,"5min":1.6120000000000001,"15min":1.2649999999999999},"min":{"1min":0.40799999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":74.043000000000006,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.48899999999999999},{"interface":"front","average":{"1min":5.0579999999999998,"5min":1.532,"15min":1.268},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.27900000000000003},"max":{"1min":74.073999999999998,"5min":74.073999999999998,"15min":74.073999999999998},"last":0.42999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:27:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.95499999999999996,"15min":1.6459999999999999},"min":{"1min":0.40400000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.3040000000000003,"5min":5.9050000000000002,"15min":161.39400000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.81000000000000005,"15min":1.6339999999999999},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.27800000000000002},"max":{"1min":4.242,"5min":5.1520000000000001,"15min":161.41200000000001},"last":0.47499999999999998}]}]},{"osd":1,"up_from":89,"seq":382252089652,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2579352,"kb_used_data":1816016,"kb_used_omap":9,"kb_used_meta":763318,"kb_avail":91165800,"statfs":{"total":95995035648,"available":93353779200,"internally_reserved":0,"allocated":1859600384,"data_stored":1857335723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":781638633},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":5.6790000000000003,"5min":6.21,"15min":4.7789999999999999},"min":{"1min":0.188,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":86.963999999999999,"5min":259.26799999999997,"15min":259.26799999999997},"last":5.6260000000000003},{"interface":"front","average":{"1min":5.798,"5min":6.2130000000000001,"15min":5.4530000000000003},"min":{"1min":0.222,"5min":0.222,"15min":0.182},"max":{"1min":83.213999999999999,"5min":258.46899999999999,"15min":258.46899999999999},"last":6.1859999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":4.5019999999999998,"15min":3.052},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.694,"5min":259.315,"15min":259.315},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.7509999999999999,"5min":4.742,"15min":3.3919999999999999},"min":{"1min":0.26300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":11.026,"5min":259.733,"15min":259.733},"last":3.738}]},{"osd":3,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.7989999999999999,"15min":2.798},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.8680000000000001,"5min":71.745999999999995,"15min":71.745999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.665,"15min":3.2189999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8520000000000003,"5min":75.613,"15min":75.613},"last":6.1500000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":4.3529999999999998,"15min":3.008},"min":{"1min":0.318,"5min":0.27800000000000002,"15min":0.26900000000000002},"max":{"1min":6.9370000000000003,"5min":257.83300000000003,"15min":257.83300000000003},"last":3.371},{"interface":"front","average":{"1min":1.1439999999999999,"5min":4.2830000000000004,"15min":2.8420000000000001},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":7.4989999999999997,"5min":257.863,"15min":257.863},"last":1.819}]},{"osd":5,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.97399999999999998,"15min":2.177},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.29099999999999998},"max":{"1min":6.9649999999999999,"5min":8.2300000000000004,"15min":38.848999999999997},"last":3.339},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.018,"15min":2.2839999999999998},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":7.452,"5min":7.452,"15min":39.593000000000004},"last":1.8460000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.4179999999999999,"15min":2.028},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.5590000000000002,"5min":35.045999999999999,"15min":35.045999999999999},"last":1.8700000000000001},{"interface":"front","average":{"1min":1.21,"5min":1.502,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0259999999999998,"5min":34.960999999999999,"15min":34.960999999999999},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":4.577,"15min":3.9039999999999999},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.5289999999999999,"5min":259.08499999999998,"15min":259.08499999999998},"last":1.911},{"interface":"front","average":{"1min":1.119,"5min":4.6970000000000001,"15min":3.5489999999999999},"min":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":4.9790000000000001,"5min":259.13299999999998,"15min":259.13299999999998},"last":3.2749999999999999}]}]},{"osd":0,"up_from":84,"seq":360777253178,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2007404,"kb_used_data":1197924,"kb_used_omap":28,"kb_used_meta":809443,"kb_avail":91737748,"statfs":{"total":95995035648,"available":93939453952,"internally_reserved":0,"allocated":1226674176,"data_stored":1224277867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":828870480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.849,"15min":4.4390000000000001},"min":{"1min":0.20499999999999999,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":4.2549999999999999,"5min":96.305000000000007,"15min":209.215},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.655,"5min":2.2149999999999999,"15min":4.4859999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":16.131,"5min":95.861000000000004,"15min":120.934},"last":0.66400000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:27:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.589,"15min":2.5619999999999998},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.19600000000000001},"max":{"1min":4.0839999999999996,"5min":17.562999999999999,"15min":88.088999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.145,"5min":1.4750000000000001,"15min":2.8809999999999998},"min":{"1min":0.247,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.0780000000000003,"5min":17.655000000000001,"15min":88.004999999999995},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.95199999999999996,"15min":2.9740000000000002},"min":{"1min":0.27100000000000002,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":10.853999999999999,"5min":10.853999999999999,"15min":99.709000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.1539999999999999,"15min":3.036},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.292,"5min":12.292,"15min":99.560000000000002},"last":0.314}]},{"osd":4,"last update":"Mon Oct 9 04:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.865,"15min":2.4380000000000002},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":4.1970000000000001,"5min":86.906999999999996,"15min":88.058000000000007},"last":2.5819999999999999},{"interface":"front","average":{"1min":1.222,"5min":1.8480000000000001,"15min":2.3690000000000002},"min":{"1min":0.375,"5min":0.28299999999999997,"15min":0.24199999999999999},"max":{"1min":6.4500000000000002,"5min":86.891000000000005,"15min":86.891000000000005},"last":1.1220000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":2.25,"15min":3.3639999999999999},"min":{"1min":0.373,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.4850000000000003,"5min":134.93000000000001,"15min":156.96100000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.80100000000000005,"5min":2.0249999999999999,"15min":2.899},"min":{"1min":0.41399999999999998,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":2.0739999999999998,"5min":135.04599999999999,"15min":156.97300000000001},"last":0.80400000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:28:04 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.88700000000000001,"15min":1.6799999999999999},"min":{"1min":0.41999999999999998,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":1.8160000000000001,"5min":4.7160000000000002,"15min":63.468000000000004},"last":0.499},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.2350000000000001,"15min":2.1160000000000001},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.34100000000000003},"max":{"1min":4.1150000000000002,"5min":12.26,"15min":88.180000000000007},"last":0.96399999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:28:04 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.0589999999999999,"15min":1.8500000000000001},"min":{"1min":0.48199999999999998,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":1.8480000000000001,"5min":11.962,"15min":63.509},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.2949999999999999,"15min":2.2759999999999998},"min":{"1min":0.41699999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.165,"5min":12.321,"15min":88.120000000000005},"last":0.58799999999999997}]}]},{"osd":2,"up_from":94,"seq":403726926126,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2144396,"kb_used_data":1399876,"kb_used_omap":81,"kb_used_meta":744430,"kb_avail":91600756,"statfs":{"total":95995035648,"available":93799174144,"internally_reserved":0,"allocated":1433473024,"data_stored":1431165291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":762296538},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":2.1059999999999999,"15min":3.677},"min":{"1min":0.23400000000000001,"5min":0.183,"15min":0.17699999999999999},"max":{"1min":19.873000000000001,"5min":60.192,"15min":81.385999999999996},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.0569999999999999,"5min":2.3839999999999999,"15min":3.5630000000000002},"min":{"1min":0.218,"5min":0.216,"15min":0.20799999999999999},"max":{"1min":19.827999999999999,"5min":58.289999999999999,"15min":79.453999999999994},"last":8.6649999999999991}]},{"osd":1,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":2.226,"15min":4.335},"min":{"1min":0.224,"5min":0.224,"15min":0.191},"max":{"1min":3.2370000000000001,"5min":115.184,"15min":158.13399999999999},"last":2.3759999999999999},{"interface":"front","average":{"1min":0.75900000000000001,"5min":2.2519999999999998,"15min":4.4199999999999999},"min":{"1min":0.219,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":3.8940000000000001,"5min":113.94,"15min":161.12},"last":7.7389999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:06 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.0089999999999999,"15min":2.3879999999999999},"min":{"1min":0.23699999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":15.201000000000001,"5min":15.201000000000001,"15min":58.997},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.369,"15min":2.5920000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":15.102,"5min":15.102,"15min":66.135000000000005},"last":2.399}]},{"osd":4,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":2.1499999999999999,"15min":2.1389999999999998},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":3.8639999999999999,"5min":58.274000000000001,"15min":58.798000000000002},"last":7.6230000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.079,"15min":1.7230000000000001},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.3049999999999997,"5min":24.702000000000002,"15min":56.616},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.3560000000000001,"15min":2.3519999999999999},"min":{"1min":0.40699999999999997,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":4.4459999999999997,"5min":7.984,"15min":82.030000000000001},"last":0.373},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.9670000000000001,"15min":2.581},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.28599999999999998},"max":{"1min":3.923,"5min":58.125999999999998,"15min":82.016000000000005},"last":7.7670000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.736,"15min":1.952},"min":{"1min":0.36499999999999999,"5min":0.34399999999999997,"15min":0.308},"max":{"1min":5.2720000000000002,"5min":58.469999999999999,"15min":58.469999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.2189999999999999,"15min":2.1579999999999999},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.26800000000000002},"max":{"1min":3.198,"5min":58.067999999999998,"15min":58.067999999999998},"last":2.3380000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:27:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.99399999999999999,"15min":2.149},"min":{"1min":0.41899999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.2290000000000001,"5min":5.2290000000000001,"15min":101.675},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.179,"5min":2.2639999999999998,"15min":2.653},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":4.6239999999999997,"5min":58.207999999999998,"15min":101.72499999999999},"last":2.423}]}]},{"osd":3,"up_from":99,"seq":425201762600,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664820,"kb_used_data":1643948,"kb_used_omap":7,"kb_used_meta":1020856,"kb_avail":91080332,"statfs":{"total":95995035648,"available":93266259968,"internally_reserved":0,"allocated":1683402752,"data_stored":1681120654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1045357160},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":3.1739999999999999,"15min":3.9649999999999999},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":3.2999999999999998,"5min":65.810000000000002,"15min":125.765},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":3.0409999999999999,"15min":4.4379999999999997},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.185},"max":{"1min":6.2709999999999999,"5min":65.924999999999997,"15min":125.46899999999999},"last":1.9750000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":2.8210000000000002,"15min":4.6440000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":6.3079999999999998,"5min":170.196,"15min":186.21299999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.88100000000000001,"5min":2.778,"15min":4.524},"min":{"1min":0.23300000000000001,"5min":0.23000000000000001,"15min":0.193},"max":{"1min":5.1319999999999997,"5min":166.953,"15min":191.78},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.3620000000000001,"15min":2.5790000000000002},"min":{"1min":0.28000000000000003,"5min":0.22800000000000001,"15min":0.187},"max":{"1min":3.702,"5min":9.2720000000000002,"15min":127.599},"last":2.9780000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.127,"15min":3.448},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":3.327,"5min":12.032999999999999,"15min":129.98599999999999},"last":4.4969999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":3.7229999999999999,"15min":2.6070000000000002},"min":{"1min":0.33800000000000002,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":7.7560000000000002,"5min":211.84700000000001,"15min":211.84700000000001},"last":2.0379999999999998},{"interface":"front","average":{"1min":1.488,"5min":3.9159999999999999,"15min":2.7829999999999999},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.111000000000001,"5min":211.17500000000001,"15min":211.17500000000001},"last":0.45900000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.2070000000000001,"15min":2.052},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":3.476,"5min":9.6709999999999994,"15min":17.901},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.1539999999999999,"15min":1.6100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":7.6289999999999996,"5min":7.7530000000000001,"15min":18.068000000000001},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.8120000000000001,"15min":1.71},"min":{"1min":0.40200000000000002,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":5.5369999999999999,"5min":64.378,"15min":64.378},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.5629999999999999,"15min":1.976},"min":{"1min":0.377,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":5.4870000000000001,"5min":64.353999999999999,"15min":64.353999999999999},"last":2.0640000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":3.5830000000000002,"15min":2.7879999999999998},"min":{"1min":0.41299999999999998,"5min":0.34799999999999998,"15min":0.34599999999999997},"max":{"1min":3.6259999999999999,"5min":212.62700000000001,"15min":212.62700000000001},"last":2.0129999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":3.9079999999999999,"15min":3.0179999999999998},"min":{"1min":0.41199999999999998,"5min":0.376,"15min":0.33300000000000002},"max":{"1min":5.633,"5min":212.04599999999999,"15min":212.04599999999999},"last":0.43099999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986616,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1824388,"kb_used_data":930748,"kb_used_omap":0,"kb_used_meta":893632,"kb_avail":91920764,"statfs":{"total":95995035648,"available":94126862336,"internally_reserved":0,"allocated":953085952,"data_stored":951595162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915079168},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:38 2023","interfaces":[{"interface":"back","average":{"1min":7.5609999999999999,"5min":6.6189999999999998,"15min":7.0549999999999997},"min":{"1min":0.26500000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":121.652,"5min":207.45699999999999,"15min":207.45699999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":8.4710000000000001,"5min":7.0990000000000002,"15min":7.2510000000000003},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":125.911,"5min":207.114,"15min":207.114},"last":0.495}]},{"osd":1,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":2.2309999999999999,"15min":4.9459999999999997},"min":{"1min":0.248,"5min":0.23599999999999999,"15min":0.23499999999999999},"max":{"1min":6.6959999999999997,"5min":90.900000000000006,"15min":135.25399999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.93899999999999995,"5min":2.3130000000000002,"15min":4.8600000000000003},"min":{"1min":0.28999999999999998,"5min":0.252,"15min":0.251},"max":{"1min":3.2389999999999999,"5min":90.787000000000006,"15min":134.44300000000001},"last":0.60199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":7.3550000000000004,"15min":4.4450000000000003},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":5.0199999999999996,"5min":203.61500000000001,"15min":203.61500000000001},"last":0.376},{"interface":"front","average":{"1min":1.0289999999999999,"5min":7.2190000000000003,"15min":4.4059999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.1179999999999999,"5min":203.601,"15min":203.601},"last":0.46899999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.5469999999999999,"15min":2.7050000000000001},"min":{"1min":0.31,"5min":0.29599999999999999,"15min":0.27600000000000002},"max":{"1min":6.7199999999999998,"5min":30.789999999999999,"15min":154.13200000000001},"last":1.5649999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.2869999999999999,"15min":2.7559999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":8.4329999999999998,"5min":9.2710000000000008,"15min":154.077},"last":0.41399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.748,"15min":1.651},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.9020000000000001,"5min":5.9020000000000001,"15min":111.809},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.96799999999999997,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.252},"max":{"1min":6.6559999999999997,"5min":7.2400000000000002,"15min":112.002},"last":0.48099999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.635,"15min":1.6339999999999999},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.039999999999999,"5min":47.801000000000002,"15min":76.307000000000002},"last":0.754},{"interface":"front","average":{"1min":1.091,"5min":1.3200000000000001,"15min":1.4630000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1230000000000002,"5min":47.753,"15min":78.421999999999997},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:27:38 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":0.88700000000000001,"15min":1.1140000000000001},"min":{"1min":0.41099999999999998,"5min":0.309,"15min":0.26800000000000002},"max":{"1min":5.0510000000000002,"5min":6.0839999999999996,"15min":34.039000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.732,"5min":1.23,"15min":1.3100000000000001},"min":{"1min":0.40100000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":12.061,"5min":12.061,"15min":33.433999999999997},"last":5.0039999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724980,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810808,"kb_used_data":914160,"kb_used_omap":0,"kb_used_meta":896640,"kb_avail":91934344,"statfs":{"total":95995035648,"available":94140768256,"internally_reserved":0,"allocated":936099840,"data_stored":934617568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918159360},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,5,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":4.5540000000000003,"15min":4.1980000000000004},"min":{"1min":0.23200000000000001,"5min":0.22800000000000001,"15min":0.193},"max":{"1min":7.2190000000000003,"5min":139.86799999999999,"15min":139.86799999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.77300000000000002,"5min":4.6230000000000002,"15min":4.4489999999999998},"min":{"1min":0.255,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":2.6379999999999999,"5min":139.84899999999999,"15min":139.84899999999999},"last":0.51700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":6.9960000000000004,"15min":5.3419999999999996},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":46.649000000000001,"5min":296.23099999999999,"15min":296.23099999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":4.1580000000000004,"5min":7.2859999999999996,"15min":5.6269999999999998},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":46.100999999999999,"5min":295.065,"15min":295.065},"last":0.747}]},{"osd":2,"last update":"Mon Oct 9 04:27:55 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":3.5579999999999998,"15min":3.5609999999999999},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":3.6320000000000001,"5min":168.779,"15min":168.779},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.76900000000000002,"5min":3.8700000000000001,"15min":3.3740000000000001},"min":{"1min":0.311,"5min":0.27000000000000002,"15min":0.26200000000000001},"max":{"1min":4.3970000000000002,"5min":168.03100000000001,"15min":168.03100000000001},"last":0.58499999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":3.8570000000000002,"5min":1.478,"15min":2.1400000000000001},"min":{"1min":0.35699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":44.088999999999999,"5min":44.088999999999999,"15min":44.088999999999999},"last":5.319},{"interface":"front","average":{"1min":3.3719999999999999,"5min":1.3620000000000001,"15min":2.1640000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.23499999999999999},"max":{"1min":44.103000000000002,"5min":44.103000000000002,"15min":44.103000000000002},"last":2.3900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:27:55 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.7769999999999999,"15min":1.347},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":3.23,"5min":95.668000000000006,"15min":95.668000000000006},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.6830000000000001,"15min":1.2909999999999999},"min":{"1min":0.28599999999999998,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":2.1139999999999999,"5min":98.703999999999994,"15min":98.703999999999994},"last":0.55600000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.65500000000000003,"15min":0.88200000000000001},"min":{"1min":0.33400000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":1.8400000000000001,"5min":8.5350000000000001,"15min":12.068},"last":3.367},{"interface":"front","average":{"1min":1.605,"5min":0.81599999999999995,"15min":1.042},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":11.577,"5min":11.577,"15min":11.577},"last":0.49199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:27:55 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.68799999999999994,"15min":1.018},"min":{"1min":0.36299999999999999,"5min":0.33300000000000002,"15min":0.25900000000000001},"max":{"1min":3.7210000000000001,"5min":3.7490000000000001,"15min":12.096},"last":3.1640000000000001},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.94299999999999995,"15min":1.1779999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.6640000000000001,"5min":11.522,"15min":11.522},"last":3.9540000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1235353600,"data_stored":1235353600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1881735168,"data_stored":1881735168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1429012480,"data_stored":1429012480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1695088640,"data_stored":1695088640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":950730752,"data_stored":950730752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":938999808,"data_stored":938999808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1902051328,"data_stored":1902051328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1433665536,"data_stored":1433665536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:28:10.767 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:28:10.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-10-09T04:28:10.768 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:10.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:10.876 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:10.924 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:10.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:10.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:11.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:10 smithi181 ceph-mon[196979]: from='client.46628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:11.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:10 smithi181 ceph-mon[196979]: pgmap v1350: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:28:11.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:10 smithi163 ceph-mon[221067]: from='client.46628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:11.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:10 smithi163 ceph-mon[221067]: pgmap v1350: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:28:11.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:28:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:28:10] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.43.0" 2023-10-09T04:28:11.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:10 smithi163 ceph-mon[213841]: from='client.46628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:11.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:10 smithi163 ceph-mon[213841]: pgmap v1350: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T04:28:11.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:11.119 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:11.490 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 42 16 65128 65112 96.842 66.5625 0.0133004 0.0103204 2023-10-09T04:28:11.828 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:11.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:11.947 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:11.994 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:12.067 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:12.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:12.174 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:12.218 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:12.269 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:12.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:12.374 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:12.401 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:12.475 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:12.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:12.490 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 43 16 66195 66179 96.1408 66.6875 0.00850306 0.0103959 2023-10-09T04:28:12.585 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:12.658 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:12.944 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:12.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:13.061 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:13.080 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:13.149 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:13.220 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:13.220 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:13.258 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:13 smithi163 ceph-mon[213841]: pgmap v1351: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:28:13.258 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:13 smithi163 ceph-mon[221067]: pgmap v1351: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:28:13.497 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 44 16 67291 67275 95.4968 68.5 0.00780928 0.0104656 2023-10-09T04:28:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:13 smithi181 ceph-mon[196979]: pgmap v1351: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:28:13.988 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:28:14.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:14.109 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:14.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:14.210 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:14.266 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:14.466 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:14.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:14.497 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 45 16 68181 68165 94.6111 55.625 0.0249585 0.0105623 2023-10-09T04:28:14.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:28:14.570 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:14.648 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:14.879 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:14.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:14.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:14.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:15.051 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:15.135 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:28:15.138 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:15.177 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:15.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:15 smithi163 ceph-mon[221067]: pgmap v1352: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:28:15.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3354802514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:15.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:15 smithi163 ceph-mon[213841]: pgmap v1352: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:28:15.178 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3354802514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:15.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:15.288 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:15.357 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:15.427 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:15 smithi181 ceph-mon[196979]: pgmap v1352: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:28:15.428 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3354802514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:15.497 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 46 16 69353 69337 94.1469 73.25 0.0104817 0.0106175 2023-10-09T04:28:15.607 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:15.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:15.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:15.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:15.978 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:16.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:16.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:16.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3354802514' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:16.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[221067]: failed to encode map e518 with expected crc 2023-10-09T04:28:16.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[221067]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T04:28:16.293 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[221067]: failed to encode map e518 with expected crc 2023-10-09T04:28:16.293 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:16.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3354802514' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:16.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[213841]: failed to encode map e518 with expected crc 2023-10-09T04:28:16.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[213841]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T04:28:16.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:16 smithi163 ceph-mon[213841]: failed to encode map e518 with expected crc 2023-10-09T04:28:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:16.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:16.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:16.466 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:16.497 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 47 16 70493 70477 93.6598 71.25 0.0194427 0.0106722 2023-10-09T04:28:16.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:16 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:16.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3354802514' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:16.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:16 smithi181 ceph-mon[196979]: failed to encode map e518 with expected crc 2023-10-09T04:28:16.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:16 smithi181 ceph-mon[196979]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T04:28:16.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:16 smithi181 ceph-mon[196979]: failed to encode map e518 with expected crc 2023-10-09T04:28:16.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:16.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:16.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:16.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:16.870 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:17.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:17 smithi163 ceph-mon[213841]: pgmap v1354: 119 pgs: 119 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:17.185 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:17.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:17.294 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:17.351 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:17.445 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:17.449 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:17 smithi181 ceph-mon[196979]: pgmap v1354: 119 pgs: 119 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:17.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:17.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:17.456 INFO:teuthology.orchestra.run.smithi163.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-10-09T04:28:17.477 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:17 smithi163 ceph-mon[221067]: pgmap v1354: 119 pgs: 119 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:17.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 48 16 71575 71559 93.1176 67.625 0.00869888 0.0107343 2023-10-09T04:28:17.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:17.622 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:17.691 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:17.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:17.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:17.798 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:17.877 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:18.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:18.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:18.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:18.135 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3782226684' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T04:28:18.199 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:18.256 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:18.337 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:18.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:18.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:18.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:28:18.443 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:18.479 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3782226684' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T04:28:18.479 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T04:28:18.482 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:18.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3782226684' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T04:28:18.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T04:28:18.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-10-09T04:28:18.497 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 49 16 72759 72743 92.7275 74 0.0146735 0.0107789 2023-10-09T04:28:18.553 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:18.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:18.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:18.657 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:18.704 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:28:18.771 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:18.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:18.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:18.878 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:18.957 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:19.070 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:19.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:19.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:19.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:19.211 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:19.236 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:19.254 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:19.284 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:19.298 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:19.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:19.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:19 smithi163 ceph-mon[213841]: pgmap v1355: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:19.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:19 smithi163 ceph-mon[221067]: pgmap v1355: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:19.371 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:19.391 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:19.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:19.403 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:19.437 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:19.477 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.487 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:28:19.497 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 50 16 73776 73760 92.1444 63.5625 0.013158 0.0108464 2023-10-09T04:28:19.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:19.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:19 smithi181 ceph-mon[196979]: pgmap v1355: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:19.543 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:19.605 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:19.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:19.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:19.754 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:19.808 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:19.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:19.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:19.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:19.978 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:20.245 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:20.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:20.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:20.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1848926280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:20.252 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:20.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1848926280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:20.253 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:20.352 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:20.414 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1848926280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:20.415 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:20.424 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:20.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 51 16 74703 74687 91.4739 57.9375 0.0101888 0.0109273 2023-10-09T04:28:20.538 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:20.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:20.648 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:20.706 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:20.887 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:28:20.887 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:28:20.887 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:75012 11.e 11:7525a4ea:::benchmark_data_smithi163_601242_object75011:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:20.874804+0000", 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.011029578999999999, 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011065772999999999, 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 75012 2023-10-09T04:28:20.888 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874804+0000", 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874804+0000", 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.889 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874803+0000", 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874854+0000", 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1665000000000001e-05 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.890 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874872+0000", 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7416000000000001e-05 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874879+0000", 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1430000000000001e-06 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.891 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874971+0000", 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2065999999999996e-05 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.875101+0000", 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000130519 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.892 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.875238+0000", 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000136499 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.880610+0000", 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053719029999999999 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:75019 11.e 11:7813a15b:::benchmark_data_smithi163_601242_object75018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:20.874893+0000", 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.01093999, 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011071872999999999, 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 75019 2023-10-09T04:28:20.894 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874893+0000", 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874893+0000", 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874893+0000", 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874926+0000", 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3550000000000002e-05 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874935+0000", 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1289999999999996e-06 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.874940+0000", 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0329999999999999e-06 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:20.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.877834+0000", 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028941090000000002 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.877949+0000", 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011495800000000001 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.878082+0000", 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013344299999999999 2023-10-09T04:28:20.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885833+0000", 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007751188 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.46577.0:75012 11.e e518/507)", 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:20.885962+0000", 2023-10-09T04:28:20.899 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 4294967295.9998717, 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000104418, 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885962+0000", 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:20.900 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885962+0000", 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885961+0000", 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885970+0000", 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8999999999999995e-06 2023-10-09T04:28:20.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885981+0000", 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1184e-05 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.885987+0000", 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1480000000000003e-06 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.886049+0000", 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2182999999999997e-05 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:20.886066+0000", 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6245000000000001e-05 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:20.903 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:28:20.904 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 3 2023-10-09T04:28:20.904 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:28:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:28:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:28:20] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.43.0" 2023-10-09T04:28:21.236 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:21.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:21 smithi181 ceph-mon[196979]: pgmap v1356: 119 pgs: 119 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:21.344 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.373 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:21.418 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:21.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:21.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 52 16 75532 75516 90.7115 51.8125 0.00303344 0.0109812 2023-10-09T04:28:21.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:21.567 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:21.574 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:21 smithi163 ceph-mon[213841]: pgmap v1356: 119 pgs: 119 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:21.576 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:21 smithi163 ceph-mon[221067]: pgmap v1356: 119 pgs: 119 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:28:21.895 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:21.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:21.904 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:28:22.004 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:22.088 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:22.193 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:22.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:22.305 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:22.360 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:22.459 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:22.466 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:22.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:22.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 53 16 76577 76561 90.2324 65.3125 0.0143197 0.0110778 2023-10-09T04:28:22.573 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:22.646 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:22.799 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:22.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:22.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:22.995 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:23.076 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:23.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:23.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:23.190 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:28:23.223 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:23.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:28:23.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:23 smithi181 ceph-mon[196979]: pgmap v1357: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:23.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2425058347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:23.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:23 smithi163 ceph-mon[213841]: pgmap v1357: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:23.308 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2425058347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:23.309 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:23 smithi163 ceph-mon[221067]: pgmap v1357: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:28:23.309 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2425058347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:23.310 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:23.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:23.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:23.458 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:23.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 54 16 77637 77621 89.7884 66.25 0.00348602 0.0111287 2023-10-09T04:28:23.505 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:23.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:23.608 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:23.635 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:23.748 INFO:tasks.thrashosds.thrasher:in_osds: [1, 5, 7, 3, 6, 0, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:28:23.748 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:28:24.201 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:24.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:24.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:24.306 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:24 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:24 smithi181 ceph-mon[196979]: failed to encode map e519 with expected crc 2023-10-09T04:28:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:24 smithi181 ceph-mon[196979]: failed to encode map e519 with expected crc 2023-10-09T04:28:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:24 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2425058347' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:24.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:24 smithi181 ceph-mon[196979]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T04:28:24.353 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:24.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:24.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:24.473 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:24.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 55 16 78305 78289 88.9153 41.75 0.0294521 0.0112413 2023-10-09T04:28:24.557 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:24.562 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[213841]: failed to encode map e519 with expected crc 2023-10-09T04:28:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[213841]: failed to encode map e519 with expected crc 2023-10-09T04:28:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2425058347' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:24.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[213841]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T04:28:24.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:24.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[221067]: failed to encode map e519 with expected crc 2023-10-09T04:28:24.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[221067]: failed to encode map e519 with expected crc 2023-10-09T04:28:24.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2425058347' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:24.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:24 smithi163 ceph-mon[221067]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T04:28:24.639 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:24.669 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:28:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:28:24.669 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:28:24.670 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:28:24.670 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:28:24.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:24.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:24.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:24.937 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:24.965 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:25.007 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:25.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:25.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:25.110 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:25.141 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:25.235 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:25 smithi163 ceph-mon[221067]: pgmap v1359: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T04:28:25.235 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:25 smithi163 ceph-mon[213841]: pgmap v1359: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T04:28:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:25 smithi181 ceph-mon[196979]: pgmap v1359: 119 pgs: 119 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T04:28:25.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 56 16 79085 79069 88.1982 48.75 0.00306704 0.0113295 2023-10-09T04:28:25.559 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:28:26.085 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:26.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:26 smithi163 ceph-mon[213841]: pgmap v1360: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-10-09T04:28:26.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:26 smithi163 ceph-mon[221067]: pgmap v1360: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-10-09T04:28:26.335 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:28:26.498 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 57 16 79882 79866 87.525 49.8125 0.0125281 0.0114197 2023-10-09T04:28:26.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:26 smithi181 ceph-mon[196979]: pgmap v1360: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-10-09T04:28:26.530 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:26.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:26.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:26.643 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:26.688 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:26.820 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:26.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:26.926 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:26.980 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:27.078 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:27.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:27.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:27.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:27.257 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:27.260 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/996682480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:27.280 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:27.300 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:27.345 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:27.389 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:27.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:27.397 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:27.408 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:27.437 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:27.477 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:27.496 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:27.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 58 16 80601 80585 86.7846 44.9375 0.0320656 0.0115172 2023-10-09T04:28:27.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/996682480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:27.525 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:27.560 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:27.577 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:27.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/996682480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:27.604 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.605 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:28:27.634 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:27.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:27.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:27.827 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:28.057 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:28.058 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:28.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3701280457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:28:28.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:28 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:28:28.269 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:28 smithi163 ceph-mon[213841]: pgmap v1361: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T04:28:28.280 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:28:28.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 59 16 81323 81307 86.0788 45.125 0.00360733 0.0115932 2023-10-09T04:28:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3701280457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:28:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:28 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:28:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:28 smithi181 ceph-mon[196979]: pgmap v1361: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T04:28:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3701280457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:28:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:28 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:28:28.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:28 smithi163 ceph-mon[221067]: pgmap v1361: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T04:28:28.941 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:28.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:28.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:28.990 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:28:29.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:29.097 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:29.143 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:29.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:29.249 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:29.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:28:29.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:29 smithi163 ceph-mon[213841]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T04:28:29.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:29 smithi163 ceph-mon[213841]: failed to encode map e520 with expected crc 2023-10-09T04:28:29.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:28:29.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:29 smithi163 ceph-mon[221067]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T04:28:29.279 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:29 smithi163 ceph-mon[221067]: failed to encode map e520 with expected crc 2023-10-09T04:28:29.312 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:29.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:28:29.503525+0000 min lat: 0.00118268 max lat: 0.360884 avg lat: 0.0116867 2023-10-09T04:28:29.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:28:29.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 60 16 82142 82126 85.4975 51.1875 0.00376824 0.0116867 2023-10-09T04:28:29.523 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:29.761 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:28:29.762 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:29 smithi181 ceph-mon[196979]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T04:28:29.762 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:29 smithi181 ceph-mon[196979]: failed to encode map e520 with expected crc 2023-10-09T04:28:29.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:29.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:30.002 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:28:30.072 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:30.140 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:30.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:30.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:30.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:30.404 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:30 smithi181 ceph-mon[196979]: pgmap v1363: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s 2023-10-09T04:28:30.409 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:30.409 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:30 smithi163 ceph-mon[213841]: pgmap v1363: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s 2023-10-09T04:28:30.410 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:30 smithi163 ceph-mon[221067]: pgmap v1363: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s 2023-10-09T04:28:30.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 61 16 83044 83028 85.0203 56.375 0.00390653 0.0117517 2023-10-09T04:28:30.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:30.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:30.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:30.783 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:28:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:28:30] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.43.0" 2023-10-09T04:28:30.791 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:30.815 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:30.899 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:30.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:31.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:31.061 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:31.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:31.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:31.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:31.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:31.337 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:28:31.380 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:31.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 62 16 83892 83876 84.504 53 0.00792878 0.0118289 2023-10-09T04:28:31.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2877972531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:31.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:31.534 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:31.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:31.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:31.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2877972531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:31.553 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:31.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2877972531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:31.554 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:31.583 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:31.583 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:31.651 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:31.701 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:31.818 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:28:31.826 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:31.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:31.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:31.960 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:32.192 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:32.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:32.298 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:32.371 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:32.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 63 16 84694 84678 83.9585 50.125 0.0143264 0.0119063 2023-10-09T04:28:32.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:28:32.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:28:32.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: failed to encode map e521 with expected crc 2023-10-09T04:28:32.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:32.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T04:28:32.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: failed to encode map e521 with expected crc 2023-10-09T04:28:32.523 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:32 smithi181 ceph-mon[196979]: pgmap v1365: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:32.561 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:32.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:32.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:32.568 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:28:32.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:28:32.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: failed to encode map e521 with expected crc 2023-10-09T04:28:32.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:32.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T04:28:32.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: failed to encode map e521 with expected crc 2023-10-09T04:28:32.569 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[221067]: pgmap v1365: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:32.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:28:32.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:28:32.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: failed to encode map e521 with expected crc 2023-10-09T04:28:32.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:32.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T04:28:32.570 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: failed to encode map e521 with expected crc 2023-10-09T04:28:32.571 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:32 smithi163 ceph-mon[213841]: pgmap v1365: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:28:32.616 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:28:32.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:32.696 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:32.742 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:32.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:32.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:32.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:32.909 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:33.095 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:33.116 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:33.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:33.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:33.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:33.283 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:33.300 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:33.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 64 16 85509 85493 83.4427 50.9375 0.0120371 0.0119787 2023-10-09T04:28:33.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:33.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:33.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:33.665 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:33.758 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:33.918 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:33.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:34.027 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:34.054 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:34.105 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:34.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:34.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:34.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:34.287 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:34.388 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:34.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:34.498 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:34.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 65 16 86311 86295 82.9303 50.125 0.0242876 0.0120536 2023-10-09T04:28:34.567 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:34.663 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:34.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:34.777 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:34.846 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:34.858 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:28:34.932 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:34.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:34.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:35.043 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:35.090 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:35.106 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:28:35.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:35 smithi181 ceph-mon[196979]: pgmap v1366: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s 2023-10-09T04:28:35.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:35 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/934326750' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:28:35.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:28:35.108 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:35.201 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:35.201 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:35.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:35.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:35.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:35.373 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[213841]: pgmap v1366: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/934326750' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[221067]: pgmap v1366: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/934326750' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:28:35.378 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:35.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:35.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:35.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 66 16 87241 87225 82.5545 58.125 0.00306286 0.0121044 2023-10-09T04:28:35.580 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:35.614 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:35.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:35.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:35.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:35.795 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:35.866 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:35.866 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:35.867 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:35.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:35.936 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:35.978 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:36.016 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:36.056 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:36.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:36.080 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.081 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:28:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:36.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:36.166 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:28:36.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:36.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:28:36.263 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:36.352 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:36.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/934326750' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:28:36.358 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:36 smithi181 ceph-mon[196979]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T04:28:36.359 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:36 smithi181 ceph-mon[196979]: failed to encode map e522 with expected crc 2023-10-09T04:28:36.359 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:36 smithi181 ceph-mon[196979]: failed to encode map e522 with expected crc 2023-10-09T04:28:36.359 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1869029183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:36.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:36.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:36.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:36.490 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:36.503 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 67 16 87928 87912 81.9634 42.9375 0.0157216 0.0121942 2023-10-09T04:28:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/934326750' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:28:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[213841]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T04:28:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[213841]: failed to encode map e522 with expected crc 2023-10-09T04:28:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[213841]: failed to encode map e522 with expected crc 2023-10-09T04:28:36.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1869029183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:36.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/934326750' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:28:36.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[221067]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T04:28:36.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[221067]: failed to encode map e522 with expected crc 2023-10-09T04:28:36.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[221067]: failed to encode map e522 with expected crc 2023-10-09T04:28:36.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1869029183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:36.641 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:36.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:36.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:36.736 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:36.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:36.810 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:36.817 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:36.908 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:36.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:36.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:37.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:37.037 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:37.081 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:37.235 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:37 smithi181 ceph-mon[196979]: pgmap v1368: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T04:28:37.391 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:37 smithi163 ceph-mon[221067]: pgmap v1368: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T04:28:37.392 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:37 smithi163 ceph-mon[213841]: pgmap v1368: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T04:28:37.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:37.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:37.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:37.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 68 16 88900 88884 81.6515 60.75 0.0164815 0.0122425 2023-10-09T04:28:37.531 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:37.574 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:37.637 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:37.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:37.741 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:37.790 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:38.365 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:38.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:38.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:38.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:38.513 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 69 16 89737 89721 81.2264 52.3125 0.0198919 0.0123067 2023-10-09T04:28:38.547 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:38.847 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:28:38.891 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:28:38.891 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:28:38.891 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:28:38.891 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:28:38.891 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:110800 9.7 9:edeb27ba:::benchmark_data_smithi163_464650_object12074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:20:00.967321+0000", 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 517.91710407899996, 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.133645886, 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 110800 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.892 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967321+0000", 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967321+0000", 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.893 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967320+0000", 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967327+0000", 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7200000000000006e-06 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967341+0000", 2023-10-09T04:28:38.894 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4019000000000001e-05 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967348+0000", 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1210000000000001e-06 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967524+0000", 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017569300000000001 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.895 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967643+0000", 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119367 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.967759+0000", 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115884 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.896 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.971245+0000", 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0034856560000000002 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.972618+0000", 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013727710000000001 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.986601+0000", 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013982827 2023-10-09T04:28:38.897 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:00.986611+0000", 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0759e-05 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:02.030984+0000", 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0443723739999999 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.898 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:02.030995+0000", 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1212e-05 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:03.100902+0000", 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.069907159 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.899 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:20:03.100967+0000", 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4851000000000006e-05 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18942 10.6 10:65a48f0d:::benchmark_data_smithi163_523108_object18941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998930+0000", 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 372.88549533000003, 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1092201259999999, 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.900 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18942 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998930+0000", 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.901 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998930+0000", 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998928+0000", 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.902 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998967+0000", 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8402000000000003e-05 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998981+0000", 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4372999999999999e-05 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998989+0000", 2023-10-09T04:28:38.903 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6279999999999996e-06 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999285+0000", 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029607 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999296+0000", 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1276e-05 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.904 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992447+0000", 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99315083699999995 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992457+0000", 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7880000000000001e-06 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.905 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.099919+0000", 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.107462452 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.100039+0000", 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000120226 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.100200+0000", 2023-10-09T04:28:38.906 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016061299999999999 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.103011+0000", 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002811502 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.107047+0000", 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0040354299999999996 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.907 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.107095+0000", 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8204000000000001e-05 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.107151+0000", 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5983999999999998e-05 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.908 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.108150+0000", 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0009989720000000001 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18934 10.a 10:5bb0a4f3:::benchmark_data_smithi163_523108_object18933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 372.88542884100002, 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.089428222, 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.909 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18934 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.910 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.998996+0000", 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999025+0000", 2023-10-09T04:28:38.911 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9019000000000001e-05 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999033+0000", 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4149999999999999e-06 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999038+0000", 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8799999999999999e-06 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.912 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999631+0000", 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00059334500000000001 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:25.999639+0000", 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2660000000000004e-06 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.913 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992278+0000", 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99263969799999996 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.992293+0000", 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4462e-05 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.083758+0000", 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.091465635 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.914 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.084269+0000", 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0005109 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.084451+0000", 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000181521 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.915 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.085405+0000", 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00095371799999999999 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.085693+0000", 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028888200000000002 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.087633+0000", 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019396140000000001 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.916 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.087675+0000", 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.2209000000000002e-05 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.088425+0000", 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00074923399999999999 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.917 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:18946 10.6 10:6fc4e677:::benchmark_data_smithi163_523108_object18945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:22:26.014450+0000", 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 372.86997467200001, 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1049869910000001, 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.918 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 18946 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014450+0000", 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014450+0000", 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.919 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014449+0000", 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014536+0000", 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.7131000000000003e-05 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.920 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014549+0000", 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2559e-05 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014554+0000", 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7010000000000001e-06 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.921 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014605+0000", 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0887999999999998e-05 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.014615+0000", 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7159999999999998e-06 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.994586+0000", 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.979970966 2023-10-09T04:28:38.922 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:26.994596+0000", 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0206e-05 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.101902+0000", 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.107305389 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.923 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.102016+0000", 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011417900000000001 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.102161+0000", 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014525900000000001 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.924 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.105251+0000", 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0030898129999999998 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.110748+0000", 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005497325 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.118450+0000", 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0077021490000000001 2023-10-09T04:28:38.925 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.118525+0000", 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4603000000000004e-05 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:22:28.119437+0000", 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00091260099999999997 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.926 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:83379 10.13 10:cd45996f:::benchmark_data_smithi163_523108_object9363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e433)", 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:24:21.427243+0000", 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 257.45718168799999, 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0241408009999999, 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.927 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 83379 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427243+0000", 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427243+0000", 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.928 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427242+0000", 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427249+0000", 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2420000000000003e-06 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.929 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427277+0000", 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7767e-05 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427285+0000", 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9140000000000008e-06 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427350+0000", 2023-10-09T04:28:38.930 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5129999999999995e-05 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:21.427361+0000", 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0841e-05 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:22.451301+0000", 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.023939822 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.931 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:22.451384+0000", 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3659000000000007e-05 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116390 10.19 10:9fb1e220:::benchmark_data_smithi163_523108_object42374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:08.686298+0000", 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 150.198126898, 2023-10-09T04:28:38.932 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2121686770000002, 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116390 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686298+0000", 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.933 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686298+0000", 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686297+0000", 2023-10-09T04:28:38.934 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686304+0000", 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.238e-06 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686319+0000", 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4839e-05 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.935 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686326+0000", 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3479999999999998e-06 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686381+0000", 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4443e-05 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.936 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686532+0000", 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015127200000000001 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.686654+0000", 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121499 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.694658+0000", 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080045529999999993 2023-10-09T04:28:38.937 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.694668+0000", 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6169999999999996e-06 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.745344+0000", 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.050676253 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.938 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.745353+0000", 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3060000000000003e-06 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.880595+0000", 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1352419629999999 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.880724+0000", 2023-10-09T04:28:38.939 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012829200000000001 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.898365+0000", 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017641114999999999 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.898467+0000", 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102223 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.940 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116394 10.13 10:cc7432af:::benchmark_data_smithi163_523108_object42378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:08.717745+0000", 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 150.16667969299999, 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.25461892, 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.941 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116394 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717745+0000", 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.942 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717745+0000", 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717744+0000", 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717749+0000", 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2669999999999997e-06 2023-10-09T04:28:38.943 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717762+0000", 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2279999999999999e-05 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717767+0000", 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0880000000000002e-06 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717812+0000", 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5391e-05 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.717825+0000", 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2816000000000001e-05 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.743904+0000", 2023-10-09T04:28:38.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0260792169999999 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.743916+0000", 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1987000000000001e-05 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.882737+0000", 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1388202709999999 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.946 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.882891+0000", 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015418699999999999 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883051+0000", 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015984300000000001 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.947 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.890129+0000", 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007078562 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.890138+0000", 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5089999999999993e-06 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.971433+0000", 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.081295390999999995 2023-10-09T04:28:38.948 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.971498+0000", 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4721000000000006e-05 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.972364+0000", 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086659300000000003 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.949 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116407 10.9 10:95c21b87:::benchmark_data_smithi163_523108_object42391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:08.920103+0000", 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 149.96432229000001, 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9767536859999999, 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116407 2023-10-09T04:28:38.950 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920103+0000", 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920103+0000", 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.951 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920101+0000", 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920111+0000", 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6269999999999995e-06 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.952 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920135+0000", 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3859999999999999e-05 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920143+0000", 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.0930000000000003e-06 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920196+0000", 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3708000000000001e-05 2023-10-09T04:28:38.953 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.920205+0000", 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9949999999999994e-06 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.749913+0000", 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.82970755399999996 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.954 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.749921+0000", 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3950000000000001e-06 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.877463+0000", 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.127541417 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.955 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.877598+0000", 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000135048 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.877933+0000", 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00033538900000000001 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.886333+0000", 2023-10-09T04:28:38.956 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0083995149999999998 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.886367+0000", 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4001000000000002e-05 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.895837+0000", 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0094706249999999999 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.957 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.895896+0000", 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8851999999999998e-05 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.896857+0000", 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096041300000000002 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.958 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116419 10.3 10:c5a65a54:::benchmark_data_smithi163_523108_object42403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.046145+0000", 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 149.838279954, 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.939067009, 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116419 2023-10-09T04:28:38.959 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046145+0000", 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046145+0000", 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.960 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046144+0000", 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046151+0000", 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7470000000000005e-06 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046169+0000", 2023-10-09T04:28:38.961 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7448e-05 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046176+0000", 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8379999999999996e-06 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046236+0000", 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0713e-05 2023-10-09T04:28:38.962 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.046246+0000", 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5650000000000007e-06 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.744857+0000", 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.69861123000000003 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.963 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.744867+0000", 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.91e-06 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.969366+0000", 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.224499147 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.969470+0000", 2023-10-09T04:28:38.964 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103259 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.969630+0000", 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016028700000000001 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.973000+0000", 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003370343 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.965 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.973712+0000", 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00071128399999999996 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.984375+0000", 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010663053 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.966 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.984437+0000", 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2472999999999996e-05 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.985212+0000", 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00077514700000000001 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.967 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116428 10.19 10:9860fefa:::benchmark_data_smithi163_523108_object42412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.777555+0000", 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 149.10686984500001, 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1244661600000001, 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116428 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.968 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777555+0000", 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777555+0000", 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777555+0000", 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967296 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777557+0000", 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7400000000000001e-06 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777560+0000", 2023-10-09T04:28:38.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1159999999999999e-06 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777562+0000", 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2249999999999999e-06 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777612+0000", 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0223999999999999e-05 2023-10-09T04:28:38.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.777620+0000", 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.576e-06 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.880746+0000", 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.103125637 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.880876+0000", 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013064700000000001 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.881260+0000", 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000384096 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.888045+0000", 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067845689999999998 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.898264+0000", 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010219143 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.901049+0000", 2023-10-09T04:28:38.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002784973 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.901106+0000", 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6530000000000003e-05 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.902021+0000", 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000915894 2023-10-09T04:28:38.975 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116444 10.19 10:9f58b9ed:::benchmark_data_smithi163_523108_object42428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.827580+0000", 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 149.05684478699999, 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0808723689999999, 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.976 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116444 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827580+0000", 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827580+0000", 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827580+0000", 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827583+0000", 2023-10-09T04:28:38.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7690000000000001e-06 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827588+0000", 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7559999999999996e-06 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827592+0000", 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9480000000000001e-06 2023-10-09T04:28:38.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827659+0000", 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6497999999999999e-05 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.827671+0000", 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2383e-05 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883157+0000", 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0554860880000001 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883298+0000", 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014063899999999999 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:28:38.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883583+0000", 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00028501399999999999 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.898338+0000", 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014754793 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.900762+0000", 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0024238100000000002 2023-10-09T04:28:38.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.907429+0000", 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066677230000000004 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.907509+0000", 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9450000000000007e-05 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.908453+0000", 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00094395199999999996 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:120721 10.3 10:c6366834:::benchmark_data_smithi163_523108_object46705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:20.329930+0000", 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 138.55449541300001, 2023-10-09T04:28:38.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1570701589999999, 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120721 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.329930+0000", 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.329930+0000", 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.329929+0000", 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:28:38.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.329934+0000", 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5049999999999998e-06 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.329946+0000", 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1646e-05 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.329952+0000", 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2280000000000003e-06 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.332757+0000", 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028045470000000001 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.332928+0000", 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017091099999999999 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.333348+0000", 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00042065700000000001 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.335565+0000", 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002216759 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337408+0000", 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001843077 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.366395+0000", 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.028987176999999999 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.366404+0000", 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1279999999999997e-06 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.477032+0000", 2023-10-09T04:28:38.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1106274700000001 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.477150+0000", 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117855 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.486930+0000", 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0097799229999999994 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.487000+0000", 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0290999999999995e-05 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121923 10.9 10:94745115:::benchmark_data_smithi163_523108_object47907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:23.671684+0000", 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.212740798, 2023-10-09T04:28:38.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0976354580000001, 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121923 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:38.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671684+0000", 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671684+0000", 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671683+0000", 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:38.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671690+0000", 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1640000000000004e-06 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671702+0000", 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2417e-05 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671709+0000", 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8510000000000001e-06 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671775+0000", 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5654999999999994e-05 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.671922+0000", 2023-10-09T04:28:38.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014744700000000001 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.672357+0000", 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000435038 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.674403+0000", 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020453450000000001 2023-10-09T04:28:38.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.680070+0000", 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056670610000000001 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.680288+0000", 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00021815100000000001 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:38.999 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745714+0000", 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0654256200000001 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for missing object", 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745838+0000", 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012393899999999999 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.769248+0000", 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023410561999999999 2023-10-09T04:28:39.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.769320+0000", 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1636000000000003e-05 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121927 10.13 10:ceff809e:::benchmark_data_smithi163_523108_object47911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:23.688594+0000", 2023-10-09T04:28:39.001 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.195830839, 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0451364809999999, 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121927 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688594+0000", 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688594+0000", 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688593+0000", 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:39.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688599+0000", 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-06 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688612+0000", 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3157e-05 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688618+0000", 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3849999999999999e-06 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688668+0000", 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9314000000000003e-05 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.688675+0000", 2023-10-09T04:28:39.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.418e-06 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.707449+0000", 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0187736409999999 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.707555+0000", 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106631 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.707649+0000", 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3213000000000004e-05 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.717045+0000", 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093966770000000008 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.717381+0000", 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00033625000000000001 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733625+0000", 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016243934000000002 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733684+0000", 2023-10-09T04:28:39.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8625999999999997e-05 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733731+0000", 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6743999999999998e-05 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121931 10.13 10:c8d70889:::benchmark_data_smithi163_523108_object47915:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.009 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:23.698961+0000", 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.18546426, 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0348780419999999, 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121931 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.698961+0000", 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.698961+0000", 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.698960+0000", 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.698965+0000", 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5770000000000004e-06 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.698975+0000", 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0699999999999999e-05 2023-10-09T04:28:39.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.698980+0000", 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6140000000000002e-06 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.699034+0000", 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3482000000000001e-05 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.699041+0000", 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5889999999999997e-06 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.709977+0000", 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0109357990000001 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.710080+0000", 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010334 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.710183+0000", 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000102611 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.717455+0000", 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0072725719999999997 2023-10-09T04:28:39.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.718693+0000", 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012372259999999999 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733767+0000", 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015073849 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733805+0000", 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.82e-05 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733839+0000", 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4162999999999998e-05 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.017 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121947 10.13 10:c802c200:::benchmark_data_smithi163_523108_object47931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:23.763327+0000", 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.12109825600001, 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97060582200000001, 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.018 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121947 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763327+0000", 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763327+0000", 2023-10-09T04:28:39.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763326+0000", 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763332+0000", 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0000000000000002e-06 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763343+0000", 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1667e-05 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763349+0000", 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0749999999999997e-06 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.021 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763410+0000", 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1659e-05 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.763422+0000", 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1999e-05 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.712368+0000", 2023-10-09T04:28:39.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.94894587399999997 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.712474+0000", 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105651 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.712684+0000", 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000210504 2023-10-09T04:28:39.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.717461+0000", 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0047763609999999998 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.718802+0000", 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0013414519999999999 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733868+0000", 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015066313 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733902+0000", 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4122000000000003e-05 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.733933+0000", 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0222e-05 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121951 10.9 10:90e455f2:::benchmark_data_smithi163_523108_object47935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:23.810183+0000", 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.07424258399999, 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.95889491900000001, 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.026 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.029 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121951 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810183+0000", 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810183+0000", 2023-10-09T04:28:39.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810181+0000", 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810188+0000", 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7209999999999994e-06 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810206+0000", 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.7600000000000001e-05 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810212+0000", 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0970000000000002e-06 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810264+0000", 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1881e-05 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.810272+0000", 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3499999999999997e-06 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.748136+0000", 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93786346600000003 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.748251+0000", 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114809 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.748361+0000", 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000110605 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.752691+0000", 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043298299999999998 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.760157+0000", 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0074657860000000003 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.768946+0000", 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0087894500000000007 2023-10-09T04:28:39.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.768993+0000", 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6417e-05 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.769077+0000", 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4728999999999995e-05 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.036 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121952 10.19 10:9a4445dd:::benchmark_data_smithi163_523108_object47936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:23.811975+0000", 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 135.07245053599999, 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.933401115, 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.037 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121952 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.811975+0000", 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.811975+0000", 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.811974+0000", 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.811978+0000", 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5059999999999997e-06 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.811989+0000", 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0592e-05 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.811993+0000", 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1659999999999996e-06 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.812039+0000", 2023-10-09T04:28:39.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5382e-05 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:23.812047+0000", 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.072e-06 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.718529+0000", 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.90648283299999999 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.718626+0000", 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6591999999999999e-05 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.718726+0000", 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9831e-05 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.725518+0000", 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067924630000000003 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745223+0000", 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.01970421 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745290+0000", 2023-10-09T04:28:39.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7816000000000003e-05 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745340+0000", 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9988999999999999e-05 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745376+0000", 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5349999999999999e-05 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.044 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:121994 10.19 10:9bdec6f3:::benchmark_data_smithi163_523108_object47978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:24.075167+0000", 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 134.80925815500001, 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.67033666800000002, 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 121994 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075167+0000", 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075167+0000", 2023-10-09T04:28:39.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075165+0000", 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075172+0000", 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.007e-06 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075191+0000", 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9026999999999999e-05 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075199+0000", 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9629999999999998e-06 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075249+0000", 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9394999999999997e-05 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.075257+0000", 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5709999999999998e-06 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.720680+0000", 2023-10-09T04:28:39.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.64542279499999999 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.720769+0000", 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.8765999999999997e-05 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.720859+0000", 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0292999999999998e-05 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.725524+0000", 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.004664975 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745405+0000", 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.019880937000000001 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745429+0000", 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.4196999999999999e-05 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745468+0000", 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8778e-05 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745504+0000", 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5729000000000003e-05 2023-10-09T04:28:39.052 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:122011 10.19 10:9ab8d928:::benchmark_data_smithi163_523108_object47995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:24.160915+0000", 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 134.723509834, 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.58470906700000003, 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:28:39.053 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 122011 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160915+0000", 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160915+0000", 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160914+0000", 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:39.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160920+0000", 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3110000000000002e-06 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160935+0000", 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5197e-05 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160942+0000", 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0620000000000003e-06 2023-10-09T04:28:39.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160987+0000", 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4937e-05 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.160997+0000", 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.2860000000000005e-06 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:39.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.722843+0000", 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.56184590499999998 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.722941+0000", 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8147999999999996e-05 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.723045+0000", 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010377799999999999 2023-10-09T04:28:39.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.725529+0000", 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002484207 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745534+0000", 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.020004937 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745561+0000", 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7362e-05 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745592+0000", 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1417000000000001e-05 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:24.745624+0000", 2023-10-09T04:28:39.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.1902000000000001e-05 2023-10-09T04:28:39.061 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.061 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.061 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.061 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:39.061 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:39.061 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:28:39.062 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:38 smithi181 ceph-mon[196979]: pgmap v1369: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-09T04:28:39.081 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:38 smithi163 ceph-mon[213841]: pgmap v1369: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-09T04:28:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:38 smithi163 ceph-mon[221067]: pgmap v1369: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-09T04:28:39.090 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:39.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:39.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:39.233 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:39.275 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:39.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:39.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:39.379 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:39.448 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:39.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 70 16 90420 90404 80.676 42.6875 0.00836918 0.0123898 2023-10-09T04:28:39.709 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:28:39.845 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:39.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:39.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:39.889 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:39.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:39.991 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:40.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:40.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:40.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:40.196 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:40.255 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:40.266 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:40.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:40.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:40.451 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:40.496 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:40.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 71 16 91262 91246 80.2811 52.625 0.0116256 0.0124515 2023-10-09T04:28:40.604 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:40.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:40.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:40.713 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.793 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:40.847 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:40.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:40.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:40.950 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:40.974 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:28:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:28:40] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.43.0" 2023-10-09T04:28:40.976 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:41.027 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:41.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:41.133 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:28:41.135 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:41.141 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:28:41.206 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:41 smithi181 ceph-mon[196979]: pgmap v1370: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s 2023-10-09T04:28:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/723755013' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:41 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/8937187' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:28:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:41 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:28:41.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[221067]: pgmap v1370: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s 2023-10-09T04:28:41.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/723755013' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:41.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:41.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/8937187' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:28:41.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:28:41.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[213841]: pgmap v1370: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s 2023-10-09T04:28:41.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/723755013' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:41.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:28:41.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/8937187' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:28:41.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:41 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:28:41.483 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:41.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:41.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 72 16 92009 91993 79.8143 46.6875 0.0219041 0.0125233 2023-10-09T04:28:41.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:41.624 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:42.143 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:42 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:42.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:42 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:42.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:42 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:28:42.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:42 smithi181 ceph-mon[196979]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T04:28:42.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:42 smithi181 ceph-mon[196979]: failed to encode map e523 with expected crc 2023-10-09T04:28:42.144 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:42 smithi181 ceph-mon[196979]: failed to encode map e523 with expected crc 2023-10-09T04:28:42.469 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:28:42.469 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:28:42.469 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:28:42.469 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:28:42.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:42.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:42.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:28:42.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[213841]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[213841]: failed to encode map e523 with expected crc 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[213841]: failed to encode map e523 with expected crc 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[221067]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[221067]: failed to encode map e523 with expected crc 2023-10-09T04:28:42.479 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:42 smithi163 ceph-mon[221067]: failed to encode map e523 with expected crc 2023-10-09T04:28:42.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 73 16 92722 92706 79.3316 44.5625 0.0196177 0.0125998 2023-10-09T04:28:42.615 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:42.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:42.723 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:42.786 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:42.788 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:28:42.832 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:42.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:42.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:42.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:42.965 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:43.087 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:43.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:43.160 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:43.199 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:43.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:43.241 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:43.245 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:43.289 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:43.322 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:43.329 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:43.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:43.369 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:43.381 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:43.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:28:43.411 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:43.424 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:43 smithi181 ceph-mon[196979]: pgmap v1372: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:28:43.424 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1953611798' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:43.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:43.475 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.489 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:28:43.493 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:43.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 74 16 93456 93440 78.8796 45.875 0.019164 0.012672 2023-10-09T04:28:43.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:43 smithi163 ceph-mon[213841]: pgmap v1372: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:28:43.515 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1953611798' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:43.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:43 smithi163 ceph-mon[221067]: pgmap v1372: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:28:43.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1953611798' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:43.627 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:43.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:43.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:43.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:43.778 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:43.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:43.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:43.972 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:43.980 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:43.999 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:44.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:44.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:44.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:44.207 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:44.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:44.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:44.504 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 75 16 94248 94232 78.488 49.5 0.0315239 0.0127358 2023-10-09T04:28:44.513 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:28:44.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:44.624 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:44.690 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:44.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:44.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:44.821 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:44.861 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:44.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:44.964 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:45.011 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:45.157 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:45 smithi181 ceph-mon[196979]: pgmap v1373: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T04:28:45.286 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:45.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:45.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:45.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:45.478 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:45 smithi163 ceph-mon[213841]: pgmap v1373: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T04:28:45.478 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:45 smithi163 ceph-mon[221067]: pgmap v1373: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T04:28:45.479 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:45.486 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:45.505 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 76 16 94930 94914 78.0162 42.625 0.0156603 0.0128126 2023-10-09T04:28:45.714 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:45.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:45.822 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:45.903 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:45.954 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:45.954 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:45.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:45.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:46.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:46.165 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:46.418 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:46.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:46.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:28:46.505 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 77 16 95651 95635 77.5879 45.0625 0.0111184 0.0128775 2023-10-09T04:28:46.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:46.582 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:46.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:46.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:46.756 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:46.783 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:46.911 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:28:47.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:47.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:47.181 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:47 smithi181 ceph-mon[196979]: pgmap v1374: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-10-09T04:28:47.181 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2042745663' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:28:47.191 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:28:47.230 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:47.319 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:47.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:47 smithi163 ceph-mon[213841]: pgmap v1374: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-10-09T04:28:47.419 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2042745663' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:28:47.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:47 smithi163 ceph-mon[221067]: pgmap v1374: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-10-09T04:28:47.419 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2042745663' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:28:47.463 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:47.505 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 78 16 96328 96312 77.1358 42.3125 0.0074417 0.0129598 2023-10-09T04:28:47.528 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:47.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:47.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:47.640 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:47.703 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:47.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:47.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:47.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:47.969 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:28:47.983 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:48.072 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:48.124 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:48.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:48.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:48.211 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2042745663' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[221067]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[221067]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[221067]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[221067]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[221067]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2042745663' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[213841]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[213841]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[213841]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[213841]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:48 smithi163 ceph-mon[213841]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.231 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:48.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2042745663' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:28:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:48 smithi181 ceph-mon[196979]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T04:28:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:48 smithi181 ceph-mon[196979]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:48 smithi181 ceph-mon[196979]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:48 smithi181 ceph-mon[196979]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:48 smithi181 ceph-mon[196979]: failed to encode map e524 with expected crc 2023-10-09T04:28:48.335 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:48.479 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:48.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:48.505 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 79 16 97084 97068 76.7576 47.25 0.0084808 0.0130044 2023-10-09T04:28:48.584 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:48.642 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:48.936 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:48.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:49.045 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:49.101 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:49.211 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:49.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:49.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:49.315 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:49 smithi163 ceph-mon[213841]: pgmap v1376: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T04:28:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/558129064' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:49 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:49 smithi163 ceph-mon[221067]: pgmap v1376: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T04:28:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/558129064' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:49 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:49.398 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:49.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:2023-10-09T04:28:49.506017+0000 min lat: 0.00118268 max lat: 0.360884 avg lat: 0.0130714 2023-10-09T04:28:49.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:28:49.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 80 16 97954 97938 76.4779 54.375 0.00832093 0.0130714 2023-10-09T04:28:49.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:49 smithi181 ceph-mon[196979]: pgmap v1376: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T04:28:49.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/558129064' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:49.527 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:49 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:28:49.532 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:28:49.532 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:28:49.557 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:49.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:49.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:49.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:49.696 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:49.760 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:49.762 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:28:49.833 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:49.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:49.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:49.999 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:50.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 81 16 98496 98480 75.952 33.875 0.00339028 0.0131496 2023-10-09T04:28:50.538 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[221067]: pgmap v1378: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 30 MiB/s wr, 484 op/s 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:50 smithi163 ceph-mon[213841]: pgmap v1378: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 30 MiB/s wr, 484 op/s 2023-10-09T04:28:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:28:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:28:50.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T04:28:50.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: failed to encode map e525 with expected crc 2023-10-09T04:28:50.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:50 smithi181 ceph-mon[196979]: pgmap v1378: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 30 MiB/s wr, 484 op/s 2023-10-09T04:28:50.880 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:50.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:50.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:50.987 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:51.034 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:28:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:28:50] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2023-10-09T04:28:51.040 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:51.143 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:51.465 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:51.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:51.508 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 82 16 99373 99357 75.694 54.8125 0.00995659 0.0132057 2023-10-09T04:28:51.546 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:28:51.575 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:28:51.580 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:51.615 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:28:51.653 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:28:51.668 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:51.689 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:28:51.737 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:28:51.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3515319713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:51.781 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:28:51.823 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:28:51.837 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:51.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.838 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.839 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.839 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.839 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:28:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:51.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:51.947 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:51.980 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:52.019 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3515319713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:52.020 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3515319713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:28:52.033 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:52.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:52.136 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:52.168 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:52.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 83 16 100018 100002 75.2678 40.3125 0.00366922 0.0132787 2023-10-09T04:28:52.918 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:52 smithi181 ceph-mon[196979]: pgmap v1379: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-09T04:28:52.983 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:53.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:52 smithi163 ceph-mon[213841]: pgmap v1379: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-09T04:28:53.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:52 smithi163 ceph-mon[221067]: pgmap v1379: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-09T04:28:53.104 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:53.192 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:53.225 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:28:53.225 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:100641 11.4 11:232b4ee6:::benchmark_data_smithi163_601242_object100640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:53.213092+0000", 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.011564071, 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011602045, 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:53.226 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 100641 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213092+0000", 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.227 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213092+0000", 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213090+0000", 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:53.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213119+0000", 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8242000000000002e-05 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213130+0000", 2023-10-09T04:28:53.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1127999999999999e-05 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213136+0000", 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1829999999999999e-06 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213185+0000", 2023-10-09T04:28:53.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8878999999999997e-05 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213327+0000", 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014251800000000001 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.213498+0000", 2023-10-09T04:28:53.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000170376 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220669+0000", 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0071718329999999999 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.222140+0000", 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0014701930000000001 2023-10-09T04:28:53.232 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:100645 11.f 11:f9e5926e:::benchmark_data_smithi163_601242_object100644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:53.216551+0000", 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0081044900000000007, 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082420529999999992, 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:53.233 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 100645 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216551+0000", 2023-10-09T04:28:53.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216551+0000", 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216550+0000", 2023-10-09T04:28:53.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216605+0000", 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4901999999999997e-05 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216619+0000", 2023-10-09T04:28:53.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3956e-05 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216625+0000", 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1140000000000002e-06 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216678+0000", 2023-10-09T04:28:53.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3269000000000001e-05 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216817+0000", 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013887799999999999 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.216975+0000", 2023-10-09T04:28:53.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000157458 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:100649 11.5 11:a2835b81:::benchmark_data_smithi163_601242_object100648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:53.220454+0000", 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0042020470000000004, 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044172810000000003, 2023-10-09T04:28:53.239 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 100649 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220454+0000", 2023-10-09T04:28:53.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220454+0000", 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220452+0000", 2023-10-09T04:28:53.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220514+0000", 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1408000000000006e-05 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:53.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220528+0000", 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4347e-05 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.220535+0000", 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8050000000000001e-06 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221204+0000", 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00066926500000000005 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221320+0000", 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115709 2023-10-09T04:28:53.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221457+0000", 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000137016 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.245 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop(client.46577.0:100648 11.0 e525/507 11:01396234:::benchmark_data_smithi163_601242_object100647:head v 525'6377, mlcod=525'6375)", 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:53.221446+0000", 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.0032099229999999999, 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0035009080000000001, 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221446+0000", 2023-10-09T04:28:53.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221446+0000", 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221445+0000", 2023-10-09T04:28:53.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221499+0000", 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4447000000000002e-05 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221511+0000", 2023-10-09T04:28:53.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1398e-05 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221516+0000", 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4099999999999999e-06 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221606+0000", 2023-10-09T04:28:53.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9990999999999999e-05 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:53.221625+0000", 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.8394999999999999e-05 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:28:53.250 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:28:53.251 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 4 2023-10-09T04:28:53.251 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:28:53.298 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:53.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:53.406 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:28:53.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:28:53.481 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:53.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 84 16 100904 100888 75.031 55.375 0.0150389 0.0133213 2023-10-09T04:28:53.602 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:28:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:53.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:28:53.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:53.738 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:53.780 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1724151076' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:28:53.881 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:28:53.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:53.882 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:53.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:53.983 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:54.017 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1724151076' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:28:54.027 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:54.088 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1724151076' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:28:54.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:54.178 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:28:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:54.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:54.280 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:54.309 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:54.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:54.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:54.461 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:54.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 85 16 101493 101477 74.5815 36.8125 0.00952613 0.0134022 2023-10-09T04:28:54.533 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:54.783 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:54.786 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:54.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:54.794 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:28:54.884 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:54.896 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:28:54.971 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:55.008 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:28:55.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1724151076' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:28:55.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:54 smithi181 ceph-mon[196979]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T04:28:55.069 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:54 smithi181 ceph-mon[196979]: failed to encode map e526 with expected crc 2023-10-09T04:28:55.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:54 smithi181 ceph-mon[196979]: failed to encode map e526 with expected crc 2023-10-09T04:28:55.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:54 smithi181 ceph-mon[196979]: pgmap v1381: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 34 MiB/s wr, 548 op/s 2023-10-09T04:28:55.070 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:54 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:55.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1724151076' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:28:55.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[221067]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T04:28:55.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[221067]: failed to encode map e526 with expected crc 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[221067]: failed to encode map e526 with expected crc 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[221067]: pgmap v1381: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 34 MiB/s wr, 548 op/s 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1724151076' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[213841]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[213841]: failed to encode map e526 with expected crc 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[213841]: failed to encode map e526 with expected crc 2023-10-09T04:28:55.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[213841]: pgmap v1381: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 34 MiB/s wr, 548 op/s 2023-10-09T04:28:55.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:54 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:55.218 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:28:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:55.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:28:55.328 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:55.363 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:55.507 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 86 16 102206 102190 74.2325 44.5625 0.00687718 0.013466 2023-10-09T04:28:56.124 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:56.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:56.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:28:56.274 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:56.387 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:28:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:56.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:28:56.499 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:56.506 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 87 16 102763 102747 73.7795 34.8125 0.0173564 0.0135491 2023-10-09T04:28:56.559 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:56.618 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:28:56.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:56.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:28:56.722 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:56.820 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:56.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:56.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:56.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:56.918 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:56.994 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:28:56.994 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:28:56.994 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:28:56.994 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:28:56.994 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:28:57.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:28:57.104 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:57.273 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:57 smithi163 ceph-mon[213841]: pgmap v1382: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T04:28:57.273 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:57 smithi163 ceph-mon[221067]: pgmap v1382: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T04:28:57.288 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:28:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:57.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:28:57.401 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:28:57.426 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:57 smithi181 ceph-mon[196979]: pgmap v1382: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-09T04:28:57.429 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:57.479 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:57.509 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 88 16 103397 103381 73.3889 39.625 0.0306213 0.0136206 2023-10-09T04:28:57.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:28:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:57.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:28:58.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:28:58.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:28:58.145 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:58.155 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:28:58.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4292826075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:58.510 INFO:tasks.radosbench.radosbench.0.smithi163.stdout: 89 16 104004 103988 72.9901 37.9375 0.0143921 0.0136956 2023-10-09T04:28:58.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4292826075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:58.519 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4292826075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:28:58.690 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:28:58.795 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:28:58.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:58.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:28:58.901 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:28:58.927 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:59.106 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:28:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:28:59.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:28:59.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:59 smithi181 ceph-mon[196979]: pgmap v1383: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T04:28:59.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:59 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:59.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4292826075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:59.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:59 smithi181 ceph-mon[196979]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T04:28:59.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:59 smithi181 ceph-mon[196979]: failed to encode map e527 with expected crc 2023-10-09T04:28:59.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:28:59 smithi181 ceph-mon[196979]: failed to encode map e527 with expected crc 2023-10-09T04:28:59.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:28:59.273 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:28:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[221067]: pgmap v1383: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T04:28:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4292826075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[221067]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T04:28:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[221067]: failed to encode map e527 with expected crc 2023-10-09T04:28:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[221067]: failed to encode map e527 with expected crc 2023-10-09T04:28:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[213841]: pgmap v1383: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T04:28:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:28:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4292826075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:28:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[213841]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T04:28:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[213841]: failed to encode map e527 with expected crc 2023-10-09T04:28:59.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:28:59 smithi163 ceph-mon[213841]: failed to encode map e527 with expected crc 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total time run: 90.0076 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Total writes made: 104776 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Write size: 65536 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Object size: 65536 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Bandwidth (MB/sec): 72.755 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Bandwidth: 33.692 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max bandwidth (MB/sec): 200.812 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min bandwidth (MB/sec): 33.875 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average IOPS: 1164 2023-10-09T04:28:59.511 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev IOPS: 539.072 2023-10-09T04:28:59.512 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max IOPS: 3213 2023-10-09T04:28:59.512 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min IOPS: 542 2023-10-09T04:28:59.512 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Average Latency(s): 0.013741 2023-10-09T04:28:59.512 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Stddev Latency(s): 0.0184288 2023-10-09T04:28:59.512 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Max latency(s): 0.360884 2023-10-09T04:28:59.512 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Min latency(s): 0.00118268 2023-10-09T04:28:59.515 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:29:00.582 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:00.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:00.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:00.690 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:00.757 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:00.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:00.890 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:29:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:29:00] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2023-10-09T04:29:00.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:00.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:01.007 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:01.078 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87402 9.d 9:b58a4af0:::benchmark_data_smithi163_464650_object87401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-10-09T04:29:01.085 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.18137050400003, 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9327528470000002, 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87402 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896605+0000", 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896603+0000", 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896662+0000", 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8721999999999998e-05 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896672+0000", 2023-10-09T04:29:01.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0342e-05 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.896678+0000", 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5890000000000002e-06 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897746+0000", 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001067901 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.897884+0000", 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013837699999999999 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.898027+0000", 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014309899999999999 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.903780+0000", 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057525900000000001 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913854+0000", 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010074411 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.913865+0000", 2023-10-09T04:29:01.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0667e-05 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904284+0000", 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.99041863600000002 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904295+0000", 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1622e-05 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829288+0000", 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9249930480000002 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829357+0000", 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9139000000000003e-05 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87427 9.d 9:b85b518c:::benchmark_data_smithi163_464650_object87426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.14823149400002, 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.9083351629999998, 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.094 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87427 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929744+0000", 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929742+0000", 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929812+0000", 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9973000000000002e-05 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929833+0000", 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0372000000000001e-05 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929840+0000", 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0380000000000002e-06 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929955+0000", 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115658 2023-10-09T04:29:01.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.929964+0000", 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.02e-06 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904451+0000", 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.97448704799999997 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904459+0000", 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829570+0000", 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9251109199999998 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829657+0000", 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6693000000000005e-05 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.829772+0000", 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115109 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837839+0000", 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0080666330000000001 2023-10-09T04:29:01.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837847+0000", 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5720000000000003e-06 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837892+0000", 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4297e-05 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837929+0000", 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7622e-05 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837980+0000", 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0816000000000003e-05 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838079+0000", 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8758000000000006e-05 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87460 9.d 9:be691015:::benchmark_data_smithi163_464650_object87459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.10730387900003, 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8737964319999998, 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.103 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87460 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970671+0000", 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970670+0000", 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970704+0000", 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3708999999999999e-05 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970712+0000", 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2910000000000007e-06 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.970718+0000", 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2909999999999998e-06 2023-10-09T04:29:01.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971579+0000", 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00086131599999999999 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.971589+0000", 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0108e-05 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904489+0000", 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93289962699999995 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904496+0000", 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3939999999999998e-06 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832602+0000", 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9281065480000001 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.832830+0000", 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00022731300000000001 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.833032+0000", 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202256 2023-10-09T04:29:01.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.837853+0000", 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048209000000000004 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838118+0000", 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000265083 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838174+0000", 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5485999999999997e-05 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844377+0000", 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062031949999999999 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844410+0000", 2023-10-09T04:29:01.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3266999999999998e-05 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844468+0000", 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7698000000000002e-05 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87469 9.d 9:b0d8b7f7:::benchmark_data_smithi163_464650_object87468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.112 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.10409522500004, 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8824357369999998, 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87469 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973880+0000", 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973879+0000", 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973909+0000", 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0063999999999999e-05 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973921+0000", 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.204e-05 2023-10-09T04:29:01.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.973927+0000", 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.2049999999999999e-06 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974432+0000", 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00050552799999999999 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:25.974441+0000", 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5380000000000002e-06 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904518+0000", 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.93007710099999996 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904525+0000", 2023-10-09T04:29:01.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9280000000000004e-06 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835034+0000", 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9305090869999999 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835189+0000", 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015497800000000001 2023-10-09T04:29:01.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.835319+0000", 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012983499999999999 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838122+0000", 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0028034420000000002 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838149+0000", 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6636999999999999e-05 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838193+0000", 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4202999999999997e-05 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856143+0000", 2023-10-09T04:29:01.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017949975 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856198+0000", 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4968e-05 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856316+0000", 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117589 2023-10-09T04:29:01.121 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87510 9.d 9:b1e25812:::benchmark_data_smithi163_464650_object87509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.05473781199998, 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8391414909999999, 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.122 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87510 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023237+0000", 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023236+0000", 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023264+0000", 2023-10-09T04:29:01.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7639000000000001e-05 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023280+0000", 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6189999999999999e-05 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023285+0000", 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4380000000000003e-06 2023-10-09T04:29:01.125 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023338+0000", 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2713999999999997e-05 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.023350+0000", 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1912999999999999e-05 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.126 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904548+0000", 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.88119805299999998 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904555+0000", 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1690000000000003e-06 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.127 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838206+0000", 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9336506519999999 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838292+0000", 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.6051999999999995e-05 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.838426+0000", 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013363900000000001 2023-10-09T04:29:01.128 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844098+0000", 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056724699999999998 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844218+0000", 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012015000000000001 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.129 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844519+0000", 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000300842 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862218+0000", 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017698935999999998 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862272+0000", 2023-10-09T04:29:01.130 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4085999999999997e-05 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862379+0000", 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.131 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87520 9.d 9:b10313ad:::benchmark_data_smithi163_464650_object87519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.04610998700002, 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.830771683, 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87520 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.132 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031865+0000", 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.133 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031864+0000", 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031940+0000", 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6348999999999997e-05 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031956+0000", 2023-10-09T04:29:01.134 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5733e-05 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.031962+0000", 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1330000000000004e-06 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032020+0000", 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8168999999999998e-05 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.135 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.032031+0000", 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0546e-05 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904577+0000", 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.87254678500000005 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.136 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904584+0000", 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8279999999999997e-06 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841092+0000", 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.936507684 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841230+0000", 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138245 2023-10-09T04:29:01.137 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.841385+0000", 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015503399999999999 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844512+0000", 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0031265519999999999 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.138 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850232+0000", 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057197300000000001 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850519+0000", 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000287342 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862488+0000", 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011968807999999999 2023-10-09T04:29:01.139 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862529+0000", 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1087999999999997e-05 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862637+0000", 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000108094 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.140 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87523 9.d 9:b2b1ff3b:::benchmark_data_smithi163_464650_object87522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.04026123699998, 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8252149690000001, 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.141 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87523 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037714+0000", 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037713+0000", 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037758+0000", 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5336000000000001e-05 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037769+0000", 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1139e-05 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037774+0000", 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6249999999999998e-06 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037811+0000", 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7234999999999998e-05 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.037819+0000", 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8599999999999993e-06 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904623+0000", 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86680350299999998 2023-10-09T04:29:01.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904630+0000", 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.5780000000000001e-06 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844552+0000", 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9399218949999999 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844685+0000", 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013337900000000001 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.844852+0000", 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000166682 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850445+0000", 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055933379999999998 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850487+0000", 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1508000000000003e-05 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856104+0000", 2023-10-09T04:29:01.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0056174129999999999 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862788+0000", 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0066833480000000004 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862832+0000", 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4190000000000002e-05 2023-10-09T04:29:01.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862929+0000", 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7040999999999995e-05 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87524 9.d 9:bf5a5ce2:::benchmark_data_smithi163_464650_object87523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:29:01.150 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.03753712599996, 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8227152589999998, 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87524 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.151 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040438+0000", 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040437+0000", 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:29:01.152 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040544+0000", 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010666399999999999 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040556+0000", 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2697999999999999e-05 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.153 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040562+0000", 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0839999999999997e-06 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040647+0000", 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.4971999999999999e-05 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.040656+0000", 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9479999999999997e-06 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904660+0000", 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86400389600000005 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904667+0000", 2023-10-09T04:29:01.155 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847526+0000", 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9428587660000001 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847651+0000", 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125136 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.156 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.847838+0000", 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018702700000000001 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850552+0000", 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.002713853 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856113+0000", 2023-10-09T04:29:01.157 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0055611580000000001 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862179+0000", 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0060656729999999997 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863021+0000", 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000842759 2023-10-09T04:29:01.158 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863060+0000", 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8272000000000002e-05 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863153+0000", 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.3684000000000006e-05 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.159 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87531 9.d 9:b4d1cf98:::benchmark_data_smithi163_464650_object87530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.036509615, 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.836533765, 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.160 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87531 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041466+0000", 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041465+0000", 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041513+0000", 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8566999999999998e-05 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.162 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041522+0000", 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9020000000000005e-06 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041528+0000", 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041602+0000", 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.4530000000000006e-05 2023-10-09T04:29:01.163 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.041611+0000", 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2819999999999996e-06 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904688+0000", 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.86307752000000004 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.164 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904695+0000", 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4989999999999999e-06 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850572+0000", 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9458773759999999 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850685+0000", 2023-10-09T04:29:01.165 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011277699999999999 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.850827+0000", 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014244099999999999 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862146+0000", 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011318701 2023-10-09T04:29:01.166 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862451+0000", 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00030508599999999999 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862741+0000", 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00029011400000000001 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.167 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877820+0000", 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015078775000000001 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877869+0000", 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9407999999999999e-05 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.877999+0000", 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000129868 2023-10-09T04:29:01.168 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87536 9.d 9:b13d2384:::benchmark_data_smithi163_464650_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.02602720200002, 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.826285253, 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.169 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87536 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.170 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051948+0000", 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.051947+0000", 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052032+0000", 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5885999999999997e-05 2023-10-09T04:29:01.171 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052049+0000", 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6867e-05 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052056+0000", 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9349999999999997e-06 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.172 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052117+0000", 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0758999999999999e-05 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.052127+0000", 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0301e-05 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904719+0000", 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.85259186799999997 2023-10-09T04:29:01.173 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904726+0000", 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7989999999999996e-06 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853627+0000", 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9489006930000001 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.174 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853769+0000", 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000142619 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.853901+0000", 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013205399999999999 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862421+0000", 2023-10-09T04:29:01.175 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085194190000000003 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862735+0000", 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00031399200000000003 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862747+0000", 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2238e-05 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.176 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878064+0000", 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015316585000000001 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878120+0000", 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6338999999999997e-05 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878233+0000", 2023-10-09T04:29:01.177 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113268 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87542 9.d 9:b8ae8bf6:::benchmark_data_smithi163_464650_object87541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.01652562200002, 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.817112244, 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.178 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87542 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.179 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061450+0000", 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061448+0000", 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.180 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061495+0000", 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6932999999999999e-05 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061515+0000", 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061522+0000", 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8499999999999996e-06 2023-10-09T04:29:01.181 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061580+0000", 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8037e-05 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061591+0000", 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1099e-05 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.182 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904748+0000", 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84315744100000001 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904755+0000", 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7880000000000001e-06 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856351+0000", 2023-10-09T04:29:01.183 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9515959619999999 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856435+0000", 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3497000000000004e-05 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.856590+0000", 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015579299999999999 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.184 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862697+0000", 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0061066360000000004 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862753+0000", 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6106000000000001e-05 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.185 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862991+0000", 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023799100000000001 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878381+0000", 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.015390452000000001 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878432+0000", 2023-10-09T04:29:01.186 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0779999999999998e-05 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878562+0000", 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012949399999999999 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87544 9.d 9:b2959611:::benchmark_data_smithi163_464650_object87543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.187 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 575.01638059699997, 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.8172143809999999, 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87544 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.188 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061595+0000", 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061594+0000", 2023-10-09T04:29:01.189 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061631+0000", 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6940000000000002e-05 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061641+0000", 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.5650000000000007e-06 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.190 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061645+0000", 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9400000000000001e-06 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061692+0000", 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6638000000000003e-05 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.191 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.061705+0000", 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2525e-05 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904777+0000", 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.84307196799999995 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904783+0000", 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7599999999999997e-06 2023-10-09T04:29:01.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859252+0000", 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.954468756 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859377+0000", 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000124423 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.859542+0000", 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016522800000000001 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862758+0000", 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032164730000000001 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.194 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.862963+0000", 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000205105 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871931+0000", 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0089675880000000003 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878651+0000", 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0067196959999999998 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.195 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878706+0000", 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5047999999999999e-05 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878809+0000", 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000103266 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.196 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87579 9.d 9:b6955889:::benchmark_data_smithi163_464650_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 574.95908359400005, 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.760215788, 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.197 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87579 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118892+0000", 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.198 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118890+0000", 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118943+0000", 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2917000000000002e-05 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.199 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118957+0000", 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3553000000000001e-05 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.118963+0000", 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9329999999999998e-06 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119027+0000", 2023-10-09T04:29:01.200 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4450999999999997e-05 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.119039+0000", 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1657999999999999e-05 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904808+0000", 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.78576894799999997 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.201 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904814+0000", 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.472e-06 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863179+0000", 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.958364789 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.202 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863297+0000", 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011818 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.863439+0000", 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000141836 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.871995+0000", 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085565290000000002 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.203 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872008+0000", 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2779000000000001e-05 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878306+0000", 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062979710000000003 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.204 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878882+0000", 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00057614199999999997 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878977+0000", 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4944000000000004e-05 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879107+0000", 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013005199999999999 2023-10-09T04:29:01.205 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87597 9.d 9:ba722c6f:::benchmark_data_smithi163_464650_object87596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 574.93070487800003, 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.741504747, 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.206 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87597 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:29:01.207 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147270+0000", 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147306+0000", 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6227999999999999e-05 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.208 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147315+0000", 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4420000000000004e-06 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147321+0000", 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3900000000000001e-06 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.209 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147373+0000", 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1922000000000002e-05 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.147383+0000", 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8160000000000005e-06 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904836+0000", 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75745349399999995 2023-10-09T04:29:01.210 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904843+0000", 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9360000000000002e-06 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.211 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866166+0000", 2023-10-09T04:29:01.211 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:29:01.212 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:29:01.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:01 smithi163 ceph-mon[221067]: pgmap v1385: 119 pgs: 119 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-10-09T04:29:01.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2349394683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:29:01.213 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:01 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:29:01.213 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9613231419999999 2023-10-09T04:29:01.213 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.213 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.213 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.213 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866285+0000", 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011870399999999999 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.866435+0000", 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015030299999999999 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.875034+0000", 2023-10-09T04:29:01.214 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0085985039999999999 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878346+0000", 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033125379999999999 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878607+0000", 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000261158 2023-10-09T04:29:01.215 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888598+0000", 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0099901480000000008 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888652+0000", 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4688999999999998e-05 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888775+0000", 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012282100000000001 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87606 9.d 9:bd98b1c5:::benchmark_data_smithi163_464650_object87605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 574.909855472, 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7216060710000001, 2023-10-09T04:29:01.217 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87606 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:29:01.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168120+0000", 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168119+0000", 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.219 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168170+0000", 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1724000000000001e-05 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168183+0000", 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2322e-05 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.220 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168188+0000", 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.011e-06 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168245+0000", 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7197999999999997e-05 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.168257+0000", 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1742999999999999e-05 2023-10-09T04:29:01.221 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904867+0000", 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.73660977999999999 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.904873+0000", 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3450000000000002e-06 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.222 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.868939+0000", 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9640662180000001 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869059+0000", 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012025 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.223 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.869200+0000", 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014040799999999999 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878017+0000", 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0088170209999999995 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878849+0000", 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00083247799999999999 2023-10-09T04:29:01.224 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879166+0000", 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000316373 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889551+0000", 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010385553 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.225 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889609+0000", 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7849999999999997e-05 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889726+0000", 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000116743 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.226 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.55504.0:87628 9.d 9:b09d3eb4:::benchmark_data_smithi163_464650_object87627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 574.82539482100003, 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.6374180369999998, 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.55504", 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/319769322", 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 87628 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.227 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252580+0000", 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.228 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252579+0000", 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252624+0000", 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5071999999999998e-05 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.229 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252637+0000", 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.272e-05 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252642+0000", 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8999999999999997e-06 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252685+0000", 2023-10-09T04:29:01.230 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3056000000000002e-05 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.252691+0000", 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.3790000000000003e-06 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905181+0000", 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.65248916000000001 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.231 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:26.905191+0000", 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0231000000000001e-05 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872040+0000", 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.966849243 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.232 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872156+0000", 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115611 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.872308+0000", 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152707 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.878257+0000", 2023-10-09T04:29:01.233 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0059484389999999998 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.879226+0000", 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00096947500000000002 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.888556+0000", 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093301310000000002 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.234 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889825+0000", 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001268948 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889882+0000", 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6885999999999997e-05 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.235 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:19:28.889998+0000", 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011613899999999999 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:81266 10.10 10:0ba0e01a:::benchmark_data_smithi163_523108_object7250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 288.90945970500002, 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.55363772300000003, 2023-10-09T04:29:01.236 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 81266 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.237 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168515+0000", 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168512+0000", 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999971 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.238 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168529+0000", 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6223999999999999e-05 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168551+0000", 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.2459e-05 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.239 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.168562+0000", 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1326e-05 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.313120+0000", 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.14455803 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.313133+0000", 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2466000000000001e-05 2023-10-09T04:29:01.240 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718112+0000", 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.404979266 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718376+0000", 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00026423799999999997 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.241 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.718728+0000", 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00035156 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.720984+0000", 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0022562839999999999 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.242 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721631+0000", 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00064677500000000004 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721710+0000", 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8672999999999997e-05 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.721809+0000", 2023-10-09T04:29:01.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.8904e-05 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:24:12.722153+0000", 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00034452899999999999 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:102183 11.4 11:2b8ba2ed:::benchmark_data_smithi163_601242_object102182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T04:29:01.244 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:55.484338+0000", 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 5.5936375260000002, 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.247574093, 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 102183 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.245 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.484338+0000", 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.484338+0000", 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.246 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.484337+0000", 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.484389+0000", 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2414999999999999e-05 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.484397+0000", 2023-10-09T04:29:01.247 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1350000000000003e-06 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.484401+0000", 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6450000000000004e-06 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.487986+0000", 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003584336 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.248 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488118+0000", 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000132352 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488300+0000", 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018231599999999999 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.249 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.726774+0000", 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.23847365500000001 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.727317+0000", 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00054347 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.731830+0000", 2023-10-09T04:29:01.250 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0045123589999999996 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.731871+0000", 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0951999999999997e-05 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.731912+0000", 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.1013999999999997e-05 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.251 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:102188 11.f 11:fe60e49f:::benchmark_data_smithi163_601242_object102187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:55.488677+0000", 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 5.5892977720000001, 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.248948, 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.252 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 102188 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488677+0000", 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.253 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488677+0000", 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488676+0000", 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488743+0000", 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.6904999999999997e-05 2023-10-09T04:29:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488752+0000", 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0759999999999992e-06 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488757+0000", 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0080000000000002e-06 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.255 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488813+0000", 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5621000000000002e-05 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488930+0000", 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117508 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:29:01.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.489058+0000", 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000127973 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.734044+0000", 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24498556799999999 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.734278+0000", 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00023384100000000001 2023-10-09T04:29:01.257 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.737505+0000", 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.003227494 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.737573+0000", 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7423999999999997e-05 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.258 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.737625+0000", 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2649999999999999e-05 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:102189 11.f 11:fdf9ee6f:::benchmark_data_smithi163_601242_object102188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:28:55.488773+0000", 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 5.5892020809999998, 2023-10-09T04:29:01.259 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.24905786099999999, 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 102189 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488773+0000", 2023-10-09T04:29:01.260 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488773+0000", 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488773+0000", 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.261 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488821+0000", 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8588999999999998e-05 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488828+0000", 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8159999999999996e-06 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:29:01.262 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.488832+0000", 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0969999999999999e-06 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.730335+0000", 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.241502413 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.730478+0000", 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000143079 2023-10-09T04:29:01.263 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.730628+0000", 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015048399999999999 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.735464+0000", 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048359800000000001 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.264 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.736659+0000", 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001195205 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.737732+0000", 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0010732319999999999 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.737770+0000", 2023-10-09T04:29:01.265 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7243999999999998e-05 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:28:55.737831+0000", 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1198999999999995e-05 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:29:01.266 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:29:01.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:01.268 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:01.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:01.268 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:01.273 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:01.320 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:01.365 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:01.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:01.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:01 smithi181 ceph-mon[196979]: pgmap v1385: 119 pgs: 119 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-10-09T04:29:01.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2349394683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:29:01.417 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:01 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:29:01.424 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:01.435 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:01.477 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:01 smithi163 ceph-mon[213841]: pgmap v1385: 119 pgs: 119 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s 2023-10-09T04:29:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2349394683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:29:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:01 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:29:01.680 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:01.731 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:01.735 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:02.006 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:02.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:02.111 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:02.159 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:29:02.187 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:02 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:29:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:02 smithi181 ceph-mon[196979]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T04:29:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:02 smithi181 ceph-mon[196979]: failed to encode map e528 with expected crc 2023-10-09T04:29:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:02 smithi181 ceph-mon[196979]: failed to encode map e528 with expected crc 2023-10-09T04:29:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1151775182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:02.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:29:02.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[213841]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T04:29:02.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[213841]: failed to encode map e528 with expected crc 2023-10-09T04:29:02.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[213841]: failed to encode map e528 with expected crc 2023-10-09T04:29:02.428 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1151775182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:02.428 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:29:02.429 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[221067]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T04:29:02.429 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[221067]: failed to encode map e528 with expected crc 2023-10-09T04:29:02.429 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[221067]: failed to encode map e528 with expected crc 2023-10-09T04:29:02.429 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1151775182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:02.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:29:02.889 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:02.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:02.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:03.051 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:03.123 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:03.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:03.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:03.231 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:03.271 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:03.323 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:03.330 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:03.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:03 smithi163 ceph-mon[221067]: pgmap v1387: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 48 MiB/s wr, 868 op/s 2023-10-09T04:29:03.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:03.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:03.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:03 smithi163 ceph-mon[213841]: pgmap v1387: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 48 MiB/s wr, 868 op/s 2023-10-09T04:29:03.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:03.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:03.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:03.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:03 smithi181 ceph-mon[196979]: pgmap v1387: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 48 MiB/s wr, 868 op/s 2023-10-09T04:29:03.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:03.339 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:03.433 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:03.440 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:03.558 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:03.852 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:03.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:03.958 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:04.000 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:04.079 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:04.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:04.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:04.220 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:04.295 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:04.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:04.401 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:04.470 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:05.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:29:05.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[213841]: pgmap v1388: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 533 op/s 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[221067]: pgmap v1388: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 533 op/s 2023-10-09T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:05.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:04 smithi181 ceph-mon[196979]: pgmap v1388: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 533 op/s 2023-10-09T04:29:05.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:05.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:05.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:05.490 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:05.571 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:05.613 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:05.699 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:05.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:05.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:05.817 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:05.852 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:05.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:05.894 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:05.895 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.912 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:05.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:05.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:05.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:05.984 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:05.987 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:05.987 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:05.987 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:06.017 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:06.091 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:06.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:06.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:06.408 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:06.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:06.496 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:06.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:06.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:06.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:06.782 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:06.809 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:06.810 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:07.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:06 smithi163 ceph-mon[221067]: pgmap v1389: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 795 op/s 2023-10-09T04:29:07.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:06 smithi163 ceph-mon[213841]: pgmap v1389: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 795 op/s 2023-10-09T04:29:07.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:06 smithi181 ceph-mon[196979]: pgmap v1389: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 795 op/s 2023-10-09T04:29:07.573 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:07.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:07.682 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:07.750 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:07.957 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:29:08.077 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:08.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:08.185 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:08.217 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1255511440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:08.217 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1255511440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:08.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1255511440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:08.225 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:08.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:08.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:29:08.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:08.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:08.398 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:08.494 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:08.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:08.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:08.859 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:08.894 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:08.982 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:08.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:08.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:09.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:09.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:29:09.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1255511440' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:09.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T04:29:09.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.034 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[213841]: pgmap v1391: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 894 op/s 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1255511440' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.035 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.036 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.036 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:08 smithi163 ceph-mon[221067]: pgmap v1391: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 894 op/s 2023-10-09T04:29:09.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:09.167 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:09.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1255511440' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: failed to encode map e529 with expected crc 2023-10-09T04:29:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:08 smithi181 ceph-mon[196979]: pgmap v1391: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 894 op/s 2023-10-09T04:29:09.860 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:09.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:09.974 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:09.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:10.082 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:10.191 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1959102663' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:29:10.207 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:29:10.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:10.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:10.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:10.392 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:10.462 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1959102663' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:29:10.475 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:10.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1959102663' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:29:10.757 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:10.757 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:10.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:10.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:10.764 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:29:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:29:10] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-10-09T04:29:10.796 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:10.832 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:10.862 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:10.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:10.883 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:10.891 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:10.904 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:10.922 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:10.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.924 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.925 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.925 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.925 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:29:10.944 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:10.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:11.005 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:29:11.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:11.087 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:11.149 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:11.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:11.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:11.256 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:11.297 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: pgmap v1392: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 3.3 MiB/s wr, 579 op/s 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1959102663' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1549053846' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: pgmap v1392: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 3.3 MiB/s wr, 579 op/s 2023-10-09T04:29:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1959102663' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:29:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T04:29:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1549053846' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:11.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:11.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:11.463 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:29:11.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:11.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: pgmap v1392: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 3.3 MiB/s wr, 579 op/s 2023-10-09T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1959102663' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: failed to encode map e530 with expected crc 2023-10-09T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1549053846' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:11.523 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:11.601 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:11.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:11.708 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:11.750 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:11.798 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:11.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:11.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:12.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:12.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:12.118 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:12.195 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:12.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:12.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:12.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:12.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:12.637 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:12.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:12.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:13.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:13.096 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:13.158 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:13.176 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:13.221 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:13.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:13 smithi163 ceph-mon[213841]: pgmap v1394: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 80 KiB/s wr, 1.01k op/s 2023-10-09T04:29:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:13.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:13.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:13.378 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:13.443 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:13.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:13.491 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:13 smithi163 ceph-mon[221067]: pgmap v1394: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 80 KiB/s wr, 1.01k op/s 2023-10-09T04:29:13.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:13 smithi181 ceph-mon[196979]: pgmap v1394: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 80 KiB/s wr, 1.01k op/s 2023-10-09T04:29:13.553 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:13.638 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:13.970 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:13.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:14.073 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:14.110 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:14.145 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:29:14.146 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:29:14.146 INFO:teuthology.orchestra.run.smithi163.stdout: "complaint_time": 30, 2023-10-09T04:29:14.146 INFO:teuthology.orchestra.run.smithi163.stdout: "num_blocked_ops": 0 2023-10-09T04:29:14.146 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:29:14.188 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:14.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:14.293 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:14 smithi163 ceph-mon[213841]: pgmap v1395: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T04:29:14.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:14 smithi163 ceph-mon[221067]: pgmap v1395: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T04:29:14.366 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:14.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:29:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:14 smithi181 ceph-mon[196979]: pgmap v1395: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T04:29:15.913 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:15.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:15.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:16.003 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:29:16.022 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:16.171 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:16.289 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:16.331 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:16.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:16.438 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:16.487 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:16.541 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:16.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:16.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:16.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:16.722 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:16.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:16.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:16.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:17.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:17.058 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.138 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:17.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:17.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:17.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:17.279 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:17 smithi163 ceph-mon[221067]: pgmap v1396: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T04:29:17.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:17 smithi163 ceph-mon[213841]: pgmap v1396: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T04:29:17.338 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:17.346 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:17 smithi181 ceph-mon[196979]: pgmap v1396: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-09T04:29:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:17.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:17.449 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.549 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:17.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:17.746 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:17.772 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.814 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:17.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:17.917 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:17.984 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:18.154 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:29:18.178 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:29:18.314 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:18.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:18.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4206099319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:29:18.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:18 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/810290159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:18.398 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:18.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4206099319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:29:18.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:18 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/810290159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:18.399 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:18.423 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:18.486 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4206099319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:29:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:18 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/810290159' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:18.939 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:29:18.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:29:19.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: pgmap v1397: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4206099319' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T04:29:19.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[213841]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: pgmap v1397: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4206099319' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T04:29:19.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:19 smithi163 ceph-mon[221067]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: pgmap v1397: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4206099319' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T04:29:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:19 smithi181 ceph-mon[196979]: failed to encode map e531 with expected crc 2023-10-09T04:29:19.776 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:19.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:19.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:19.851 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:19.884 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:19.933 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:19.990 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:20.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:20.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:20.101 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:20.116 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:20.153 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:20.219 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:20.224 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:20.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:20.276 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:20.306 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:20.326 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:20.344 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:20.385 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:20.411 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:20.411 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:20.422 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:20.431 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:20.486 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:20.548 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:20.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:20.556 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:20.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:20.594 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.594 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.595 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:20.655 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:20.695 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:20.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:29:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:29:20] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-10-09T04:29:20.768 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:20.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:20.878 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:20.934 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:20.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:21.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:21.103 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:21.136 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:21.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:21 smithi181 ceph-mon[196979]: pgmap v1399: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-10-09T04:29:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3388908389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:21 smithi163 ceph-mon[221067]: pgmap v1399: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-10-09T04:29:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3388908389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:21 smithi163 ceph-mon[213841]: pgmap v1399: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-10-09T04:29:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3388908389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:21.880 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:29:21.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:21.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:22.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.096 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:22.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:22.377 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.409 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.451 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:22.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:22.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:22.589 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.656 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:22.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:22.805 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:22.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:23.092 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:23.289 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:23 smithi163 ceph-mon[221067]: pgmap v1400: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T04:29:23.294 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:23 smithi163 ceph-mon[213841]: pgmap v1400: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T04:29:23.337 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:23.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:23.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:23.493 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:23 smithi181 ceph-mon[196979]: pgmap v1400: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T04:29:23.493 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:23.517 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:23.562 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:23.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:23.610 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:23.610 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:23.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:23.707 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:23.787 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:23.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:23.897 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:24.023 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:29:24.030 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:24.262 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:29:24.566 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:24.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:24.671 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:24.721 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:24.800 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:24.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:24.907 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:24.945 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:24.987 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:24.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:29:25.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:25.129 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:25.172 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:25.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:25.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:25 smithi181 ceph-mon[196979]: from='client.56536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:25.347 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:25 smithi181 ceph-mon[196979]: pgmap v1401: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T04:29:25.348 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:25.350 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:25 smithi163 ceph-mon[213841]: from='client.56536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:25.351 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:25 smithi163 ceph-mon[213841]: pgmap v1401: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T04:29:25.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:25 smithi163 ceph-mon[221067]: from='client.56536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:25.353 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:25 smithi163 ceph-mon[221067]: pgmap v1401: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-09T04:29:25.385 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:25.814 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1400,"stamp":"2023-10-09T04:29:22.062207+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5762652088,"num_objects":87947,"num_object_clones":0,"num_object_copies":263841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87947,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121692,"num_write_kb":6705694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":1847710,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112377,"ondisk_log_size":112377,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23867568,"kb_used_data":17086192,"kb_used_omap":126,"kb_used_meta":6780993,"kb_avail":726093648,"statfs":{"total":767960285184,"available":743519895552,"internally_reserved":0,"allocated":17496260608,"data_stored":17480957396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6943737467},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,5,9,9,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-519831552,"num_objects":-7932,"num_object_clones":0,"num_object_copies":-23796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7932,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002817"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":703,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208518+0000","last_change":"2023-10-09T04:26:24.679564+0000","last_active":"2023-10-09T04:29:10.208518+0000","last_peered":"2023-10-09T04:29:10.208518+0000","last_clean":"2023-10-09T04:29:10.208518+0000","last_became_active":"2023-10-09T04:26:24.679142+0000","last_became_peered":"2023-10-09T04:26:24.679142+0000","last_unstale":"2023-10-09T04:29:10.208518+0000","last_undegraded":"2023-10-09T04:29:10.208518+0000","last_fullsized":"2023-10-09T04:29:10.208518+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:37.095828+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":116,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193058+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:29:10.193058+0000","last_peered":"2023-10-09T04:29:10.193058+0000","last_clean":"2023-10-09T04:29:10.193058+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:29:10.193058+0000","last_undegraded":"2023-10-09T04:29:10.193058+0000","last_fullsized":"2023-10-09T04:29:10.193058+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":93,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193570+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:29:10.193570+0000","last_peered":"2023-10-09T04:29:10.193570+0000","last_clean":"2023-10-09T04:29:10.193570+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:29:10.193570+0000","last_undegraded":"2023-10-09T04:29:10.193570+0000","last_fullsized":"2023-10-09T04:29:10.193570+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":728,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208819+0000","last_change":"2023-10-09T04:24:58.734627+0000","last_active":"2023-10-09T04:29:10.208819+0000","last_peered":"2023-10-09T04:29:10.208819+0000","last_clean":"2023-10-09T04:29:10.208819+0000","last_became_active":"2023-10-09T04:24:58.734248+0000","last_became_peered":"2023-10-09T04:24:58.734248+0000","last_unstale":"2023-10-09T04:29:10.208819+0000","last_undegraded":"2023-10-09T04:29:10.208819+0000","last_fullsized":"2023-10-09T04:29:10.208819+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:59.934563+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":539,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.215527+0000","last_change":"2023-10-09T04:24:56.158228+0000","last_active":"2023-10-09T04:29:10.215527+0000","last_peered":"2023-10-09T04:29:10.215527+0000","last_clean":"2023-10-09T04:29:10.215527+0000","last_became_active":"2023-10-09T04:24:56.157946+0000","last_became_peered":"2023-10-09T04:24:56.157946+0000","last_unstale":"2023-10-09T04:29:10.215527+0000","last_undegraded":"2023-10-09T04:29:10.215527+0000","last_fullsized":"2023-10-09T04:29:10.215527+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:37.279836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:11.881524+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:29:11.881524+0000","last_peered":"2023-10-09T04:29:11.881524+0000","last_clean":"2023-10-09T04:29:11.881524+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:29:11.881524+0000","last_undegraded":"2023-10-09T04:29:11.881524+0000","last_fullsized":"2023-10-09T04:29:11.881524+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":523,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192896+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:29:10.192896+0000","last_peered":"2023-10-09T04:29:10.192896+0000","last_clean":"2023-10-09T04:29:10.192896+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:29:10.192896+0000","last_undegraded":"2023-10-09T04:29:10.192896+0000","last_fullsized":"2023-10-09T04:29:10.192896+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":136,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204065+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:29:10.204065+0000","last_peered":"2023-10-09T04:29:10.204065+0000","last_clean":"2023-10-09T04:29:10.204065+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:29:10.204065+0000","last_undegraded":"2023-10-09T04:29:10.204065+0000","last_fullsized":"2023-10-09T04:29:10.204065+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":494,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210188+0000","last_change":"2023-10-09T04:26:09.747631+0000","last_active":"2023-10-09T04:29:10.210188+0000","last_peered":"2023-10-09T04:29:10.210188+0000","last_clean":"2023-10-09T04:29:10.210188+0000","last_became_active":"2023-10-09T04:26:09.747326+0000","last_became_peered":"2023-10-09T04:26:09.747326+0000","last_unstale":"2023-10-09T04:29:10.210188+0000","last_undegraded":"2023-10-09T04:29:10.210188+0000","last_fullsized":"2023-10-09T04:29:10.210188+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.315346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":930,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202031+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:29:10.202031+0000","last_peered":"2023-10-09T04:29:10.202031+0000","last_clean":"2023-10-09T04:29:10.202031+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:29:10.202031+0000","last_undegraded":"2023-10-09T04:29:10.202031+0000","last_fullsized":"2023-10-09T04:29:10.202031+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204887+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:29:10.204887+0000","last_peered":"2023-10-09T04:29:10.204887+0000","last_clean":"2023-10-09T04:29:10.204887+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:29:10.204887+0000","last_undegraded":"2023-10-09T04:29:10.204887+0000","last_fullsized":"2023-10-09T04:29:10.204887+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":271,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.213152+0000","last_change":"2023-10-09T04:27:51.852635+0000","last_active":"2023-10-09T04:29:10.213152+0000","last_peered":"2023-10-09T04:29:10.213152+0000","last_clean":"2023-10-09T04:29:10.213152+0000","last_became_active":"2023-10-09T04:27:51.852385+0000","last_became_peered":"2023-10-09T04:27:51.852385+0000","last_unstale":"2023-10-09T04:29:10.213152+0000","last_undegraded":"2023-10-09T04:29:10.213152+0000","last_fullsized":"2023-10-09T04:29:10.213152+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:54.313846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":837,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202233+0000","last_change":"2023-10-09T04:27:45.967966+0000","last_active":"2023-10-09T04:29:10.202233+0000","last_peered":"2023-10-09T04:29:10.202233+0000","last_clean":"2023-10-09T04:29:10.202233+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:29:10.202233+0000","last_undegraded":"2023-10-09T04:29:10.202233+0000","last_fullsized":"2023-10-09T04:29:10.202233+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:45.967906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:45.967906+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:06.528314+0000","scrub_duration":0.0012842089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":490,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208192+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:29:10.208192+0000","last_peered":"2023-10-09T04:29:10.208192+0000","last_clean":"2023-10-09T04:29:10.208192+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:29:10.208192+0000","last_undegraded":"2023-10-09T04:29:10.208192+0000","last_fullsized":"2023-10-09T04:29:10.208192+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:15.794143+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:29:15.794143+0000","last_peered":"2023-10-09T04:29:15.794143+0000","last_clean":"2023-10-09T04:29:15.794143+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:29:15.794143+0000","last_undegraded":"2023-10-09T04:29:15.794143+0000","last_fullsized":"2023-10-09T04:29:15.794143+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":735,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.209990+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:29:10.209990+0000","last_peered":"2023-10-09T04:29:10.209990+0000","last_clean":"2023-10-09T04:29:10.209990+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:29:10.209990+0000","last_undegraded":"2023-10-09T04:29:10.209990+0000","last_fullsized":"2023-10-09T04:29:10.209990+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":605,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202499+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:29:10.202499+0000","last_peered":"2023-10-09T04:29:10.202499+0000","last_clean":"2023-10-09T04:29:10.202499+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:29:10.202499+0000","last_undegraded":"2023-10-09T04:29:10.202499+0000","last_fullsized":"2023-10-09T04:29:10.202499+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210090+0000","last_change":"2023-10-09T04:27:51.852273+0000","last_active":"2023-10-09T04:29:10.210090+0000","last_peered":"2023-10-09T04:29:10.210090+0000","last_clean":"2023-10-09T04:29:10.210090+0000","last_became_active":"2023-10-09T04:27:51.851992+0000","last_became_peered":"2023-10-09T04:27:51.851992+0000","last_unstale":"2023-10-09T04:29:10.210090+0000","last_undegraded":"2023-10-09T04:29:10.210090+0000","last_fullsized":"2023-10-09T04:29:10.210090+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:49.300670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":710,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.218239+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:29:10.218239+0000","last_peered":"2023-10-09T04:29:10.218239+0000","last_clean":"2023-10-09T04:29:10.218239+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:29:10.218239+0000","last_undegraded":"2023-10-09T04:29:10.218239+0000","last_fullsized":"2023-10-09T04:29:10.218239+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":106,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193103+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:29:10.193103+0000","last_peered":"2023-10-09T04:29:10.193103+0000","last_clean":"2023-10-09T04:29:10.193103+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:29:10.193103+0000","last_undegraded":"2023-10-09T04:29:10.193103+0000","last_fullsized":"2023-10-09T04:29:10.193103+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193245+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:29:10.193245+0000","last_peered":"2023-10-09T04:29:10.193245+0000","last_clean":"2023-10-09T04:29:10.193245+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:29:10.193245+0000","last_undegraded":"2023-10-09T04:29:10.193245+0000","last_fullsized":"2023-10-09T04:29:10.193245+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":563,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204392+0000","last_change":"2023-10-09T04:27:02.906226+0000","last_active":"2023-10-09T04:29:10.204392+0000","last_peered":"2023-10-09T04:29:10.204392+0000","last_clean":"2023-10-09T04:29:10.204392+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:29:10.204392+0000","last_undegraded":"2023-10-09T04:29:10.204392+0000","last_fullsized":"2023-10-09T04:29:10.204392+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:21.516499+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":574,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202033+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:29:10.202033+0000","last_peered":"2023-10-09T04:29:10.202033+0000","last_clean":"2023-10-09T04:29:10.202033+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:29:10.202033+0000","last_undegraded":"2023-10-09T04:29:10.202033+0000","last_fullsized":"2023-10-09T04:29:10.202033+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":306,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202135+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:29:10.202135+0000","last_peered":"2023-10-09T04:29:10.202135+0000","last_clean":"2023-10-09T04:29:10.202135+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:29:10.202135+0000","last_undegraded":"2023-10-09T04:29:10.202135+0000","last_fullsized":"2023-10-09T04:29:10.202135+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":695,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208598+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:29:10.208598+0000","last_peered":"2023-10-09T04:29:10.208598+0000","last_clean":"2023-10-09T04:29:10.208598+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:29:10.208598+0000","last_undegraded":"2023-10-09T04:29:10.208598+0000","last_fullsized":"2023-10-09T04:29:10.208598+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":136,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204788+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:29:10.204788+0000","last_peered":"2023-10-09T04:29:10.204788+0000","last_clean":"2023-10-09T04:29:10.204788+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:29:10.204788+0000","last_undegraded":"2023-10-09T04:29:10.204788+0000","last_fullsized":"2023-10-09T04:29:10.204788+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":509,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208797+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:29:10.208797+0000","last_peered":"2023-10-09T04:29:10.208797+0000","last_clean":"2023-10-09T04:29:10.208797+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:29:10.208797+0000","last_undegraded":"2023-10-09T04:29:10.208797+0000","last_fullsized":"2023-10-09T04:29:10.208797+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":677,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.215208+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:29:10.215208+0000","last_peered":"2023-10-09T04:29:10.215208+0000","last_clean":"2023-10-09T04:29:10.215208+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:29:10.215208+0000","last_undegraded":"2023-10-09T04:29:10.215208+0000","last_fullsized":"2023-10-09T04:29:10.215208+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192814+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:29:10.192814+0000","last_peered":"2023-10-09T04:29:10.192814+0000","last_clean":"2023-10-09T04:29:10.192814+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:29:10.192814+0000","last_undegraded":"2023-10-09T04:29:10.192814+0000","last_fullsized":"2023-10-09T04:29:10.192814+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":136,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.205317+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:29:10.205317+0000","last_peered":"2023-10-09T04:29:10.205317+0000","last_clean":"2023-10-09T04:29:10.205317+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:29:10.205317+0000","last_undegraded":"2023-10-09T04:29:10.205317+0000","last_fullsized":"2023-10-09T04:29:10.205317+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":695,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210382+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:29:10.210382+0000","last_peered":"2023-10-09T04:29:10.210382+0000","last_clean":"2023-10-09T04:29:10.210382+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:29:10.210382+0000","last_undegraded":"2023-10-09T04:29:10.210382+0000","last_fullsized":"2023-10-09T04:29:10.210382+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":146,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204182+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:29:10.204182+0000","last_peered":"2023-10-09T04:29:10.204182+0000","last_clean":"2023-10-09T04:29:10.204182+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:29:10.204182+0000","last_undegraded":"2023-10-09T04:29:10.204182+0000","last_fullsized":"2023-10-09T04:29:10.204182+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":196,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.213403+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:29:10.213403+0000","last_peered":"2023-10-09T04:29:10.213403+0000","last_clean":"2023-10-09T04:29:10.213403+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:29:10.213403+0000","last_undegraded":"2023-10-09T04:29:10.213403+0000","last_fullsized":"2023-10-09T04:29:10.213403+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"531'7573","reported_seq":7603,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.150500+0000","last_change":"2023-10-09T04:27:20.720040+0000","last_active":"2023-10-09T04:29:20.150500+0000","last_peered":"2023-10-09T04:29:20.150500+0000","last_clean":"2023-10-09T04:29:20.150500+0000","last_became_active":"2023-10-09T04:27:20.719715+0000","last_became_peered":"2023-10-09T04:27:20.719715+0000","last_unstale":"2023-10-09T04:29:20.150500+0000","last_undegraded":"2023-10-09T04:29:20.150500+0000","last_fullsized":"2023-10-09T04:29:20.150500+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7573,"log_dups_size":0,"ondisk_log_size":7573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:37.979204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361693184,"num_objects":5519,"num_object_clones":0,"num_object_copies":16557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7573,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":973,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202247+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:29:10.202247+0000","last_peered":"2023-10-09T04:29:10.202247+0000","last_clean":"2023-10-09T04:29:10.202247+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:29:10.202247+0000","last_undegraded":"2023-10-09T04:29:10.202247+0000","last_fullsized":"2023-10-09T04:29:10.202247+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":575,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202202+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:29:10.202202+0000","last_peered":"2023-10-09T04:29:10.202202+0000","last_clean":"2023-10-09T04:29:10.202202+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:29:10.202202+0000","last_undegraded":"2023-10-09T04:29:10.202202+0000","last_fullsized":"2023-10-09T04:29:10.202202+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":492,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208420+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:29:10.208420+0000","last_peered":"2023-10-09T04:29:10.208420+0000","last_clean":"2023-10-09T04:29:10.208420+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:29:10.208420+0000","last_undegraded":"2023-10-09T04:29:10.208420+0000","last_fullsized":"2023-10-09T04:29:10.208420+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"531'7715","reported_seq":7748,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.875763+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:29:21.875763+0000","last_peered":"2023-10-09T04:29:21.875763+0000","last_clean":"2023-10-09T04:29:21.875763+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:29:21.875763+0000","last_undegraded":"2023-10-09T04:29:21.875763+0000","last_fullsized":"2023-10-09T04:29:21.875763+0000","mapping_epoch":507,"log_start":"511'300","ondisk_log_start":"511'300","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7415,"log_dups_size":0,"ondisk_log_size":7415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362086400,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7715,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":764,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.214750+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:29:10.214750+0000","last_peered":"2023-10-09T04:29:10.214750+0000","last_clean":"2023-10-09T04:29:10.214750+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:29:10.214750+0000","last_undegraded":"2023-10-09T04:29:10.214750+0000","last_fullsized":"2023-10-09T04:29:10.214750+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":551,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202379+0000","last_change":"2023-10-09T04:24:56.212579+0000","last_active":"2023-10-09T04:29:10.202379+0000","last_peered":"2023-10-09T04:29:10.202379+0000","last_clean":"2023-10-09T04:29:10.202379+0000","last_became_active":"2023-10-09T04:24:56.184403+0000","last_became_peered":"2023-10-09T04:24:56.184403+0000","last_unstale":"2023-10-09T04:29:10.202379+0000","last_undegraded":"2023-10-09T04:29:10.202379+0000","last_fullsized":"2023-10-09T04:29:10.202379+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:10.486526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192909+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:29:10.192909+0000","last_peered":"2023-10-09T04:29:10.192909+0000","last_clean":"2023-10-09T04:29:10.192909+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:29:10.192909+0000","last_undegraded":"2023-10-09T04:29:10.192909+0000","last_fullsized":"2023-10-09T04:29:10.192909+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"531'7563","reported_seq":7595,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.147663+0000","last_change":"2023-10-09T04:27:20.720676+0000","last_active":"2023-10-09T04:29:20.147663+0000","last_peered":"2023-10-09T04:29:20.147663+0000","last_clean":"2023-10-09T04:29:20.147663+0000","last_became_active":"2023-10-09T04:27:20.720342+0000","last_became_peered":"2023-10-09T04:27:20.720342+0000","last_unstale":"2023-10-09T04:29:20.147663+0000","last_undegraded":"2023-10-09T04:29:20.147663+0000","last_fullsized":"2023-10-09T04:29:20.147663+0000","mapping_epoch":507,"log_start":"512'1400","ondisk_log_start":"512'1400","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:37.485070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362741760,"num_objects":5535,"num_object_clones":0,"num_object_copies":16605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7563,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":735,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.209899+0000","last_change":"2023-10-09T04:24:58.648512+0000","last_active":"2023-10-09T04:29:10.209899+0000","last_peered":"2023-10-09T04:29:10.209899+0000","last_clean":"2023-10-09T04:29:10.209899+0000","last_became_active":"2023-10-09T04:24:58.648143+0000","last_became_peered":"2023-10-09T04:24:58.648143+0000","last_unstale":"2023-10-09T04:29:10.209899+0000","last_undegraded":"2023-10-09T04:29:10.209899+0000","last_fullsized":"2023-10-09T04:29:10.209899+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:26.096956+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":502,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202375+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:29:10.202375+0000","last_peered":"2023-10-09T04:29:10.202375+0000","last_clean":"2023-10-09T04:29:10.202375+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:29:10.202375+0000","last_undegraded":"2023-10-09T04:29:10.202375+0000","last_fullsized":"2023-10-09T04:29:10.202375+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":157,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201095+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:29:10.201095+0000","last_peered":"2023-10-09T04:29:10.201095+0000","last_clean":"2023-10-09T04:29:10.201095+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:29:10.201095+0000","last_undegraded":"2023-10-09T04:29:10.201095+0000","last_fullsized":"2023-10-09T04:29:10.201095+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"531'7452","reported_seq":7484,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.201995+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:29:20.201995+0000","last_peered":"2023-10-09T04:29:20.201995+0000","last_clean":"2023-10-09T04:29:20.201995+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:29:20.201995+0000","last_undegraded":"2023-10-09T04:29:20.201995+0000","last_fullsized":"2023-10-09T04:29:20.201995+0000","mapping_epoch":507,"log_start":"512'1300","ondisk_log_start":"512'1300","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6152,"log_dups_size":0,"ondisk_log_size":6152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353894400,"num_objects":5400,"num_object_clones":0,"num_object_copies":16200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7452,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":787,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193720+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:29:10.193720+0000","last_peered":"2023-10-09T04:29:10.193720+0000","last_clean":"2023-10-09T04:29:10.193720+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:29:10.193720+0000","last_undegraded":"2023-10-09T04:29:10.193720+0000","last_fullsized":"2023-10-09T04:29:10.193720+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":530,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.211673+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:29:10.211673+0000","last_peered":"2023-10-09T04:29:10.211673+0000","last_clean":"2023-10-09T04:29:10.211673+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:29:10.211673+0000","last_undegraded":"2023-10-09T04:29:10.211673+0000","last_fullsized":"2023-10-09T04:29:10.211673+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":154,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204834+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:29:10.204834+0000","last_peered":"2023-10-09T04:29:10.204834+0000","last_clean":"2023-10-09T04:29:10.204834+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:29:10.204834+0000","last_undegraded":"2023-10-09T04:29:10.204834+0000","last_fullsized":"2023-10-09T04:29:10.204834+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"531'7654","reported_seq":7686,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.187230+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:29:20.187230+0000","last_peered":"2023-10-09T04:29:20.187230+0000","last_clean":"2023-10-09T04:29:20.187230+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:29:20.187230+0000","last_undegraded":"2023-10-09T04:29:20.187230+0000","last_fullsized":"2023-10-09T04:29:20.187230+0000","mapping_epoch":507,"log_start":"512'1500","ondisk_log_start":"512'1500","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6154,"log_dups_size":0,"ondisk_log_size":6154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365297664,"num_objects":5574,"num_object_clones":0,"num_object_copies":16722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7654,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":994,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201797+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:29:10.201797+0000","last_peered":"2023-10-09T04:29:10.201797+0000","last_clean":"2023-10-09T04:29:10.201797+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:29:10.201797+0000","last_undegraded":"2023-10-09T04:29:10.201797+0000","last_fullsized":"2023-10-09T04:29:10.201797+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":166,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204597+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:29:10.204597+0000","last_peered":"2023-10-09T04:29:10.204597+0000","last_clean":"2023-10-09T04:29:10.204597+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:29:10.204597+0000","last_undegraded":"2023-10-09T04:29:10.204597+0000","last_fullsized":"2023-10-09T04:29:10.204597+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193083+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:29:10.193083+0000","last_peered":"2023-10-09T04:29:10.193083+0000","last_clean":"2023-10-09T04:29:10.193083+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:29:10.193083+0000","last_undegraded":"2023-10-09T04:29:10.193083+0000","last_fullsized":"2023-10-09T04:29:10.193083+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"531'7703","reported_seq":7736,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.863768+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:29:21.863768+0000","last_peered":"2023-10-09T04:29:21.863768+0000","last_clean":"2023-10-09T04:29:21.863768+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:29:21.863768+0000","last_undegraded":"2023-10-09T04:29:21.863768+0000","last_fullsized":"2023-10-09T04:29:21.863768+0000","mapping_epoch":507,"log_start":"511'300","ondisk_log_start":"511'300","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351862784,"num_objects":5369,"num_object_clones":0,"num_object_copies":16107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7703,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":740,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208677+0000","last_change":"2023-10-09T04:26:23.679008+0000","last_active":"2023-10-09T04:29:10.208677+0000","last_peered":"2023-10-09T04:29:10.208677+0000","last_clean":"2023-10-09T04:29:10.208677+0000","last_became_active":"2023-10-09T04:26:23.678629+0000","last_became_peered":"2023-10-09T04:26:23.678629+0000","last_unstale":"2023-10-09T04:29:10.208677+0000","last_undegraded":"2023-10-09T04:29:10.208677+0000","last_fullsized":"2023-10-09T04:29:10.208677+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:41.534992+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192925+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:29:10.192925+0000","last_peered":"2023-10-09T04:29:10.192925+0000","last_clean":"2023-10-09T04:29:10.192925+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:29:10.192925+0000","last_undegraded":"2023-10-09T04:29:10.192925+0000","last_fullsized":"2023-10-09T04:29:10.192925+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":113,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193495+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:29:10.193495+0000","last_peered":"2023-10-09T04:29:10.193495+0000","last_clean":"2023-10-09T04:29:10.193495+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:29:10.193495+0000","last_undegraded":"2023-10-09T04:29:10.193495+0000","last_fullsized":"2023-10-09T04:29:10.193495+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"531'7636","reported_seq":7669,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.732942+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:29:20.732942+0000","last_peered":"2023-10-09T04:29:20.732942+0000","last_clean":"2023-10-09T04:29:20.732942+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:29:20.732942+0000","last_undegraded":"2023-10-09T04:29:20.732942+0000","last_fullsized":"2023-10-09T04:29:20.732942+0000","mapping_epoch":507,"log_start":"512'1700","ondisk_log_start":"512'1700","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358481920,"num_objects":5470,"num_object_clones":0,"num_object_copies":16410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7636,"num_write_kb":419392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":637,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204923+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:29:10.204923+0000","last_peered":"2023-10-09T04:29:10.204923+0000","last_clean":"2023-10-09T04:29:10.204923+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:29:10.204923+0000","last_undegraded":"2023-10-09T04:29:10.204923+0000","last_fullsized":"2023-10-09T04:29:10.204923+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":116,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192924+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:29:10.192924+0000","last_peered":"2023-10-09T04:29:10.192924+0000","last_clean":"2023-10-09T04:29:10.192924+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:29:10.192924+0000","last_undegraded":"2023-10-09T04:29:10.192924+0000","last_fullsized":"2023-10-09T04:29:10.192924+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193790+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:29:10.193790+0000","last_peered":"2023-10-09T04:29:10.193790+0000","last_clean":"2023-10-09T04:29:10.193790+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:29:10.193790+0000","last_undegraded":"2023-10-09T04:29:10.193790+0000","last_fullsized":"2023-10-09T04:29:10.193790+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"531'7605","reported_seq":7638,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:19.094708+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:29:19.094708+0000","last_peered":"2023-10-09T04:29:19.094708+0000","last_clean":"2023-10-09T04:29:19.094708+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:29:19.094708+0000","last_undegraded":"2023-10-09T04:29:19.094708+0000","last_fullsized":"2023-10-09T04:29:19.094708+0000","mapping_epoch":507,"log_start":"512'1600","ondisk_log_start":"512'1600","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6005,"log_dups_size":0,"ondisk_log_size":6005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371916800,"num_objects":5675,"num_object_clones":0,"num_object_copies":17025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7605,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":735,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210190+0000","last_change":"2023-10-09T04:26:23.680981+0000","last_active":"2023-10-09T04:29:10.210190+0000","last_peered":"2023-10-09T04:29:10.210190+0000","last_clean":"2023-10-09T04:29:10.210190+0000","last_became_active":"2023-10-09T04:26:23.680615+0000","last_became_peered":"2023-10-09T04:26:23.680615+0000","last_unstale":"2023-10-09T04:29:10.210190+0000","last_undegraded":"2023-10-09T04:29:10.210190+0000","last_fullsized":"2023-10-09T04:29:10.210190+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:36.804345+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":197,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.212913+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:29:10.212913+0000","last_peered":"2023-10-09T04:29:10.212913+0000","last_clean":"2023-10-09T04:29:10.212913+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:29:10.212913+0000","last_undegraded":"2023-10-09T04:29:10.212913+0000","last_fullsized":"2023-10-09T04:29:10.212913+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193379+0000","last_change":"2023-10-09T04:26:09.746900+0000","last_active":"2023-10-09T04:29:10.193379+0000","last_peered":"2023-10-09T04:29:10.193379+0000","last_clean":"2023-10-09T04:29:10.193379+0000","last_became_active":"2023-10-09T04:26:09.746607+0000","last_became_peered":"2023-10-09T04:26:09.746607+0000","last_unstale":"2023-10-09T04:29:10.193379+0000","last_undegraded":"2023-10-09T04:29:10.193379+0000","last_fullsized":"2023-10-09T04:29:10.193379+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:07.695043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"531'7569","reported_seq":7602,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.156015+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:29:20.156015+0000","last_peered":"2023-10-09T04:29:20.156015+0000","last_clean":"2023-10-09T04:29:20.156015+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:29:20.156015+0000","last_undegraded":"2023-10-09T04:29:20.156015+0000","last_fullsized":"2023-10-09T04:29:20.156015+0000","mapping_epoch":507,"log_start":"512'1300","ondisk_log_start":"512'1300","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6269,"log_dups_size":0,"ondisk_log_size":6269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362151960,"num_objects":5527,"num_object_clones":0,"num_object_copies":16581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5527,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7569,"num_write_kb":419009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":657,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201387+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:29:10.201387+0000","last_peered":"2023-10-09T04:29:10.201387+0000","last_clean":"2023-10-09T04:29:10.201387+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:29:10.201387+0000","last_undegraded":"2023-10-09T04:29:10.201387+0000","last_fullsized":"2023-10-09T04:29:10.201387+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":135,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157403+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:29:18.157403+0000","last_peered":"2023-10-09T04:29:18.157403+0000","last_clean":"2023-10-09T04:29:18.157403+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:29:18.157403+0000","last_undegraded":"2023-10-09T04:29:18.157403+0000","last_fullsized":"2023-10-09T04:29:18.157403+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":547,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208676+0000","last_change":"2023-10-09T04:26:21.351382+0000","last_active":"2023-10-09T04:29:10.208676+0000","last_peered":"2023-10-09T04:29:10.208676+0000","last_clean":"2023-10-09T04:29:10.208676+0000","last_became_active":"2023-10-09T04:26:21.348023+0000","last_became_peered":"2023-10-09T04:26:21.348023+0000","last_unstale":"2023-10-09T04:29:10.208676+0000","last_undegraded":"2023-10-09T04:29:10.208676+0000","last_fullsized":"2023-10-09T04:29:10.208676+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:45.177721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":410,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210352+0000","last_change":"2023-10-09T04:26:09.749151+0000","last_active":"2023-10-09T04:29:10.210352+0000","last_peered":"2023-10-09T04:29:10.210352+0000","last_clean":"2023-10-09T04:29:10.210352+0000","last_became_active":"2023-10-09T04:26:09.748857+0000","last_became_peered":"2023-10-09T04:26:09.748857+0000","last_unstale":"2023-10-09T04:29:10.210352+0000","last_undegraded":"2023-10-09T04:29:10.210352+0000","last_fullsized":"2023-10-09T04:29:10.210352+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:38.228161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"531'7851","reported_seq":7884,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.855429+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:29:21.855429+0000","last_peered":"2023-10-09T04:29:21.855429+0000","last_clean":"2023-10-09T04:29:21.855429+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:29:21.855429+0000","last_undegraded":"2023-10-09T04:29:21.855429+0000","last_fullsized":"2023-10-09T04:29:21.855429+0000","mapping_epoch":507,"log_start":"511'500","ondisk_log_start":"511'500","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365101056,"num_objects":5571,"num_object_clones":0,"num_object_copies":16713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7851,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":738,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208247+0000","last_change":"2023-10-09T04:26:23.684553+0000","last_active":"2023-10-09T04:29:10.208247+0000","last_peered":"2023-10-09T04:29:10.208247+0000","last_clean":"2023-10-09T04:29:10.208247+0000","last_became_active":"2023-10-09T04:26:23.684125+0000","last_became_peered":"2023-10-09T04:26:23.684125+0000","last_unstale":"2023-10-09T04:29:10.208247+0000","last_undegraded":"2023-10-09T04:29:10.208247+0000","last_fullsized":"2023-10-09T04:29:10.208247+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:01.715214+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":514,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210223+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:29:10.210223+0000","last_peered":"2023-10-09T04:29:10.210223+0000","last_clean":"2023-10-09T04:29:10.210223+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:29:10.210223+0000","last_undegraded":"2023-10-09T04:29:10.210223+0000","last_fullsized":"2023-10-09T04:29:10.210223+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193323+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:29:10.193323+0000","last_peered":"2023-10-09T04:29:10.193323+0000","last_clean":"2023-10-09T04:29:10.193323+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:29:10.193323+0000","last_undegraded":"2023-10-09T04:29:10.193323+0000","last_fullsized":"2023-10-09T04:29:10.193323+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":26,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208333+0000","last_change":"2023-10-09T04:27:51.852322+0000","last_active":"2023-10-09T04:29:10.208333+0000","last_peered":"2023-10-09T04:29:10.208333+0000","last_clean":"2023-10-09T04:29:10.208333+0000","last_became_active":"2023-10-09T04:27:51.851990+0000","last_became_peered":"2023-10-09T04:27:51.851990+0000","last_unstale":"2023-10-09T04:29:10.208333+0000","last_undegraded":"2023-10-09T04:29:10.208333+0000","last_fullsized":"2023-10-09T04:29:10.208333+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:19.671405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"531'7415","reported_seq":7447,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.184726+0000","last_change":"2023-10-09T04:27:20.717622+0000","last_active":"2023-10-09T04:29:20.184726+0000","last_peered":"2023-10-09T04:29:20.184726+0000","last_clean":"2023-10-09T04:29:20.184726+0000","last_became_active":"2023-10-09T04:27:20.717291+0000","last_became_peered":"2023-10-09T04:27:20.717291+0000","last_unstale":"2023-10-09T04:29:20.184726+0000","last_undegraded":"2023-10-09T04:29:20.184726+0000","last_fullsized":"2023-10-09T04:29:20.184726+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7415,"log_dups_size":0,"ondisk_log_size":7415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:34.329924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356450304,"num_objects":5439,"num_object_clones":0,"num_object_copies":16317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7415,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":705,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204700+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:29:10.204700+0000","last_peered":"2023-10-09T04:29:10.204700+0000","last_clean":"2023-10-09T04:29:10.204700+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:29:10.204700+0000","last_undegraded":"2023-10-09T04:29:10.204700+0000","last_fullsized":"2023-10-09T04:29:10.204700+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":382,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204293+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:29:10.204293+0000","last_peered":"2023-10-09T04:29:10.204293+0000","last_clean":"2023-10-09T04:29:10.204293+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:29:10.204293+0000","last_undegraded":"2023-10-09T04:29:10.204293+0000","last_fullsized":"2023-10-09T04:29:10.204293+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":471,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.216869+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:29:10.216869+0000","last_peered":"2023-10-09T04:29:10.216869+0000","last_clean":"2023-10-09T04:29:10.216869+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:29:10.216869+0000","last_undegraded":"2023-10-09T04:29:10.216869+0000","last_fullsized":"2023-10-09T04:29:10.216869+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":236,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210140+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:29:10.210140+0000","last_peered":"2023-10-09T04:29:10.210140+0000","last_clean":"2023-10-09T04:29:10.210140+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:29:10.210140+0000","last_undegraded":"2023-10-09T04:29:10.210140+0000","last_fullsized":"2023-10-09T04:29:10.210140+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"531'7477","reported_seq":7509,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.138591+0000","last_change":"2023-10-09T04:27:20.717505+0000","last_active":"2023-10-09T04:29:20.138591+0000","last_peered":"2023-10-09T04:29:20.138591+0000","last_clean":"2023-10-09T04:29:20.138591+0000","last_became_active":"2023-10-09T04:27:20.717202+0000","last_became_peered":"2023-10-09T04:27:20.717202+0000","last_unstale":"2023-10-09T04:29:20.138591+0000","last_undegraded":"2023-10-09T04:29:20.138591+0000","last_fullsized":"2023-10-09T04:29:20.138591+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7477,"log_dups_size":0,"ondisk_log_size":7477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:51.034299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359596032,"num_objects":5487,"num_object_clones":0,"num_object_copies":16461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7477,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":534,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208401+0000","last_change":"2023-10-09T04:26:21.429887+0000","last_active":"2023-10-09T04:29:10.208401+0000","last_peered":"2023-10-09T04:29:10.208401+0000","last_clean":"2023-10-09T04:29:10.208401+0000","last_became_active":"2023-10-09T04:26:21.392380+0000","last_became_peered":"2023-10-09T04:26:21.392380+0000","last_unstale":"2023-10-09T04:29:10.208401+0000","last_undegraded":"2023-10-09T04:29:10.208401+0000","last_fullsized":"2023-10-09T04:29:10.208401+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:05:55.650685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":985,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201804+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:29:10.201804+0000","last_peered":"2023-10-09T04:29:10.201804+0000","last_clean":"2023-10-09T04:29:10.201804+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:29:10.201804+0000","last_undegraded":"2023-10-09T04:29:10.201804+0000","last_fullsized":"2023-10-09T04:29:10.201804+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":467,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.210302+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:29:10.210302+0000","last_peered":"2023-10-09T04:29:10.210302+0000","last_clean":"2023-10-09T04:29:10.210302+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:29:10.210302+0000","last_undegraded":"2023-10-09T04:29:10.210302+0000","last_fullsized":"2023-10-09T04:29:10.210302+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193414+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:29:10.193414+0000","last_peered":"2023-10-09T04:29:10.193414+0000","last_clean":"2023-10-09T04:29:10.193414+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:29:10.193414+0000","last_undegraded":"2023-10-09T04:29:10.193414+0000","last_fullsized":"2023-10-09T04:29:10.193414+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"531'7630","reported_seq":7661,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.886053+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:29:21.886053+0000","last_peered":"2023-10-09T04:29:21.886053+0000","last_clean":"2023-10-09T04:29:21.886053+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:29:21.886053+0000","last_undegraded":"2023-10-09T04:29:21.886053+0000","last_fullsized":"2023-10-09T04:29:21.886053+0000","mapping_epoch":507,"log_start":"511'300","ondisk_log_start":"511'300","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354156544,"num_objects":5404,"num_object_clones":0,"num_object_copies":16212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7630,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":626,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193344+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:29:10.193344+0000","last_peered":"2023-10-09T04:29:10.193344+0000","last_clean":"2023-10-09T04:29:10.193344+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:29:10.193344+0000","last_undegraded":"2023-10-09T04:29:10.193344+0000","last_fullsized":"2023-10-09T04:29:10.193344+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":531,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201797+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:29:10.201797+0000","last_peered":"2023-10-09T04:29:10.201797+0000","last_clean":"2023-10-09T04:29:10.201797+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:29:10.201797+0000","last_undegraded":"2023-10-09T04:29:10.201797+0000","last_fullsized":"2023-10-09T04:29:10.201797+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157345+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:29:18.157345+0000","last_peered":"2023-10-09T04:29:18.157345+0000","last_clean":"2023-10-09T04:29:18.157345+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:29:18.157345+0000","last_undegraded":"2023-10-09T04:29:18.157345+0000","last_fullsized":"2023-10-09T04:29:18.157345+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":154,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.205253+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:29:10.205253+0000","last_peered":"2023-10-09T04:29:10.205253+0000","last_clean":"2023-10-09T04:29:10.205253+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:29:10.205253+0000","last_undegraded":"2023-10-09T04:29:10.205253+0000","last_fullsized":"2023-10-09T04:29:10.205253+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"531'7664","reported_seq":7696,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.171707+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:29:20.171707+0000","last_peered":"2023-10-09T04:29:20.171707+0000","last_clean":"2023-10-09T04:29:20.171707+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:29:20.171707+0000","last_undegraded":"2023-10-09T04:29:20.171707+0000","last_fullsized":"2023-10-09T04:29:20.171707+0000","mapping_epoch":507,"log_start":"512'1400","ondisk_log_start":"512'1400","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360841216,"num_objects":5506,"num_object_clones":0,"num_object_copies":16518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7664,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":649,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201226+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:29:10.201226+0000","last_peered":"2023-10-09T04:29:10.201226+0000","last_clean":"2023-10-09T04:29:10.201226+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:29:10.201226+0000","last_undegraded":"2023-10-09T04:29:10.201226+0000","last_fullsized":"2023-10-09T04:29:10.201226+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":554,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202397+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:29:10.202397+0000","last_peered":"2023-10-09T04:29:10.202397+0000","last_clean":"2023-10-09T04:29:10.202397+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:29:10.202397+0000","last_undegraded":"2023-10-09T04:29:10.202397+0000","last_fullsized":"2023-10-09T04:29:10.202397+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":289,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208407+0000","last_change":"2023-10-09T04:26:21.355315+0000","last_active":"2023-10-09T04:29:10.208407+0000","last_peered":"2023-10-09T04:29:10.208407+0000","last_clean":"2023-10-09T04:29:10.208407+0000","last_became_active":"2023-10-09T04:26:21.355038+0000","last_became_peered":"2023-10-09T04:26:21.355038+0000","last_unstale":"2023-10-09T04:29:10.208407+0000","last_undegraded":"2023-10-09T04:29:10.208407+0000","last_fullsized":"2023-10-09T04:29:10.208407+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:45.819056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":250,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208329+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:29:10.208329+0000","last_peered":"2023-10-09T04:29:10.208329+0000","last_clean":"2023-10-09T04:29:10.208329+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:29:10.208329+0000","last_undegraded":"2023-10-09T04:29:10.208329+0000","last_fullsized":"2023-10-09T04:29:10.208329+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"531'7650","reported_seq":7682,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.113407+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:29:20.113407+0000","last_peered":"2023-10-09T04:29:20.113407+0000","last_clean":"2023-10-09T04:29:20.113407+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:29:20.113407+0000","last_undegraded":"2023-10-09T04:29:20.113407+0000","last_fullsized":"2023-10-09T04:29:20.113407+0000","mapping_epoch":507,"log_start":"512'1500","ondisk_log_start":"512'1500","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6150,"log_dups_size":0,"ondisk_log_size":6150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358875136,"num_objects":5476,"num_object_clones":0,"num_object_copies":16428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7650,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":588,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193926+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:29:10.193926+0000","last_peered":"2023-10-09T04:29:10.193926+0000","last_clean":"2023-10-09T04:29:10.193926+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:29:10.193926+0000","last_undegraded":"2023-10-09T04:29:10.193926+0000","last_fullsized":"2023-10-09T04:29:10.193926+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":491,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202503+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:29:10.202503+0000","last_peered":"2023-10-09T04:29:10.202503+0000","last_clean":"2023-10-09T04:29:10.202503+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:29:10.202503+0000","last_undegraded":"2023-10-09T04:29:10.202503+0000","last_fullsized":"2023-10-09T04:29:10.202503+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193854+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:29:10.193854+0000","last_peered":"2023-10-09T04:29:10.193854+0000","last_clean":"2023-10-09T04:29:10.193854+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:29:10.193854+0000","last_undegraded":"2023-10-09T04:29:10.193854+0000","last_fullsized":"2023-10-09T04:29:10.193854+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":439,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202104+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:29:10.202104+0000","last_peered":"2023-10-09T04:29:10.202104+0000","last_clean":"2023-10-09T04:29:10.202104+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:29:10.202104+0000","last_undegraded":"2023-10-09T04:29:10.202104+0000","last_fullsized":"2023-10-09T04:29:10.202104+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"531'7474","reported_seq":7506,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:20.169122+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:29:20.169122+0000","last_peered":"2023-10-09T04:29:20.169122+0000","last_clean":"2023-10-09T04:29:20.169122+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:29:20.169122+0000","last_undegraded":"2023-10-09T04:29:20.169122+0000","last_fullsized":"2023-10-09T04:29:20.169122+0000","mapping_epoch":507,"log_start":"512'1200","ondisk_log_start":"512'1200","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6274,"log_dups_size":0,"ondisk_log_size":6274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356909056,"num_objects":5446,"num_object_clones":0,"num_object_copies":16338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7474,"num_write_kb":413440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":707,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.214669+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:29:10.214669+0000","last_peered":"2023-10-09T04:29:10.214669+0000","last_clean":"2023-10-09T04:29:10.214669+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:29:10.214669+0000","last_undegraded":"2023-10-09T04:29:10.214669+0000","last_fullsized":"2023-10-09T04:29:10.214669+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":330,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201418+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:29:10.201418+0000","last_peered":"2023-10-09T04:29:10.201418+0000","last_clean":"2023-10-09T04:29:10.201418+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:29:10.201418+0000","last_undegraded":"2023-10-09T04:29:10.201418+0000","last_fullsized":"2023-10-09T04:29:10.201418+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":166,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.205467+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:29:10.205467+0000","last_peered":"2023-10-09T04:29:10.205467+0000","last_clean":"2023-10-09T04:29:10.205467+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:29:10.205467+0000","last_undegraded":"2023-10-09T04:29:10.205467+0000","last_fullsized":"2023-10-09T04:29:10.205467+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192826+0000","last_change":"2023-10-09T04:24:56.169203+0000","last_active":"2023-10-09T04:29:10.192826+0000","last_peered":"2023-10-09T04:29:10.192826+0000","last_clean":"2023-10-09T04:29:10.192826+0000","last_became_active":"2023-10-09T04:24:56.168737+0000","last_became_peered":"2023-10-09T04:24:56.168737+0000","last_unstale":"2023-10-09T04:29:10.192826+0000","last_undegraded":"2023-10-09T04:29:10.192826+0000","last_fullsized":"2023-10-09T04:29:10.192826+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:21.515886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":574,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.204365+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:29:10.204365+0000","last_peered":"2023-10-09T04:29:10.204365+0000","last_clean":"2023-10-09T04:29:10.204365+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:29:10.204365+0000","last_undegraded":"2023-10-09T04:29:10.204365+0000","last_fullsized":"2023-10-09T04:29:10.204365+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":154,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.205059+0000","last_change":"2023-10-09T04:24:56.170052+0000","last_active":"2023-10-09T04:29:10.205059+0000","last_peered":"2023-10-09T04:29:10.205059+0000","last_clean":"2023-10-09T04:29:10.205059+0000","last_became_active":"2023-10-09T04:24:56.169607+0000","last_became_peered":"2023-10-09T04:24:56.169607+0000","last_unstale":"2023-10-09T04:29:10.205059+0000","last_undegraded":"2023-10-09T04:29:10.205059+0000","last_fullsized":"2023-10-09T04:29:10.205059+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.849923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":71,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201598+0000","last_change":"2023-10-09T04:26:21.345230+0000","last_active":"2023-10-09T04:29:10.201598+0000","last_peered":"2023-10-09T04:29:10.201598+0000","last_clean":"2023-10-09T04:29:10.201598+0000","last_became_active":"2023-10-09T04:26:21.344939+0000","last_became_peered":"2023-10-09T04:26:21.344939+0000","last_unstale":"2023-10-09T04:29:10.201598+0000","last_undegraded":"2023-10-09T04:29:10.201598+0000","last_fullsized":"2023-10-09T04:29:10.201598+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.495703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":605,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.201308+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:29:10.201308+0000","last_peered":"2023-10-09T04:29:10.201308+0000","last_clean":"2023-10-09T04:29:10.201308+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:29:10.201308+0000","last_undegraded":"2023-10-09T04:29:10.201308+0000","last_fullsized":"2023-10-09T04:29:10.201308+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":548,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208510+0000","last_change":"2023-10-09T04:24:56.181978+0000","last_active":"2023-10-09T04:29:10.208510+0000","last_peered":"2023-10-09T04:29:10.208510+0000","last_clean":"2023-10-09T04:29:10.208510+0000","last_became_active":"2023-10-09T04:24:56.164297+0000","last_became_peered":"2023-10-09T04:24:56.164297+0000","last_unstale":"2023-10-09T04:29:10.208510+0000","last_undegraded":"2023-10-09T04:29:10.208510+0000","last_fullsized":"2023-10-09T04:29:10.208510+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:15.703672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193175+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:29:10.193175+0000","last_peered":"2023-10-09T04:29:10.193175+0000","last_clean":"2023-10-09T04:29:10.193175+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:29:10.193175+0000","last_undegraded":"2023-10-09T04:29:10.193175+0000","last_fullsized":"2023-10-09T04:29:10.193175+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":559,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.208600+0000","last_change":"2023-10-09T04:24:56.202659+0000","last_active":"2023-10-09T04:29:10.208600+0000","last_peered":"2023-10-09T04:29:10.208600+0000","last_clean":"2023-10-09T04:29:10.208600+0000","last_became_active":"2023-10-09T04:24:56.166522+0000","last_became_peered":"2023-10-09T04:24:56.166522+0000","last_unstale":"2023-10-09T04:29:10.208600+0000","last_undegraded":"2023-10-09T04:29:10.208600+0000","last_fullsized":"2023-10-09T04:29:10.208600+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:15.858204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":33,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192800+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:29:10.192800+0000","last_peered":"2023-10-09T04:29:10.192800+0000","last_clean":"2023-10-09T04:29:10.192800+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:29:10.192800+0000","last_undegraded":"2023-10-09T04:29:10.192800+0000","last_fullsized":"2023-10-09T04:29:10.192800+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":784,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.202054+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:29:10.202054+0000","last_peered":"2023-10-09T04:29:10.202054+0000","last_clean":"2023-10-09T04:29:10.202054+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:29:10.202054+0000","last_undegraded":"2023-10-09T04:29:10.202054+0000","last_fullsized":"2023-10-09T04:29:10.202054+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":265,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.213531+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:29:10.213531+0000","last_peered":"2023-10-09T04:29:10.213531+0000","last_clean":"2023-10-09T04:29:10.213531+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:29:10.213531+0000","last_undegraded":"2023-10-09T04:29:10.213531+0000","last_fullsized":"2023-10-09T04:29:10.213531+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":382,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:16.888182+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:29:16.888182+0000","last_peered":"2023-10-09T04:29:16.888182+0000","last_clean":"2023-10-09T04:29:16.888182+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:29:16.888182+0000","last_undegraded":"2023-10-09T04:29:16.888182+0000","last_fullsized":"2023-10-09T04:29:16.888182+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":642,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.193330+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:29:10.193330+0000","last_peered":"2023-10-09T04:29:10.193330+0000","last_clean":"2023-10-09T04:29:10.193330+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:29:10.193330+0000","last_undegraded":"2023-10-09T04:29:10.193330+0000","last_fullsized":"2023-10-09T04:29:10.193330+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":159,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157974+0000","last_change":"2023-10-09T04:26:21.347607+0000","last_active":"2023-10-09T04:29:18.157974+0000","last_peered":"2023-10-09T04:29:18.157974+0000","last_clean":"2023-10-09T04:29:18.157974+0000","last_became_active":"2023-10-09T04:26:21.347136+0000","last_became_peered":"2023-10-09T04:26:21.347136+0000","last_unstale":"2023-10-09T04:29:18.157974+0000","last_undegraded":"2023-10-09T04:29:18.157974+0000","last_fullsized":"2023-10-09T04:29:18.157974+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:52.721389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-09T04:29:10.192714+0000","last_change":"2023-10-09T04:24:56.183973+0000","last_active":"2023-10-09T04:29:10.192714+0000","last_peered":"2023-10-09T04:29:10.192714+0000","last_clean":"2023-10-09T04:29:10.192714+0000","last_became_active":"2023-10-09T04:24:56.181453+0000","last_became_peered":"2023-10-09T04:24:56.181453+0000","last_unstale":"2023-10-09T04:29:10.192714+0000","last_undegraded":"2023-10-09T04:29:10.192714+0000","last_fullsized":"2023-10-09T04:29:10.192714+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:35.829983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5762056216,"num_objects":87923,"num_object_clones":0,"num_object_copies":263769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87923,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121631,"num_write_kb":6705665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17347194880,"data_stored":17347182664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107331,"ondisk_log_size":107331,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":11614,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1836096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201722,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3049088,"kb_used_data":2136888,"kb_used_omap":0,"kb_used_meta":912192,"kb_avail":90696064,"statfs":{"total":95995035648,"available":92872769536,"internally_reserved":0,"allocated":2188173312,"data_stored":2186670245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":934084608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":6.1619999999999999,"5min":2.6819999999999999,"15min":2.758},"min":{"1min":0.313,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":59.780000000000001,"5min":70.614000000000004,"15min":148.65899999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":5.5549999999999997,"5min":2.8180000000000001,"15min":2.7469999999999999},"min":{"1min":0.247,"5min":0.23000000000000001,"15min":0.221},"max":{"1min":59.709000000000003,"5min":70.721999999999994,"15min":148.642},"last":0.255}]},{"osd":1,"last update":"Mon Oct 9 04:28:21 2023","interfaces":[{"interface":"back","average":{"1min":5.2939999999999996,"5min":2.5510000000000002,"15min":5.391},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.22600000000000001},"max":{"1min":65.361999999999995,"5min":74.933999999999997,"15min":261.935},"last":0.46300000000000002},{"interface":"front","average":{"1min":4.5460000000000003,"5min":2.4950000000000001,"15min":5.2149999999999999},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":62.767000000000003,"5min":69.355000000000004,"15min":262.07600000000002},"last":2.6469999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":2.024,"15min":2.4790000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":7.734,"5min":66.328999999999994,"15min":253.41200000000001},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.9350000000000001,"15min":2.8050000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.246},"max":{"1min":5.9100000000000001,"5min":66.314999999999998,"15min":253.39400000000001},"last":0.81200000000000006}]},{"osd":3,"last update":"Mon Oct 9 04:28:21 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.8919999999999999,"15min":2.9540000000000002},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":56.354999999999997,"5min":56.354999999999997,"15min":255.97300000000001},"last":2.661},{"interface":"front","average":{"1min":4.601,"5min":1.994,"15min":3.1890000000000001},"min":{"1min":0.373,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":56.334000000000003,"5min":56.334000000000003,"15min":256.06099999999998},"last":2.6760000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":0.91500000000000004,"15min":0.86699999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.3480000000000008,"5min":9.3480000000000008,"15min":9.3480000000000008},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.2490000000000001,"5min":0.94899999999999995,"15min":1.1970000000000001},"min":{"1min":0.28699999999999998,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":6.1299999999999999,"5min":6.1299999999999999,"15min":12.566000000000001},"last":0.29699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.367,"15min":1.7909999999999999},"min":{"1min":0.35099999999999998,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":6.1440000000000001,"5min":17.882999999999999,"15min":120.066},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.764,"5min":1.2430000000000001,"15min":1.579},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":7.8360000000000003,"5min":17.568999999999999,"15min":120.09699999999999},"last":0.31900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.25,"15min":1.335},"min":{"1min":0.31,"5min":0.31,"15min":0.307},"max":{"1min":9.5069999999999997,"5min":9.5069999999999997,"15min":9.5069999999999997},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.601,"5min":1.1040000000000001,"15min":1.036},"min":{"1min":0.34699999999999998,"5min":0.33200000000000002,"15min":0.30299999999999999},"max":{"1min":7.7850000000000001,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.38500000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463358,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3425520,"kb_used_data":2839848,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":90319632,"statfs":{"total":95995035648,"available":92487303168,"internally_reserved":0,"allocated":2908004352,"data_stored":2906516680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:53 2023","interfaces":[{"interface":"back","average":{"1min":19.07,"5min":5.2350000000000003,"15min":4.4900000000000002},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.193},"max":{"1min":162.524,"5min":162.524,"15min":162.524},"last":0.46400000000000002},{"interface":"front","average":{"1min":18.472999999999999,"5min":5.1420000000000003,"15min":4.4640000000000004},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":161.73599999999999,"5min":161.73599999999999,"15min":161.73599999999999},"last":11.856999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6549999999999998,"5min":2.2759999999999998,"15min":4.9960000000000004},"min":{"1min":0.33200000000000002,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":18.971,"5min":64.738,"15min":315.73599999999999},"last":8.407},{"interface":"front","average":{"1min":2.4729999999999999,"5min":2.3559999999999999,"15min":5.085},"min":{"1min":0.30499999999999999,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":18.943999999999999,"5min":64.789000000000001,"15min":316.91500000000002},"last":8.5549999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:28:53 2023","interfaces":[{"interface":"back","average":{"1min":8.1120000000000001,"5min":4.6950000000000003,"15min":3.0609999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.23899999999999999},"max":{"1min":116.084,"5min":116.084,"15min":139.06899999999999},"last":5.0529999999999999},{"interface":"front","average":{"1min":8.1560000000000006,"5min":4.7460000000000004,"15min":3.1040000000000001},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":116.11199999999999,"5min":116.11199999999999,"15min":139.173},"last":5.0869999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":1.3939999999999999,"15min":1.853},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.25600000000000001},"max":{"1min":10.496,"5min":14.083,"15min":43.475999999999999},"last":1.819},{"interface":"front","average":{"1min":1.367,"5min":1.4019999999999999,"15min":1.784},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":9.9399999999999995,"5min":14.052,"15min":43.369},"last":0.55000000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":0.91300000000000003,"15min":1.496},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.24399999999999999},"max":{"1min":17.545000000000002,"5min":17.545000000000002,"15min":161.27500000000001},"last":0.495},{"interface":"front","average":{"1min":1.3149999999999999,"5min":0.81699999999999995,"15min":1.556},"min":{"1min":0.28599999999999998,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":15.212999999999999,"5min":15.212999999999999,"15min":160.32599999999999},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.5720000000000001,"15min":1.264},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.92600000000000005,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.4690000000000001,"15min":1.2609999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.27900000000000003},"max":{"1min":0.876,"5min":74.073999999999998,"15min":74.073999999999998},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.93999999999999995,"15min":1.6639999999999999},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.6030000000000002,"5min":5.9050000000000002,"15min":161.39400000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.84899999999999998,"15min":1.653},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":5.6520000000000001,"5min":5.6520000000000001,"15min":161.41200000000001},"last":0.88800000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089667,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3739172,"kb_used_data":2903644,"kb_used_omap":9,"kb_used_meta":835510,"kb_avail":90005980,"statfs":{"total":95995035648,"available":92166123520,"internally_reserved":0,"allocated":2973331456,"data_stored":2971047080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":855563241},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":2.3759999999999999,"15min":4.3419999999999996},"min":{"1min":0.28699999999999998,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":5.6260000000000003,"5min":86.963999999999999,"15min":259.26799999999997},"last":8.1690000000000005},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.629,"15min":5.25},"min":{"1min":0.32500000000000001,"5min":0.222,"15min":0.19400000000000001},"max":{"1min":10.948,"5min":83.213999999999999,"15min":258.46899999999999},"last":4.6369999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.929,"15min":2.96},"min":{"1min":0.247,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.0600000000000005,"5min":74.474000000000004,"15min":259.315},"last":3.0579999999999998},{"interface":"front","average":{"1min":2.085,"5min":2.1179999999999999,"15min":3.3679999999999999},"min":{"1min":0.28000000000000003,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.2249999999999996,"5min":75.653000000000006,"15min":259.733},"last":2.984}]},{"osd":3,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.8959999999999999,"15min":2.3090000000000002},"min":{"1min":0.318,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":12.615,"5min":71.745999999999995,"15min":71.745999999999995},"last":3.7080000000000002},{"interface":"front","average":{"1min":2.645,"5min":2.0049999999999999,"15min":2.7330000000000001},"min":{"1min":0.29199999999999998,"5min":0.27300000000000002,"15min":0.24299999999999999},"max":{"1min":12.565,"5min":75.613,"15min":75.613},"last":3.6899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.617,"15min":2.9169999999999998},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":10.975,"5min":39.052,"15min":257.83300000000003},"last":3.0070000000000001},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.3069999999999999,"15min":2.6080000000000001},"min":{"1min":0.41699999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":6.1230000000000002,"5min":39.040999999999997,"15min":257.863},"last":2.9100000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.167,"15min":2.0649999999999999},"min":{"1min":0.38600000000000001,"5min":0.29599999999999999,"15min":0.29099999999999998},"max":{"1min":10.903,"5min":10.903,"15min":38.848999999999997},"last":2.9529999999999998},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.26,"15min":2.1080000000000001},"min":{"1min":0.54300000000000004,"5min":0.32400000000000001,"15min":0.29799999999999999},"max":{"1min":13.316000000000001,"5min":13.316000000000001,"15min":39.593000000000004},"last":2.9649999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.647,"15min":1.8580000000000001},"min":{"1min":0.52200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":13.353,"5min":35.045999999999999,"15min":35.045999999999999},"last":2.996},{"interface":"front","average":{"1min":1.6180000000000001,"5min":1.54,"15min":1.6359999999999999},"min":{"1min":0.40799999999999997,"5min":0.36699999999999999,"15min":0.29999999999999999},"max":{"1min":4.1699999999999999,"5min":34.960999999999999,"15min":34.960999999999999},"last":3.5870000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":1.742,"15min":3.7400000000000002},"min":{"1min":0.499,"5min":0.35899999999999999,"15min":0.34799999999999998},"max":{"1min":7.5149999999999997,"5min":36.765999999999998,"15min":259.08499999999998},"last":2.9409999999999998},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.716,"15min":3.4820000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":4.2800000000000002,"5min":36.75,"15min":259.13299999999998},"last":3.0840000000000001}]}]},{"osd":0,"up_from":84,"seq":360777253193,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2591288,"kb_used_data":1769712,"kb_used_omap":28,"kb_used_meta":821539,"kb_avail":91153864,"statfs":{"total":95995035648,"available":93341556736,"internally_reserved":0,"allocated":1812185088,"data_stored":1809761588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":841256784},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":1.0369999999999999,"15min":4.2590000000000003},"min":{"1min":0.189,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":209.215},"last":2.2679999999999998},{"interface":"front","average":{"1min":1.593,"5min":1.3160000000000001,"15min":4.2110000000000003},"min":{"1min":0.28399999999999997,"5min":0.22800000000000001,"15min":0.20899999999999999},"max":{"1min":9.4049999999999994,"5min":16.131,"15min":120.934},"last":1.109}]},{"osd":2,"last update":"Mon Oct 9 04:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.4970000000000001,"15min":2.508},"min":{"1min":0.218,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":16.096,"5min":17.562999999999999,"15min":88.088999999999999},"last":0.247},{"interface":"front","average":{"1min":2.5960000000000001,"5min":1.526,"15min":2.8319999999999999},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.9659999999999993,"5min":9.9659999999999993,"15min":88.004999999999995},"last":2.5190000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":0.98999999999999999,"15min":2.7679999999999998},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":5.1390000000000002,"5min":10.853999999999999,"15min":99.709000000000003},"last":2.2320000000000002},{"interface":"front","average":{"1min":2.1040000000000001,"5min":1.425,"15min":2.9129999999999998},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.8689999999999998,"5min":12.292,"15min":99.560000000000002},"last":0.29499999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":2,"15min":2.504},"min":{"1min":0.38900000000000001,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":11.153,"5min":86.906999999999996,"15min":88.058000000000007},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.9089999999999998,"5min":2.1240000000000001,"15min":2.4540000000000002},"min":{"1min":0.49099999999999999,"5min":0.28299999999999997,"15min":0.24199999999999999},"max":{"1min":9.8810000000000002,"5min":86.891000000000005,"15min":86.891000000000005},"last":1.6479999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":7.7919999999999998,"5min":3.4929999999999999,"15min":3.774},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":84.983000000000004,"5min":134.93000000000001,"15min":156.96100000000001},"last":1.6870000000000001},{"interface":"front","average":{"1min":6.9589999999999996,"5min":3.246,"15min":3.2629999999999999},"min":{"1min":0.42399999999999999,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":81.691999999999993,"5min":135.04599999999999,"15min":156.97300000000001},"last":2.2949999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":2.641,"5min":1.212,"15min":1.758},"min":{"1min":0.39300000000000002,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":7.0469999999999997,"5min":7.0469999999999997,"15min":63.468000000000004},"last":2.3420000000000001},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.3899999999999999,"15min":2.1640000000000001},"min":{"1min":0.438,"5min":0.376,"15min":0.34100000000000003},"max":{"1min":11.128,"5min":12.26,"15min":88.180000000000007},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":7.3419999999999996,"5min":2.2999999999999998,"15min":2.2360000000000002},"min":{"1min":0.42499999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":78.307000000000002,"5min":78.307000000000002,"15min":78.307000000000002},"last":2.3180000000000001},{"interface":"front","average":{"1min":7.194,"5min":2.3580000000000001,"15min":2.649},"min":{"1min":0.436,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":78.216999999999999,"5min":78.216999999999999,"15min":88.120000000000005},"last":0.45000000000000001}]}]},{"osd":2,"up_from":94,"seq":403726926141,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2919560,"kb_used_data":2126656,"kb_used_omap":81,"kb_used_meta":792814,"kb_avail":90825592,"statfs":{"total":95995035648,"available":93005406208,"internally_reserved":0,"allocated":2177695744,"data_stored":2175375549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":811841754},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":8.2379999999999995,"5min":3.5049999999999999,"15min":3.8769999999999998},"min":{"1min":0.25,"5min":0.183,"15min":0.17699999999999999},"max":{"1min":132.87299999999999,"5min":132.87299999999999,"15min":132.87299999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":9.3670000000000009,"5min":3.8580000000000001,"15min":3.8330000000000002},"min":{"1min":0.249,"5min":0.216,"15min":0.20799999999999999},"max":{"1min":130.398,"5min":130.398,"15min":130.398},"last":9.7119999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":8.5839999999999996,"5min":3.5510000000000002,"15min":4.7430000000000003},"min":{"1min":0.254,"5min":0.224,"15min":0.191},"max":{"1min":122.17100000000001,"5min":122.17100000000001,"15min":158.13399999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":7.3179999999999996,"5min":3.3039999999999998,"15min":4.8360000000000003},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":121.91800000000001,"5min":121.91800000000001,"15min":161.12},"last":0.64000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7470000000000001,"15min":2.4620000000000002},"min":{"1min":0.40400000000000003,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.9560000000000004,"5min":45.023000000000003,"15min":58.997},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.2360000000000002,"5min":1.863,"15min":2.5899999999999999},"min":{"1min":0.38600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.1959999999999997,"5min":44.923999999999999,"15min":66.135000000000005},"last":1.0189999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":11.311,"5min":4.0970000000000004,"15min":2.8410000000000002},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":196.57599999999999,"5min":196.57599999999999,"15min":196.57599999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":10.571,"5min":3.036,"15min":2.3380000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":201.81100000000001,"5min":201.81100000000001,"15min":201.81100000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":10.795999999999999,"5min":3.073,"15min":2.9569999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":190.06899999999999,"5min":190.06899999999999,"15min":190.06899999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":11.347,"5min":3.9409999999999998,"15min":3.286},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":190.08500000000001,"5min":190.08500000000001,"15min":190.08500000000001},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.607,"15min":1.903},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":2.0329999999999999,"5min":58.469999999999999,"15min":58.469999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.6879999999999999,"5min":2.0619999999999998,"15min":2.1669999999999998},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.26800000000000002},"max":{"1min":5.9710000000000001,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.72299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":9.9800000000000004,"5min":2.7970000000000002,"15min":2.726},"min":{"1min":0.35099999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":191.792,"5min":191.792,"15min":191.792},"last":0.55000000000000004},{"interface":"front","average":{"1min":10.778,"5min":3.9470000000000001,"15min":3.266},"min":{"1min":0.40400000000000003,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":191.828,"5min":191.828,"15min":191.828},"last":0.80800000000000005}]}]},{"osd":3,"up_from":99,"seq":425201762615,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3515644,"kb_used_data":2472436,"kb_used_omap":7,"kb_used_meta":1043000,"kb_avail":90229508,"statfs":{"total":95995035648,"available":92395016192,"internally_reserved":0,"allocated":2531774464,"data_stored":2529468932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1068032616},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":9.1310000000000002,"5min":4.6070000000000002,"15min":4.1550000000000002},"min":{"1min":0.246,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":119.97799999999999,"5min":119.97799999999999,"15min":125.765},"last":2.3879999999999999},{"interface":"front","average":{"1min":9.8260000000000005,"5min":4.6760000000000002,"15min":4.5700000000000003},"min":{"1min":0.28000000000000003,"5min":0.223,"15min":0.185},"max":{"1min":120.307,"5min":120.307,"15min":125.46899999999999},"last":8.3149999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.371,"15min":4.524},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":6.1760000000000002,"5min":9.7889999999999997,"15min":186.21299999999999},"last":2.2690000000000001},{"interface":"front","average":{"1min":1.141,"5min":1.228,"15min":4.3449999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.221},"max":{"1min":4.5110000000000001,"5min":14.042,"15min":191.78},"last":1.4139999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.341,"15min":2.46},"min":{"1min":0.29299999999999998,"5min":0.245,"15min":0.187},"max":{"1min":5.8300000000000001,"5min":9.2720000000000002,"15min":127.599},"last":1.3819999999999999},{"interface":"front","average":{"1min":1.847,"5min":1.238,"15min":3.2839999999999998},"min":{"1min":0.25800000000000001,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":6.1349999999999998,"5min":12.032999999999999,"15min":129.98599999999999},"last":1.8200000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.8939999999999999,"15min":2.532},"min":{"1min":0.34000000000000002,"5min":0.30399999999999999,"15min":0.25700000000000001},"max":{"1min":6.625,"5min":68.251999999999995,"15min":211.84700000000001},"last":1.7709999999999999},{"interface":"front","average":{"1min":1.351,"5min":1.9670000000000001,"15min":2.7120000000000002},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.29099999999999998},"max":{"1min":5.6929999999999996,"5min":68.310000000000002,"15min":211.17500000000001},"last":2.3420000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.304,"15min":1.9810000000000001},"min":{"1min":0.36499999999999999,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":5.7220000000000004,"5min":9.6709999999999994,"15min":17.901},"last":2.4199999999999999},{"interface":"front","average":{"1min":1.252,"5min":1.2150000000000001,"15min":1.5029999999999999},"min":{"1min":0.42299999999999999,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":4.5369999999999999,"5min":7.7530000000000001,"15min":18.068000000000001},"last":0.45700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.869,"15min":1.6479999999999999},"min":{"1min":0.433,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":4.5700000000000003,"5min":64.378,"15min":64.378},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.538,"5min":1.6859999999999999,"15min":1.857},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":4.1109999999999998,"5min":64.353999999999999,"15min":64.353999999999999},"last":1.7350000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.827,"15min":2.7130000000000001},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.0459999999999994,"5min":65.132999999999996,"15min":212.62700000000001},"last":1.794},{"interface":"front","average":{"1min":1.677,"5min":1.98,"15min":2.9319999999999999},"min":{"1min":0.38800000000000001,"5min":0.376,"15min":0.33300000000000002},"max":{"1min":5.7400000000000002,"5min":65.116,"15min":212.04599999999999},"last":2.3130000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986631,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2317688,"kb_used_data":1415856,"kb_used_omap":0,"kb_used_meta":901824,"kb_avail":91427464,"statfs":{"total":95995035648,"available":93621723136,"internally_reserved":0,"allocated":1449836544,"data_stored":1448340814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923467776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":7.1310000000000002,"15min":6.7389999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":30.488,"5min":207.45699999999999,"15min":207.45699999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":3.0920000000000001,"5min":7.5090000000000003,"15min":6.9219999999999997},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":30.5,"5min":207.114,"15min":207.114},"last":1.3620000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":3.4700000000000002,"5min":2.6320000000000001,"15min":5.0350000000000001},"min":{"1min":0.251,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":33.433999999999997,"5min":90.900000000000006,"15min":135.25399999999999},"last":1.2190000000000001},{"interface":"front","average":{"1min":2.5590000000000002,"5min":2.589,"15min":4.835},"min":{"1min":0.35199999999999998,"5min":0.252,"15min":0.251},"max":{"1min":32.683,"5min":90.787000000000006,"15min":134.44300000000001},"last":1.8089999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":5.3819999999999997,"15min":3.976},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":7.6840000000000002,"5min":203.61500000000001,"15min":203.61500000000001},"last":1.53},{"interface":"front","average":{"1min":1.4450000000000001,"5min":5.3399999999999999,"15min":3.9580000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.9320000000000004,"5min":203.601,"15min":203.601},"last":1.54}]},{"osd":3,"last update":"Mon Oct 9 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":1.786,"15min":2.3570000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27600000000000002},"max":{"1min":41.060000000000002,"5min":41.060000000000002,"15min":154.13200000000001},"last":1.6950000000000001},{"interface":"front","average":{"1min":3.2120000000000002,"5min":1.7410000000000001,"15min":2.3959999999999999},"min":{"1min":0.33200000000000002,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":40.972000000000001,"5min":40.972000000000001,"15min":154.077},"last":0.49199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:29:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.02,"15min":1.7749999999999999},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":16.753,"5min":16.753,"15min":111.809},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.7130000000000001,"5min":0.98599999999999999,"15min":1.9650000000000001},"min":{"1min":0.28799999999999998,"5min":0.253,"15min":0.252},"max":{"1min":17.792000000000002,"5min":17.792000000000002,"15min":112.002},"last":0.33900000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.81299999999999994,"15min":1.641},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.6160000000000001,"5min":10.039999999999999,"15min":76.307000000000002},"last":0.313},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.74099999999999999,"15min":1.4750000000000001},"min":{"1min":0.36399999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.1779999999999999,"5min":5.1230000000000002,"15min":78.421999999999997},"last":0.40600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":0.873,"15min":1.153},"min":{"1min":0.317,"5min":0.309,"15min":0.26800000000000002},"max":{"1min":4.8609999999999998,"5min":5.0510000000000002,"15min":34.039000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.0900000000000001,"15min":1.361},"min":{"1min":0.30199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.0679999999999996,"5min":12.061,"15min":33.433999999999997},"last":0.36199999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724995,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2309608,"kb_used_data":1421152,"kb_used_omap":0,"kb_used_meta":888448,"kb_avail":91435544,"statfs":{"total":95995035648,"available":93629997056,"internally_reserved":0,"allocated":1455259648,"data_stored":1453776508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909770752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":8.1240000000000006,"5min":5.7590000000000003,"15min":4.5999999999999996},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.193},"max":{"1min":127.627,"5min":139.86799999999999,"15min":139.86799999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":7.7599999999999998,"5min":5.6779999999999999,"15min":4.7530000000000001},"min":{"1min":0.24299999999999999,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":128.066,"5min":139.84899999999999,"15min":139.84899999999999},"last":3.1259999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:29:17 2023","interfaces":[{"interface":"back","average":{"1min":13.365,"5min":3.6589999999999998,"15min":5.5309999999999997},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":131.86199999999999,"5min":131.86199999999999,"15min":296.23099999999999},"last":0.311},{"interface":"front","average":{"1min":12.981,"5min":3.7930000000000001,"15min":5.7649999999999997},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":132.74600000000001,"5min":132.74600000000001,"15min":295.065},"last":0.80700000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":7.0300000000000002,"5min":2.504,"15min":3.7160000000000002},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":115.66500000000001,"5min":115.66500000000001,"15min":168.779},"last":0.86099999999999999},{"interface":"front","average":{"1min":7.423,"5min":3.0270000000000001,"15min":3.5710000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":116.881,"5min":116.881,"15min":168.03100000000001},"last":1.4910000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:29:17 2023","interfaces":[{"interface":"back","average":{"1min":8.2810000000000006,"5min":3.278,"15min":2.5979999999999999},"min":{"1min":0.497,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":116.56,"5min":116.56,"15min":116.56},"last":3.1030000000000002},{"interface":"front","average":{"1min":8.3160000000000007,"5min":3.1970000000000001,"15min":2.625},"min":{"1min":0.44900000000000001,"5min":0.29699999999999999,"15min":0.23499999999999999},"max":{"1min":116.527,"5min":116.527,"15min":116.527},"last":3.0880000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.821,"15min":1.387},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":3.9670000000000001,"5min":95.668000000000006,"15min":95.668000000000006},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.752,"15min":1.3320000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4649999999999999,"5min":98.703999999999994,"15min":98.703999999999994},"last":0.32200000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:29:17 2023","interfaces":[{"interface":"back","average":{"1min":7.6920000000000002,"5min":2.093,"15min":1.4019999999999999},"min":{"1min":0.34499999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":145.054,"5min":145.054,"15min":145.054},"last":0.38200000000000001},{"interface":"front","average":{"1min":7.9930000000000003,"5min":2.2869999999999999,"15min":1.54},"min":{"1min":0.34599999999999997,"5min":0.32500000000000001,"15min":0.307},"max":{"1min":144.85900000000001,"5min":144.85900000000001,"15min":144.85900000000001},"last":0.77900000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":0.85199999999999998,"15min":1.079},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25900000000000001},"max":{"1min":9.1839999999999993,"5min":9.1839999999999993,"15min":12.096},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.905,"5min":1.147,"15min":1.26},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":9.1430000000000007,"5min":11.522,"15min":11.522},"last":0.83599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1788542976,"data_stored":1788542976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2940993536,"data_stored":2940993536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2168066048,"data_stored":2168061976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2520973312,"data_stored":2520973312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1441468416,"data_stored":1441464344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1440415744,"data_stored":1440415744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2880311296,"data_stored":2880307224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2166423552,"data_stored":2166423552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:29:25.825 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:29:25.921 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:25.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:25.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:29:26.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:26.078 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.136 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:26.243 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:26.308 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:26.412 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.578 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:26.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:26.752 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.946 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:27.050 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:27.078 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:27.107 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:27.133 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:27.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:27.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:27.238 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:27.271 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:27.357 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:27.368 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:27 smithi181 ceph-mon[196979]: pgmap v1402: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-09T04:29:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:27.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:27.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:27.495 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:27 smithi163 ceph-mon[221067]: pgmap v1402: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-09T04:29:27.496 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:27 smithi163 ceph-mon[213841]: pgmap v1402: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-09T04:29:27.498 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:27.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:27.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:28.064 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:28.108 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:28.151 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:28.165 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:28.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:28.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2200767850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:28.221 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:28 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:28.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2200767850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:28.222 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:28 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:28.245 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:29:28.272 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:28.308 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:28.390 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:28.398 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2200767850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:28.398 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:28 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:28.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:28.500 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:28.585 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:29.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[213841]: pgmap v1403: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-10-09T04:29:29.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:29.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[213841]: failed to encode map e532 with expected crc 2023-10-09T04:29:29.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:29.463 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[213841]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[213841]: failed to encode map e532 with expected crc 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[221067]: pgmap v1403: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[221067]: failed to encode map e532 with expected crc 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[221067]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T04:29:29.464 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:29 smithi163 ceph-mon[221067]: failed to encode map e532 with expected crc 2023-10-09T04:29:29.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:29.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:29 smithi181 ceph-mon[196979]: pgmap v1403: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-10-09T04:29:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:29 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:29 smithi181 ceph-mon[196979]: failed to encode map e532 with expected crc 2023-10-09T04:29:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:29 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:29 smithi181 ceph-mon[196979]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T04:29:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:29 smithi181 ceph-mon[196979]: failed to encode map e532 with expected crc 2023-10-09T04:29:29.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:29.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:29.624 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:29.652 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:29.735 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:29.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:29.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:29.845 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:29.909 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:29.964 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:29.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:29.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:30.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:30.118 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:30.181 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:30.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:30.287 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:30.378 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:30.489 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:30.521 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:30.557 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:30.592 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:30.629 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:30.641 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:29:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:29:30] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-10-09T04:29:30.683 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:30.690 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:30.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:30.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:30.714 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:30.741 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.761 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:29:30.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:30.824 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:30.865 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:30.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:30.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:31.011 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:31.086 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:31.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:31.193 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:31.240 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:31.306 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:31.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:31.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:31 smithi163 ceph-mon[213841]: pgmap v1405: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:29:31.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:31 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1494772078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:31.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:31 smithi163 ceph-mon[221067]: pgmap v1405: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:29:31.330 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:31 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1494772078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:31.415 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:31.450 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:31 smithi181 ceph-mon[196979]: pgmap v1405: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:29:31.450 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:31 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1494772078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:31.450 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:31.548 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:29:31.560 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:31.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:31.669 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:31.712 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:31.781 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:31.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:31.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:31.931 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:32.015 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:32.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:32.123 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:32.167 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:32.232 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:32.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:32.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:32.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:32.394 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:32.465 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:32.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:32.575 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:130133 11.0 11:09b65221:::benchmark_data_smithi163_601242_object25354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:29:32.588165+0000", 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.018201281999999999, 2023-10-09T04:29:32.618 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018255958999999999, 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 130133 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588165+0000", 2023-10-09T04:29:32.619 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588165+0000", 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588163+0000", 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:32.620 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588168+0000", 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7419999999999997e-06 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588176+0000", 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.371e-06 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.621 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588182+0000", 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.84e-06 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588295+0000", 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000112994 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588423+0000", 2023-10-09T04:29:32.622 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012751799999999999 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588525+0000", 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010204600000000001 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600288+0000", 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011763447999999999 2023-10-09T04:29:32.623 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600619+0000", 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00033076900000000002 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:130135 11.0 11:021862fa:::benchmark_data_smithi163_601242_object25356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:29:32.588195+0000", 2023-10-09T04:29:32.624 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.018170624999999999, 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.018332075999999999, 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 130135 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:29:32.625 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588195+0000", 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588195+0000", 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588195+0000", 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:32.626 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588197+0000", 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4430000000000002e-06 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588203+0000", 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8159999999999999e-06 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.627 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.588208+0000", 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6090000000000003e-06 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.591603+0000", 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0033953690000000001 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.591727+0000", 2023-10-09T04:29:32.628 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012334499999999999 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.591843+0000", 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000116628 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600320+0000", 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0084766679999999997 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.629 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600867+0000", 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000547051 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:130138 11.7 e532/507 11:e47be2ec:::benchmark_data_smithi163_601242_object25359:head v 532'8127, mlcod=532'8125)", 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:29:32.600511+0000", 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.005855458, 2023-10-09T04:29:32.630 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0061137709999999996, 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600511+0000", 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:29:32.631 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600511+0000", 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600510+0000", 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600518+0000", 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6559999999999992e-06 2023-10-09T04:29:32.632 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600528+0000", 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0205e-05 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600534+0000", 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.601e-06 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.633 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600589+0000", 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4542000000000001e-05 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.600610+0000", 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1172999999999999e-05 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.634 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:130143 11.e e532/507)", 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:29:32.606416+0000", 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 4294967295.9999504, 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000273174, 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.606416+0000", 2023-10-09T04:29:32.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.606416+0000", 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.606415+0000", 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.606423+0000", 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.3259999999999996e-06 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.606438+0000", 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5367999999999999e-05 2023-10-09T04:29:32.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:29:32.606446+0000", 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7700000000000001e-06 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 4 2023-10-09T04:29:32.638 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:29:32.639 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:32.673 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:32.855 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:32.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:32.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:33.016 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:33.061 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:33.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:33.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:33.085 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:29:33.087 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:29:33.168 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:33.224 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:33.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:33 smithi181 ceph-mon[196979]: pgmap v1406: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T04:29:33.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:33 smithi163 ceph-mon[213841]: pgmap v1406: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T04:29:33.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:33 smithi163 ceph-mon[221067]: pgmap v1406: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T04:29:33.732 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:29:33.990 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:29:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:34 smithi163 ceph-mon[221067]: from='client.56551 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:34.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:34 smithi163 ceph-mon[221067]: pgmap v1407: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T04:29:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:34 smithi163 ceph-mon[213841]: from='client.56551 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:34.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:34 smithi163 ceph-mon[213841]: pgmap v1407: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T04:29:34.461 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:34.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:34.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:34 smithi181 ceph-mon[196979]: from='client.56551 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:34.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:34 smithi181 ceph-mon[196979]: pgmap v1407: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T04:29:34.549 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1406,"stamp":"2023-10-09T04:29:32.068559+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5305472952,"num_objects":80971,"num_object_clones":0,"num_object_copies":242913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80971,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128668,"num_write_kb":6705694,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":1847710,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113353,"ondisk_log_size":113353,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22548340,"kb_used_data":15738868,"kb_used_omap":126,"kb_used_meta":6808897,"kb_avail":727412876,"statfs":{"total":767960285184,"available":744870785024,"internally_reserved":0,"allocated":16116600832,"data_stored":16101293359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6972311163},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-457179136,"num_objects":-6976,"num_object_clones":0,"num_object_copies":-20928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6976,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006351"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":704,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156375+0000","last_change":"2023-10-09T04:26:24.679564+0000","last_active":"2023-10-09T04:29:18.156375+0000","last_peered":"2023-10-09T04:29:18.156375+0000","last_clean":"2023-10-09T04:29:18.156375+0000","last_became_active":"2023-10-09T04:26:24.679142+0000","last_became_peered":"2023-10-09T04:26:24.679142+0000","last_unstale":"2023-10-09T04:29:18.156375+0000","last_undegraded":"2023-10-09T04:29:18.156375+0000","last_fullsized":"2023-10-09T04:29:18.156375+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:13.371153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:13.371153+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:37.095828+0000","scrub_duration":0.0032956309999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":117,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149229+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:29:18.149229+0000","last_peered":"2023-10-09T04:29:18.149229+0000","last_clean":"2023-10-09T04:29:18.149229+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:29:18.149229+0000","last_undegraded":"2023-10-09T04:29:18.149229+0000","last_fullsized":"2023-10-09T04:29:18.149229+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":94,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150113+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:29:18.150113+0000","last_peered":"2023-10-09T04:29:18.150113+0000","last_clean":"2023-10-09T04:29:18.150113+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:29:18.150113+0000","last_undegraded":"2023-10-09T04:29:18.150113+0000","last_fullsized":"2023-10-09T04:29:18.150113+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":729,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156744+0000","last_change":"2023-10-09T04:24:58.734627+0000","last_active":"2023-10-09T04:29:18.156744+0000","last_peered":"2023-10-09T04:29:18.156744+0000","last_clean":"2023-10-09T04:29:18.156744+0000","last_became_active":"2023-10-09T04:24:58.734248+0000","last_became_peered":"2023-10-09T04:24:58.734248+0000","last_unstale":"2023-10-09T04:29:18.156744+0000","last_undegraded":"2023-10-09T04:29:18.156744+0000","last_fullsized":"2023-10-09T04:29:18.156744+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:59.934563+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":540,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.170767+0000","last_change":"2023-10-09T04:24:56.158228+0000","last_active":"2023-10-09T04:29:18.170767+0000","last_peered":"2023-10-09T04:29:18.170767+0000","last_clean":"2023-10-09T04:29:18.170767+0000","last_became_active":"2023-10-09T04:24:56.157946+0000","last_became_peered":"2023-10-09T04:24:56.157946+0000","last_unstale":"2023-10-09T04:29:18.170767+0000","last_undegraded":"2023-10-09T04:29:18.170767+0000","last_fullsized":"2023-10-09T04:29:18.170767+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:37.279836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":138,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158373+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:29:18.158373+0000","last_peered":"2023-10-09T04:29:18.158373+0000","last_clean":"2023-10-09T04:29:18.158373+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:29:18.158373+0000","last_undegraded":"2023-10-09T04:29:18.158373+0000","last_fullsized":"2023-10-09T04:29:18.158373+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":524,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149037+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:29:18.149037+0000","last_peered":"2023-10-09T04:29:18.149037+0000","last_clean":"2023-10-09T04:29:18.149037+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:29:18.149037+0000","last_undegraded":"2023-10-09T04:29:18.149037+0000","last_fullsized":"2023-10-09T04:29:18.149037+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":137,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157003+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:29:18.157003+0000","last_peered":"2023-10-09T04:29:18.157003+0000","last_clean":"2023-10-09T04:29:18.157003+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:29:18.157003+0000","last_undegraded":"2023-10-09T04:29:18.157003+0000","last_fullsized":"2023-10-09T04:29:18.157003+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":495,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158856+0000","last_change":"2023-10-09T04:26:09.747631+0000","last_active":"2023-10-09T04:29:18.158856+0000","last_peered":"2023-10-09T04:29:18.158856+0000","last_clean":"2023-10-09T04:29:18.158856+0000","last_became_active":"2023-10-09T04:26:09.747326+0000","last_became_peered":"2023-10-09T04:26:09.747326+0000","last_unstale":"2023-10-09T04:29:18.158856+0000","last_undegraded":"2023-10-09T04:29:18.158856+0000","last_fullsized":"2023-10-09T04:29:18.158856+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.315346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":931,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155645+0000","last_change":"2023-10-09T04:18:25.379528+0000","last_active":"2023-10-09T04:29:18.155645+0000","last_peered":"2023-10-09T04:29:18.155645+0000","last_clean":"2023-10-09T04:29:18.155645+0000","last_became_active":"2023-10-09T04:18:04.693145+0000","last_became_peered":"2023-10-09T04:18:04.693145+0000","last_unstale":"2023-10-09T04:29:18.155645+0000","last_undegraded":"2023-10-09T04:29:18.155645+0000","last_fullsized":"2023-10-09T04:29:18.155645+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:05.356630+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157860+0000","last_change":"2023-10-09T04:23:06.349877+0000","last_active":"2023-10-09T04:29:18.157860+0000","last_peered":"2023-10-09T04:29:18.157860+0000","last_clean":"2023-10-09T04:29:18.157860+0000","last_became_active":"2023-10-09T04:23:06.349557+0000","last_became_peered":"2023-10-09T04:23:06.349557+0000","last_unstale":"2023-10-09T04:29:18.157860+0000","last_undegraded":"2023-10-09T04:29:18.157860+0000","last_fullsized":"2023-10-09T04:29:18.157860+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:22.922285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":272,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.166502+0000","last_change":"2023-10-09T04:27:51.852635+0000","last_active":"2023-10-09T04:29:18.166502+0000","last_peered":"2023-10-09T04:29:18.166502+0000","last_clean":"2023-10-09T04:29:18.166502+0000","last_became_active":"2023-10-09T04:27:51.852385+0000","last_became_peered":"2023-10-09T04:27:51.852385+0000","last_unstale":"2023-10-09T04:29:18.166502+0000","last_undegraded":"2023-10-09T04:29:18.166502+0000","last_fullsized":"2023-10-09T04:29:18.166502+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:54.313846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":838,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154968+0000","last_change":"2023-10-09T04:27:45.967966+0000","last_active":"2023-10-09T04:29:18.154968+0000","last_peered":"2023-10-09T04:29:18.154968+0000","last_clean":"2023-10-09T04:29:18.154968+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:29:18.154968+0000","last_undegraded":"2023-10-09T04:29:18.154968+0000","last_fullsized":"2023-10-09T04:29:18.154968+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:45.967906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:45.967906+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:06.528314+0000","scrub_duration":0.0012842089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":491,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157280+0000","last_change":"2023-10-09T04:02:35.091558+0000","last_active":"2023-10-09T04:29:18.157280+0000","last_peered":"2023-10-09T04:29:18.157280+0000","last_clean":"2023-10-09T04:29:18.157280+0000","last_became_active":"2023-10-09T04:02:35.091181+0000","last_became_peered":"2023-10-09T04:02:35.091181+0000","last_unstale":"2023-10-09T04:29:18.157280+0000","last_undegraded":"2023-10-09T04:29:18.157280+0000","last_fullsized":"2023-10-09T04:29:18.157280+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:47:55.176753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":176,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.170780+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:29:18.170780+0000","last_peered":"2023-10-09T04:29:18.170780+0000","last_clean":"2023-10-09T04:29:18.170780+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:29:18.170780+0000","last_undegraded":"2023-10-09T04:29:18.170780+0000","last_fullsized":"2023-10-09T04:29:18.170780+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":736,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158731+0000","last_change":"2023-10-09T04:23:08.643566+0000","last_active":"2023-10-09T04:29:18.158731+0000","last_peered":"2023-10-09T04:29:18.158731+0000","last_clean":"2023-10-09T04:29:18.158731+0000","last_became_active":"2023-10-09T04:23:08.643186+0000","last_became_peered":"2023-10-09T04:23:08.643186+0000","last_unstale":"2023-10-09T04:29:18.158731+0000","last_undegraded":"2023-10-09T04:29:18.158731+0000","last_fullsized":"2023-10-09T04:29:18.158731+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:42.836316+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":606,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155962+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:29:18.155962+0000","last_peered":"2023-10-09T04:29:18.155962+0000","last_clean":"2023-10-09T04:29:18.155962+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:29:18.155962+0000","last_undegraded":"2023-10-09T04:29:18.155962+0000","last_fullsized":"2023-10-09T04:29:18.155962+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156506+0000","last_change":"2023-10-09T04:27:51.852273+0000","last_active":"2023-10-09T04:29:18.156506+0000","last_peered":"2023-10-09T04:29:18.156506+0000","last_clean":"2023-10-09T04:29:18.156506+0000","last_became_active":"2023-10-09T04:27:51.851992+0000","last_became_peered":"2023-10-09T04:27:51.851992+0000","last_unstale":"2023-10-09T04:29:18.156506+0000","last_undegraded":"2023-10-09T04:29:18.156506+0000","last_fullsized":"2023-10-09T04:29:18.156506+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:49.300670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":711,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.163163+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:29:18.163163+0000","last_peered":"2023-10-09T04:29:18.163163+0000","last_clean":"2023-10-09T04:29:18.163163+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:29:18.163163+0000","last_undegraded":"2023-10-09T04:29:18.163163+0000","last_fullsized":"2023-10-09T04:29:18.163163+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":107,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149910+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:29:18.149910+0000","last_peered":"2023-10-09T04:29:18.149910+0000","last_clean":"2023-10-09T04:29:18.149910+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:29:18.149910+0000","last_undegraded":"2023-10-09T04:29:18.149910+0000","last_fullsized":"2023-10-09T04:29:18.149910+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149471+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:29:18.149471+0000","last_peered":"2023-10-09T04:29:18.149471+0000","last_clean":"2023-10-09T04:29:18.149471+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:29:18.149471+0000","last_undegraded":"2023-10-09T04:29:18.149471+0000","last_fullsized":"2023-10-09T04:29:18.149471+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":564,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157168+0000","last_change":"2023-10-09T04:27:02.906226+0000","last_active":"2023-10-09T04:29:18.157168+0000","last_peered":"2023-10-09T04:29:18.157168+0000","last_clean":"2023-10-09T04:29:18.157168+0000","last_became_active":"2023-10-09T04:23:08.629558+0000","last_became_peered":"2023-10-09T04:23:08.629558+0000","last_unstale":"2023-10-09T04:29:18.157168+0000","last_undegraded":"2023-10-09T04:29:18.157168+0000","last_fullsized":"2023-10-09T04:29:18.157168+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:21.516499+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":575,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154737+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:29:18.154737+0000","last_peered":"2023-10-09T04:29:18.154737+0000","last_clean":"2023-10-09T04:29:18.154737+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:29:18.154737+0000","last_undegraded":"2023-10-09T04:29:18.154737+0000","last_fullsized":"2023-10-09T04:29:18.154737+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":307,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154739+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:29:18.154739+0000","last_peered":"2023-10-09T04:29:18.154739+0000","last_clean":"2023-10-09T04:29:18.154739+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:29:18.154739+0000","last_undegraded":"2023-10-09T04:29:18.154739+0000","last_fullsized":"2023-10-09T04:29:18.154739+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":696,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.164127+0000","last_change":"2023-10-09T04:22:16.638789+0000","last_active":"2023-10-09T04:29:18.164127+0000","last_peered":"2023-10-09T04:29:18.164127+0000","last_clean":"2023-10-09T04:29:18.164127+0000","last_became_active":"2023-10-09T04:22:16.638414+0000","last_became_peered":"2023-10-09T04:22:16.638414+0000","last_unstale":"2023-10-09T04:29:18.164127+0000","last_undegraded":"2023-10-09T04:29:18.164127+0000","last_fullsized":"2023-10-09T04:29:18.164127+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:16.338470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:16.338470+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:15.990973+0000","scrub_duration":0.011422943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":137,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158403+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:29:18.158403+0000","last_peered":"2023-10-09T04:29:18.158403+0000","last_clean":"2023-10-09T04:29:18.158403+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:29:18.158403+0000","last_undegraded":"2023-10-09T04:29:18.158403+0000","last_fullsized":"2023-10-09T04:29:18.158403+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":510,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157261+0000","last_change":"2023-10-09T04:18:15.741693+0000","last_active":"2023-10-09T04:29:18.157261+0000","last_peered":"2023-10-09T04:29:18.157261+0000","last_clean":"2023-10-09T04:29:18.157261+0000","last_became_active":"2023-10-09T04:18:15.741013+0000","last_became_peered":"2023-10-09T04:18:15.741013+0000","last_unstale":"2023-10-09T04:29:18.157261+0000","last_undegraded":"2023-10-09T04:29:18.157261+0000","last_fullsized":"2023-10-09T04:29:18.157261+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:39:55.903488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":678,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.161136+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:29:18.161136+0000","last_peered":"2023-10-09T04:29:18.161136+0000","last_clean":"2023-10-09T04:29:18.161136+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:29:18.161136+0000","last_undegraded":"2023-10-09T04:29:18.161136+0000","last_fullsized":"2023-10-09T04:29:18.161136+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.148941+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:29:18.148941+0000","last_peered":"2023-10-09T04:29:18.148941+0000","last_clean":"2023-10-09T04:29:18.148941+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:29:18.148941+0000","last_undegraded":"2023-10-09T04:29:18.148941+0000","last_fullsized":"2023-10-09T04:29:18.148941+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":137,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158262+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:29:18.158262+0000","last_peered":"2023-10-09T04:29:18.158262+0000","last_clean":"2023-10-09T04:29:18.158262+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:29:18.158262+0000","last_undegraded":"2023-10-09T04:29:18.158262+0000","last_fullsized":"2023-10-09T04:29:18.158262+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":696,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157062+0000","last_change":"2023-10-09T04:22:16.641299+0000","last_active":"2023-10-09T04:29:18.157062+0000","last_peered":"2023-10-09T04:29:18.157062+0000","last_clean":"2023-10-09T04:29:18.157062+0000","last_became_active":"2023-10-09T04:22:16.640952+0000","last_became_peered":"2023-10-09T04:22:16.640952+0000","last_unstale":"2023-10-09T04:29:18.157062+0000","last_undegraded":"2023-10-09T04:29:18.157062+0000","last_fullsized":"2023-10-09T04:29:18.157062+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:23.912298+0000","scrub_duration":0.0081178699999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":147,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157218+0000","last_change":"2023-10-09T04:23:06.368734+0000","last_active":"2023-10-09T04:29:18.157218+0000","last_peered":"2023-10-09T04:29:18.157218+0000","last_clean":"2023-10-09T04:29:18.157218+0000","last_became_active":"2023-10-09T04:23:06.340218+0000","last_became_peered":"2023-10-09T04:23:06.340218+0000","last_unstale":"2023-10-09T04:29:18.157218+0000","last_undegraded":"2023-10-09T04:29:18.157218+0000","last_fullsized":"2023-10-09T04:29:18.157218+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:54.950279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":197,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.160836+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:29:18.160836+0000","last_peered":"2023-10-09T04:29:18.160836+0000","last_clean":"2023-10-09T04:29:18.160836+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:29:18.160836+0000","last_undegraded":"2023-10-09T04:29:18.160836+0000","last_fullsized":"2023-10-09T04:29:18.160836+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"532'8010","reported_seq":8043,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.157510+0000","last_change":"2023-10-09T04:27:20.720040+0000","last_active":"2023-10-09T04:29:30.157510+0000","last_peered":"2023-10-09T04:29:30.157510+0000","last_clean":"2023-10-09T04:29:30.157510+0000","last_became_active":"2023-10-09T04:27:20.719715+0000","last_became_peered":"2023-10-09T04:27:20.719715+0000","last_unstale":"2023-10-09T04:29:30.157510+0000","last_undegraded":"2023-10-09T04:29:30.157510+0000","last_fullsized":"2023-10-09T04:29:30.157510+0000","mapping_epoch":507,"log_start":"511'300","ondisk_log_start":"511'300","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7710,"log_dups_size":0,"ondisk_log_size":7710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:37.979204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333053952,"num_objects":5082,"num_object_clones":0,"num_object_copies":15246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8010,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":974,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155714+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:29:18.155714+0000","last_peered":"2023-10-09T04:29:18.155714+0000","last_clean":"2023-10-09T04:29:18.155714+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:29:18.155714+0000","last_undegraded":"2023-10-09T04:29:18.155714+0000","last_fullsized":"2023-10-09T04:29:18.155714+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":576,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155010+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:29:18.155010+0000","last_peered":"2023-10-09T04:29:18.155010+0000","last_clean":"2023-10-09T04:29:18.155010+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:29:18.155010+0000","last_undegraded":"2023-10-09T04:29:18.155010+0000","last_fullsized":"2023-10-09T04:29:18.155010+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":493,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156282+0000","last_change":"2023-10-09T04:22:13.633467+0000","last_active":"2023-10-09T04:29:18.156282+0000","last_peered":"2023-10-09T04:29:18.156282+0000","last_clean":"2023-10-09T04:29:18.156282+0000","last_became_active":"2023-10-09T04:22:13.633156+0000","last_became_peered":"2023-10-09T04:22:13.633156+0000","last_unstale":"2023-10-09T04:29:18.156282+0000","last_undegraded":"2023-10-09T04:29:18.156282+0000","last_fullsized":"2023-10-09T04:29:18.156282+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:28.020006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"532'8158","reported_seq":8192,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:31.861952+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:29:31.861952+0000","last_peered":"2023-10-09T04:29:31.861952+0000","last_clean":"2023-10-09T04:29:31.861952+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:29:31.861952+0000","last_undegraded":"2023-10-09T04:29:31.861952+0000","last_fullsized":"2023-10-09T04:29:31.861952+0000","mapping_epoch":507,"log_start":"512'800","ondisk_log_start":"512'800","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7358,"log_dups_size":0,"ondisk_log_size":7358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333053952,"num_objects":5082,"num_object_clones":0,"num_object_copies":15246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8158,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":765,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.159972+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:29:18.159972+0000","last_peered":"2023-10-09T04:29:18.159972+0000","last_clean":"2023-10-09T04:29:18.159972+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:29:18.159972+0000","last_undegraded":"2023-10-09T04:29:18.159972+0000","last_fullsized":"2023-10-09T04:29:18.159972+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":552,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154954+0000","last_change":"2023-10-09T04:24:56.212579+0000","last_active":"2023-10-09T04:29:18.154954+0000","last_peered":"2023-10-09T04:29:18.154954+0000","last_clean":"2023-10-09T04:29:18.154954+0000","last_became_active":"2023-10-09T04:24:56.184403+0000","last_became_peered":"2023-10-09T04:24:56.184403+0000","last_unstale":"2023-10-09T04:29:18.154954+0000","last_undegraded":"2023-10-09T04:29:18.154954+0000","last_fullsized":"2023-10-09T04:29:18.154954+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:10.486526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149026+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:29:18.149026+0000","last_peered":"2023-10-09T04:29:18.149026+0000","last_clean":"2023-10-09T04:29:18.149026+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:29:18.149026+0000","last_undegraded":"2023-10-09T04:29:18.149026+0000","last_fullsized":"2023-10-09T04:29:18.149026+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"532'8034","reported_seq":8067,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.550928+0000","last_change":"2023-10-09T04:27:20.720676+0000","last_active":"2023-10-09T04:29:30.550928+0000","last_peered":"2023-10-09T04:29:30.550928+0000","last_clean":"2023-10-09T04:29:30.550928+0000","last_became_active":"2023-10-09T04:27:20.720342+0000","last_became_peered":"2023-10-09T04:27:20.720342+0000","last_unstale":"2023-10-09T04:29:30.550928+0000","last_undegraded":"2023-10-09T04:29:30.550928+0000","last_fullsized":"2023-10-09T04:29:30.550928+0000","mapping_epoch":507,"log_start":"513'1900","ondisk_log_start":"513'1900","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:37.485070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331874304,"num_objects":5064,"num_object_clones":0,"num_object_copies":15192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8034,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":736,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158653+0000","last_change":"2023-10-09T04:24:58.648512+0000","last_active":"2023-10-09T04:29:18.158653+0000","last_peered":"2023-10-09T04:29:18.158653+0000","last_clean":"2023-10-09T04:29:18.158653+0000","last_became_active":"2023-10-09T04:24:58.648143+0000","last_became_peered":"2023-10-09T04:24:58.648143+0000","last_unstale":"2023-10-09T04:29:18.158653+0000","last_undegraded":"2023-10-09T04:29:18.158653+0000","last_fullsized":"2023-10-09T04:29:18.158653+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:26.096956+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":503,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155872+0000","last_change":"2023-10-09T04:22:13.648024+0000","last_active":"2023-10-09T04:29:18.155872+0000","last_peered":"2023-10-09T04:29:18.155872+0000","last_clean":"2023-10-09T04:29:18.155872+0000","last_became_active":"2023-10-09T04:22:13.630451+0000","last_became_peered":"2023-10-09T04:22:13.630451+0000","last_unstale":"2023-10-09T04:29:18.155872+0000","last_undegraded":"2023-10-09T04:29:18.155872+0000","last_fullsized":"2023-10-09T04:29:18.155872+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:00.305859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150340+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:29:18.150340+0000","last_peered":"2023-10-09T04:29:18.150340+0000","last_clean":"2023-10-09T04:29:18.150340+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:29:18.150340+0000","last_undegraded":"2023-10-09T04:29:18.150340+0000","last_fullsized":"2023-10-09T04:29:18.150340+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"532'7921","reported_seq":7954,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.547307+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:29:30.547307+0000","last_peered":"2023-10-09T04:29:30.547307+0000","last_clean":"2023-10-09T04:29:30.547307+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:29:30.547307+0000","last_undegraded":"2023-10-09T04:29:30.547307+0000","last_fullsized":"2023-10-09T04:29:30.547307+0000","mapping_epoch":507,"log_start":"512'1700","ondisk_log_start":"512'1700","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6221,"log_dups_size":0,"ondisk_log_size":6221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158016,"num_objects":4931,"num_object_clones":0,"num_object_copies":14793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7921,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":788,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150350+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:29:18.150350+0000","last_peered":"2023-10-09T04:29:18.150350+0000","last_clean":"2023-10-09T04:29:18.150350+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:29:18.150350+0000","last_undegraded":"2023-10-09T04:29:18.150350+0000","last_fullsized":"2023-10-09T04:29:18.150350+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":531,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156922+0000","last_change":"2023-10-09T04:22:13.625825+0000","last_active":"2023-10-09T04:29:18.156922+0000","last_peered":"2023-10-09T04:29:18.156922+0000","last_clean":"2023-10-09T04:29:18.156922+0000","last_became_active":"2023-10-09T04:22:13.625591+0000","last_became_peered":"2023-10-09T04:22:13.625591+0000","last_unstale":"2023-10-09T04:29:18.156922+0000","last_undegraded":"2023-10-09T04:29:18.156922+0000","last_fullsized":"2023-10-09T04:29:18.156922+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:30.033137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":155,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.161877+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:29:18.161877+0000","last_peered":"2023-10-09T04:29:18.161877+0000","last_clean":"2023-10-09T04:29:18.161877+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:29:18.161877+0000","last_undegraded":"2023-10-09T04:29:18.161877+0000","last_fullsized":"2023-10-09T04:29:18.161877+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"532'8087","reported_seq":8120,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.493868+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:29:30.493868+0000","last_peered":"2023-10-09T04:29:30.493868+0000","last_clean":"2023-10-09T04:29:30.493868+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:29:30.493868+0000","last_undegraded":"2023-10-09T04:29:30.493868+0000","last_fullsized":"2023-10-09T04:29:30.493868+0000","mapping_epoch":507,"log_start":"513'1900","ondisk_log_start":"513'1900","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6187,"log_dups_size":0,"ondisk_log_size":6187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8087,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":995,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154502+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:29:18.154502+0000","last_peered":"2023-10-09T04:29:18.154502+0000","last_clean":"2023-10-09T04:29:18.154502+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:29:18.154502+0000","last_undegraded":"2023-10-09T04:29:18.154502+0000","last_fullsized":"2023-10-09T04:29:18.154502+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":167,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.161809+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:29:18.161809+0000","last_peered":"2023-10-09T04:29:18.161809+0000","last_clean":"2023-10-09T04:29:18.161809+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:29:18.161809+0000","last_undegraded":"2023-10-09T04:29:18.161809+0000","last_fullsized":"2023-10-09T04:29:18.161809+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149825+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:29:18.149825+0000","last_peered":"2023-10-09T04:29:18.149825+0000","last_clean":"2023-10-09T04:29:18.149825+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:29:18.149825+0000","last_undegraded":"2023-10-09T04:29:18.149825+0000","last_fullsized":"2023-10-09T04:29:18.149825+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"532'8148","reported_seq":8183,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:31.869684+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:29:31.869684+0000","last_peered":"2023-10-09T04:29:31.869684+0000","last_clean":"2023-10-09T04:29:31.869684+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:29:31.869684+0000","last_undegraded":"2023-10-09T04:29:31.869684+0000","last_fullsized":"2023-10-09T04:29:31.869684+0000","mapping_epoch":507,"log_start":"512'800","ondisk_log_start":"512'800","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322699264,"num_objects":4924,"num_object_clones":0,"num_object_copies":14772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8148,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":741,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.164193+0000","last_change":"2023-10-09T04:26:23.679008+0000","last_active":"2023-10-09T04:29:18.164193+0000","last_peered":"2023-10-09T04:29:18.164193+0000","last_clean":"2023-10-09T04:29:18.164193+0000","last_became_active":"2023-10-09T04:26:23.678629+0000","last_became_peered":"2023-10-09T04:26:23.678629+0000","last_unstale":"2023-10-09T04:29:18.164193+0000","last_undegraded":"2023-10-09T04:29:18.164193+0000","last_fullsized":"2023-10-09T04:29:18.164193+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:41.534992+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149126+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:29:18.149126+0000","last_peered":"2023-10-09T04:29:18.149126+0000","last_clean":"2023-10-09T04:29:18.149126+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:29:18.149126+0000","last_undegraded":"2023-10-09T04:29:18.149126+0000","last_fullsized":"2023-10-09T04:29:18.149126+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":114,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150019+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:29:18.150019+0000","last_peered":"2023-10-09T04:29:18.150019+0000","last_clean":"2023-10-09T04:29:18.150019+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:29:18.150019+0000","last_undegraded":"2023-10-09T04:29:18.150019+0000","last_fullsized":"2023-10-09T04:29:18.150019+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"532'8082","reported_seq":8117,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.662695+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:29:30.662695+0000","last_peered":"2023-10-09T04:29:30.662695+0000","last_clean":"2023-10-09T04:29:30.662695+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:29:30.662695+0000","last_undegraded":"2023-10-09T04:29:30.662695+0000","last_fullsized":"2023-10-09T04:29:30.662695+0000","mapping_epoch":507,"log_start":"513'2100","ondisk_log_start":"513'2100","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5982,"log_dups_size":0,"ondisk_log_size":5982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329252864,"num_objects":5024,"num_object_clones":0,"num_object_copies":15072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8082,"num_write_kb":419392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":638,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.161948+0000","last_change":"2023-10-09T04:23:08.629758+0000","last_active":"2023-10-09T04:29:18.161948+0000","last_peered":"2023-10-09T04:29:18.161948+0000","last_clean":"2023-10-09T04:29:18.161948+0000","last_became_active":"2023-10-09T04:23:08.629395+0000","last_became_peered":"2023-10-09T04:23:08.629395+0000","last_unstale":"2023-10-09T04:29:18.161948+0000","last_undegraded":"2023-10-09T04:29:18.161948+0000","last_fullsized":"2023-10-09T04:29:18.161948+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:20.917460+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149053+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:29:18.149053+0000","last_peered":"2023-10-09T04:29:18.149053+0000","last_clean":"2023-10-09T04:29:18.149053+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:29:18.149053+0000","last_undegraded":"2023-10-09T04:29:18.149053+0000","last_fullsized":"2023-10-09T04:29:18.149053+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150432+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:29:18.150432+0000","last_peered":"2023-10-09T04:29:18.150432+0000","last_clean":"2023-10-09T04:29:18.150432+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:29:18.150432+0000","last_undegraded":"2023-10-09T04:29:18.150432+0000","last_fullsized":"2023-10-09T04:29:18.150432+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"532'8025","reported_seq":8060,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:29.254566+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:29:29.254566+0000","last_peered":"2023-10-09T04:29:29.254566+0000","last_clean":"2023-10-09T04:29:29.254566+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:29:29.254566+0000","last_undegraded":"2023-10-09T04:29:29.254566+0000","last_fullsized":"2023-10-09T04:29:29.254566+0000","mapping_epoch":507,"log_start":"513'2000","ondisk_log_start":"513'2000","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344391680,"num_objects":5255,"num_object_clones":0,"num_object_copies":15765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8025,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":736,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156593+0000","last_change":"2023-10-09T04:26:23.680981+0000","last_active":"2023-10-09T04:29:18.156593+0000","last_peered":"2023-10-09T04:29:18.156593+0000","last_clean":"2023-10-09T04:29:18.156593+0000","last_became_active":"2023-10-09T04:26:23.680615+0000","last_became_peered":"2023-10-09T04:26:23.680615+0000","last_unstale":"2023-10-09T04:29:18.156593+0000","last_undegraded":"2023-10-09T04:29:18.156593+0000","last_fullsized":"2023-10-09T04:29:18.156593+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:36.804345+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":198,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.161272+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:29:18.161272+0000","last_peered":"2023-10-09T04:29:18.161272+0000","last_clean":"2023-10-09T04:29:18.161272+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:29:18.161272+0000","last_undegraded":"2023-10-09T04:29:18.161272+0000","last_fullsized":"2023-10-09T04:29:18.161272+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150467+0000","last_change":"2023-10-09T04:26:09.746900+0000","last_active":"2023-10-09T04:29:18.150467+0000","last_peered":"2023-10-09T04:29:18.150467+0000","last_clean":"2023-10-09T04:29:18.150467+0000","last_became_active":"2023-10-09T04:26:09.746607+0000","last_became_peered":"2023-10-09T04:26:09.746607+0000","last_unstale":"2023-10-09T04:29:18.150467+0000","last_undegraded":"2023-10-09T04:29:18.150467+0000","last_fullsized":"2023-10-09T04:29:18.150467+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:07.695043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"532'7992","reported_seq":8023,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.187772+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:29:30.187772+0000","last_peered":"2023-10-09T04:29:30.187772+0000","last_clean":"2023-10-09T04:29:30.187772+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:29:30.187772+0000","last_undegraded":"2023-10-09T04:29:30.187772+0000","last_fullsized":"2023-10-09T04:29:30.187772+0000","mapping_epoch":507,"log_start":"512'1700","ondisk_log_start":"512'1700","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6292,"log_dups_size":0,"ondisk_log_size":6292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334430232,"num_objects":5104,"num_object_clones":0,"num_object_copies":15312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5104,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7992,"num_write_kb":419009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":658,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150861+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:29:18.150861+0000","last_peered":"2023-10-09T04:29:18.150861+0000","last_clean":"2023-10-09T04:29:18.150861+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:29:18.150861+0000","last_undegraded":"2023-10-09T04:29:18.150861+0000","last_fullsized":"2023-10-09T04:29:18.150861+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":136,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.894308+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:29:21.894308+0000","last_peered":"2023-10-09T04:29:21.894308+0000","last_clean":"2023-10-09T04:29:21.894308+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:29:21.894308+0000","last_undegraded":"2023-10-09T04:29:21.894308+0000","last_fullsized":"2023-10-09T04:29:21.894308+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":548,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157186+0000","last_change":"2023-10-09T04:26:21.351382+0000","last_active":"2023-10-09T04:29:18.157186+0000","last_peered":"2023-10-09T04:29:18.157186+0000","last_clean":"2023-10-09T04:29:18.157186+0000","last_became_active":"2023-10-09T04:26:21.348023+0000","last_became_peered":"2023-10-09T04:26:21.348023+0000","last_unstale":"2023-10-09T04:29:18.157186+0000","last_undegraded":"2023-10-09T04:29:18.157186+0000","last_fullsized":"2023-10-09T04:29:18.157186+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:45.177721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":411,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156978+0000","last_change":"2023-10-09T04:26:09.749151+0000","last_active":"2023-10-09T04:29:18.156978+0000","last_peered":"2023-10-09T04:29:18.156978+0000","last_clean":"2023-10-09T04:29:18.156978+0000","last_became_active":"2023-10-09T04:26:09.748857+0000","last_became_peered":"2023-10-09T04:26:09.748857+0000","last_unstale":"2023-10-09T04:29:18.156978+0000","last_undegraded":"2023-10-09T04:29:18.156978+0000","last_fullsized":"2023-10-09T04:29:18.156978+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:38.228161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"532'8316","reported_seq":8350,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:31.812639+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:29:31.812639+0000","last_peered":"2023-10-09T04:29:31.812639+0000","last_clean":"2023-10-09T04:29:31.812639+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:29:31.812639+0000","last_undegraded":"2023-10-09T04:29:31.812639+0000","last_fullsized":"2023-10-09T04:29:31.812639+0000","mapping_epoch":507,"log_start":"512'900","ondisk_log_start":"512'900","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7416,"log_dups_size":0,"ondisk_log_size":7416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334626816,"num_objects":5106,"num_object_clones":0,"num_object_copies":15318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8316,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":739,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156446+0000","last_change":"2023-10-09T04:26:23.684553+0000","last_active":"2023-10-09T04:29:18.156446+0000","last_peered":"2023-10-09T04:29:18.156446+0000","last_clean":"2023-10-09T04:29:18.156446+0000","last_became_active":"2023-10-09T04:26:23.684125+0000","last_became_peered":"2023-10-09T04:26:23.684125+0000","last_unstale":"2023-10-09T04:29:18.156446+0000","last_undegraded":"2023-10-09T04:29:18.156446+0000","last_fullsized":"2023-10-09T04:29:18.156446+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:01.715214+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":515,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156675+0000","last_change":"2023-10-09T04:22:13.622952+0000","last_active":"2023-10-09T04:29:18.156675+0000","last_peered":"2023-10-09T04:29:18.156675+0000","last_clean":"2023-10-09T04:29:18.156675+0000","last_became_active":"2023-10-09T04:22:13.622224+0000","last_became_peered":"2023-10-09T04:22:13.622224+0000","last_unstale":"2023-10-09T04:29:18.156675+0000","last_undegraded":"2023-10-09T04:29:18.156675+0000","last_fullsized":"2023-10-09T04:29:18.156675+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:13.530667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149529+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:29:18.149529+0000","last_peered":"2023-10-09T04:29:18.149529+0000","last_clean":"2023-10-09T04:29:18.149529+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:29:18.149529+0000","last_undegraded":"2023-10-09T04:29:18.149529+0000","last_fullsized":"2023-10-09T04:29:18.149529+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":27,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156949+0000","last_change":"2023-10-09T04:27:51.852322+0000","last_active":"2023-10-09T04:29:18.156949+0000","last_peered":"2023-10-09T04:29:18.156949+0000","last_clean":"2023-10-09T04:29:18.156949+0000","last_became_active":"2023-10-09T04:27:51.851990+0000","last_became_peered":"2023-10-09T04:27:51.851990+0000","last_unstale":"2023-10-09T04:29:18.156949+0000","last_undegraded":"2023-10-09T04:29:18.156949+0000","last_fullsized":"2023-10-09T04:29:18.156949+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:19.671405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"532'7822","reported_seq":7855,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.135897+0000","last_change":"2023-10-09T04:27:20.717622+0000","last_active":"2023-10-09T04:29:30.135897+0000","last_peered":"2023-10-09T04:29:30.135897+0000","last_clean":"2023-10-09T04:29:30.135897+0000","last_became_active":"2023-10-09T04:27:20.717291+0000","last_became_peered":"2023-10-09T04:27:20.717291+0000","last_unstale":"2023-10-09T04:29:30.135897+0000","last_undegraded":"2023-10-09T04:29:30.135897+0000","last_fullsized":"2023-10-09T04:29:30.135897+0000","mapping_epoch":507,"log_start":"511'100","ondisk_log_start":"511'100","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:34.329924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329777152,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7822,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":706,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158319+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:29:18.158319+0000","last_peered":"2023-10-09T04:29:18.158319+0000","last_clean":"2023-10-09T04:29:18.158319+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:29:18.158319+0000","last_undegraded":"2023-10-09T04:29:18.158319+0000","last_fullsized":"2023-10-09T04:29:18.158319+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":383,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157010+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:29:18.157010+0000","last_peered":"2023-10-09T04:29:18.157010+0000","last_clean":"2023-10-09T04:29:18.157010+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:29:18.157010+0000","last_undegraded":"2023-10-09T04:29:18.157010+0000","last_fullsized":"2023-10-09T04:29:18.157010+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":472,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.164342+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:29:18.164342+0000","last_peered":"2023-10-09T04:29:18.164342+0000","last_clean":"2023-10-09T04:29:18.164342+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:29:18.164342+0000","last_undegraded":"2023-10-09T04:29:18.164342+0000","last_fullsized":"2023-10-09T04:29:18.164342+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":237,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.159011+0000","last_change":"2023-10-09T04:23:37.012345+0000","last_active":"2023-10-09T04:29:18.159011+0000","last_peered":"2023-10-09T04:29:18.159011+0000","last_clean":"2023-10-09T04:29:18.159011+0000","last_became_active":"2023-10-09T04:23:37.011999+0000","last_became_peered":"2023-10-09T04:23:37.011999+0000","last_unstale":"2023-10-09T04:29:18.159011+0000","last_undegraded":"2023-10-09T04:29:18.159011+0000","last_fullsized":"2023-10-09T04:29:18.159011+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:00.455694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"532'7879","reported_seq":7912,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.201599+0000","last_change":"2023-10-09T04:27:20.717505+0000","last_active":"2023-10-09T04:29:30.201599+0000","last_peered":"2023-10-09T04:29:30.201599+0000","last_clean":"2023-10-09T04:29:30.201599+0000","last_became_active":"2023-10-09T04:27:20.717202+0000","last_became_peered":"2023-10-09T04:27:20.717202+0000","last_unstale":"2023-10-09T04:29:30.201599+0000","last_undegraded":"2023-10-09T04:29:30.201599+0000","last_fullsized":"2023-10-09T04:29:30.201599+0000","mapping_epoch":507,"log_start":"511'100","ondisk_log_start":"511'100","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:51.034299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333250560,"num_objects":5085,"num_object_clones":0,"num_object_copies":15255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7879,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":535,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156653+0000","last_change":"2023-10-09T04:26:21.429887+0000","last_active":"2023-10-09T04:29:18.156653+0000","last_peered":"2023-10-09T04:29:18.156653+0000","last_clean":"2023-10-09T04:29:18.156653+0000","last_became_active":"2023-10-09T04:26:21.392380+0000","last_became_peered":"2023-10-09T04:26:21.392380+0000","last_unstale":"2023-10-09T04:29:18.156653+0000","last_undegraded":"2023-10-09T04:29:18.156653+0000","last_fullsized":"2023-10-09T04:29:18.156653+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:05:55.650685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":986,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154323+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:29:18.154323+0000","last_peered":"2023-10-09T04:29:18.154323+0000","last_clean":"2023-10-09T04:29:18.154323+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:29:18.154323+0000","last_undegraded":"2023-10-09T04:29:18.154323+0000","last_fullsized":"2023-10-09T04:29:18.154323+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":468,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156889+0000","last_change":"2023-10-09T04:22:13.623126+0000","last_active":"2023-10-09T04:29:18.156889+0000","last_peered":"2023-10-09T04:29:18.156889+0000","last_clean":"2023-10-09T04:29:18.156889+0000","last_became_active":"2023-10-09T04:22:13.622658+0000","last_became_peered":"2023-10-09T04:22:13.622658+0000","last_unstale":"2023-10-09T04:29:18.156889+0000","last_undegraded":"2023-10-09T04:29:18.156889+0000","last_fullsized":"2023-10-09T04:29:18.156889+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:12.671162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":94,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149862+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:29:18.149862+0000","last_peered":"2023-10-09T04:29:18.149862+0000","last_clean":"2023-10-09T04:29:18.149862+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:29:18.149862+0000","last_undegraded":"2023-10-09T04:29:18.149862+0000","last_fullsized":"2023-10-09T04:29:18.149862+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"532'8057","reported_seq":8092,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:31.897967+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:29:31.897967+0000","last_peered":"2023-10-09T04:29:31.897967+0000","last_clean":"2023-10-09T04:29:31.897967+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:29:31.897967+0000","last_undegraded":"2023-10-09T04:29:31.897967+0000","last_fullsized":"2023-10-09T04:29:31.897967+0000","mapping_epoch":507,"log_start":"512'700","ondisk_log_start":"512'700","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":7357,"log_dups_size":0,"ondisk_log_size":7357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326172672,"num_objects":4977,"num_object_clones":0,"num_object_copies":14931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8057,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":627,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149563+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:29:18.149563+0000","last_peered":"2023-10-09T04:29:18.149563+0000","last_clean":"2023-10-09T04:29:18.149563+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:29:18.149563+0000","last_undegraded":"2023-10-09T04:29:18.149563+0000","last_fullsized":"2023-10-09T04:29:18.149563+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":532,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154289+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:29:18.154289+0000","last_peered":"2023-10-09T04:29:18.154289+0000","last_clean":"2023-10-09T04:29:18.154289+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:29:18.154289+0000","last_undegraded":"2023-10-09T04:29:18.154289+0000","last_fullsized":"2023-10-09T04:29:18.154289+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":174,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.894223+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:29:21.894223+0000","last_peered":"2023-10-09T04:29:21.894223+0000","last_clean":"2023-10-09T04:29:21.894223+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:29:21.894223+0000","last_undegraded":"2023-10-09T04:29:21.894223+0000","last_fullsized":"2023-10-09T04:29:21.894223+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":155,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158191+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:29:18.158191+0000","last_peered":"2023-10-09T04:29:18.158191+0000","last_clean":"2023-10-09T04:29:18.158191+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:29:18.158191+0000","last_undegraded":"2023-10-09T04:29:18.158191+0000","last_fullsized":"2023-10-09T04:29:18.158191+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"532'8105","reported_seq":8138,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.197033+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:29:30.197033+0000","last_peered":"2023-10-09T04:29:30.197033+0000","last_clean":"2023-10-09T04:29:30.197033+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:29:30.197033+0000","last_undegraded":"2023-10-09T04:29:30.197033+0000","last_fullsized":"2023-10-09T04:29:30.197033+0000","mapping_epoch":507,"log_start":"512'1800","ondisk_log_start":"512'1800","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331939840,"num_objects":5065,"num_object_clones":0,"num_object_copies":15195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8105,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":650,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150904+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:29:18.150904+0000","last_peered":"2023-10-09T04:29:18.150904+0000","last_clean":"2023-10-09T04:29:18.150904+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:29:18.150904+0000","last_undegraded":"2023-10-09T04:29:18.150904+0000","last_fullsized":"2023-10-09T04:29:18.150904+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":555,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155917+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:29:18.155917+0000","last_peered":"2023-10-09T04:29:18.155917+0000","last_clean":"2023-10-09T04:29:18.155917+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:29:18.155917+0000","last_undegraded":"2023-10-09T04:29:18.155917+0000","last_fullsized":"2023-10-09T04:29:18.155917+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":290,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156846+0000","last_change":"2023-10-09T04:26:21.355315+0000","last_active":"2023-10-09T04:29:18.156846+0000","last_peered":"2023-10-09T04:29:18.156846+0000","last_clean":"2023-10-09T04:29:18.156846+0000","last_became_active":"2023-10-09T04:26:21.355038+0000","last_became_peered":"2023-10-09T04:26:21.355038+0000","last_unstale":"2023-10-09T04:29:18.156846+0000","last_undegraded":"2023-10-09T04:29:18.156846+0000","last_fullsized":"2023-10-09T04:29:18.156846+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:45.819056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":251,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156505+0000","last_change":"2023-10-09T04:23:06.349931+0000","last_active":"2023-10-09T04:29:18.156505+0000","last_peered":"2023-10-09T04:29:18.156505+0000","last_clean":"2023-10-09T04:29:18.156505+0000","last_became_active":"2023-10-09T04:23:06.349630+0000","last_became_peered":"2023-10-09T04:23:06.349630+0000","last_unstale":"2023-10-09T04:29:18.156505+0000","last_undegraded":"2023-10-09T04:29:18.156505+0000","last_fullsized":"2023-10-09T04:29:18.156505+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:24.023613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"532'8085","reported_seq":8118,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.532603+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:29:30.532603+0000","last_peered":"2023-10-09T04:29:30.532603+0000","last_clean":"2023-10-09T04:29:30.532603+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:29:30.532603+0000","last_undegraded":"2023-10-09T04:29:30.532603+0000","last_fullsized":"2023-10-09T04:29:30.532603+0000","mapping_epoch":507,"log_start":"512'1900","ondisk_log_start":"512'1900","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330366976,"num_objects":5041,"num_object_clones":0,"num_object_copies":15123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8085,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":589,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149942+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:29:18.149942+0000","last_peered":"2023-10-09T04:29:18.149942+0000","last_clean":"2023-10-09T04:29:18.149942+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:29:18.149942+0000","last_undegraded":"2023-10-09T04:29:18.149942+0000","last_fullsized":"2023-10-09T04:29:18.149942+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":492,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156543+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:29:18.156543+0000","last_peered":"2023-10-09T04:29:18.156543+0000","last_clean":"2023-10-09T04:29:18.156543+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:29:18.156543+0000","last_undegraded":"2023-10-09T04:29:18.156543+0000","last_fullsized":"2023-10-09T04:29:18.156543+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150506+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:29:18.150506+0000","last_peered":"2023-10-09T04:29:18.150506+0000","last_clean":"2023-10-09T04:29:18.150506+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:29:18.150506+0000","last_undegraded":"2023-10-09T04:29:18.150506+0000","last_fullsized":"2023-10-09T04:29:18.150506+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":440,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154473+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:29:18.154473+0000","last_peered":"2023-10-09T04:29:18.154473+0000","last_clean":"2023-10-09T04:29:18.154473+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:29:18.154473+0000","last_undegraded":"2023-10-09T04:29:18.154473+0000","last_fullsized":"2023-10-09T04:29:18.154473+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"532'7886","reported_seq":7919,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-09T04:29:30.167554+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:29:30.167554+0000","last_peered":"2023-10-09T04:29:30.167554+0000","last_clean":"2023-10-09T04:29:30.167554+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:29:30.167554+0000","last_undegraded":"2023-10-09T04:29:30.167554+0000","last_fullsized":"2023-10-09T04:29:30.167554+0000","mapping_epoch":507,"log_start":"512'1600","ondisk_log_start":"512'1600","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6286,"log_dups_size":0,"ondisk_log_size":6286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329908224,"num_objects":5034,"num_object_clones":0,"num_object_copies":15102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7886,"num_write_kb":413440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":708,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.159892+0000","last_change":"2023-10-09T04:22:30.010971+0000","last_active":"2023-10-09T04:29:18.159892+0000","last_peered":"2023-10-09T04:29:18.159892+0000","last_clean":"2023-10-09T04:29:18.159892+0000","last_became_active":"2023-10-09T04:22:30.010618+0000","last_became_peered":"2023-10-09T04:22:30.010618+0000","last_unstale":"2023-10-09T04:29:18.159892+0000","last_undegraded":"2023-10-09T04:29:18.159892+0000","last_fullsized":"2023-10-09T04:29:18.159892+0000","mapping_epoch":392,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:02.842528+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":331,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150939+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:29:18.150939+0000","last_peered":"2023-10-09T04:29:18.150939+0000","last_clean":"2023-10-09T04:29:18.150939+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:29:18.150939+0000","last_undegraded":"2023-10-09T04:29:18.150939+0000","last_fullsized":"2023-10-09T04:29:18.150939+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":167,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.158476+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:29:18.158476+0000","last_peered":"2023-10-09T04:29:18.158476+0000","last_clean":"2023-10-09T04:29:18.158476+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:29:18.158476+0000","last_undegraded":"2023-10-09T04:29:18.158476+0000","last_fullsized":"2023-10-09T04:29:18.158476+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.148951+0000","last_change":"2023-10-09T04:24:56.169203+0000","last_active":"2023-10-09T04:29:18.148951+0000","last_peered":"2023-10-09T04:29:18.148951+0000","last_clean":"2023-10-09T04:29:18.148951+0000","last_became_active":"2023-10-09T04:24:56.168737+0000","last_became_peered":"2023-10-09T04:24:56.168737+0000","last_unstale":"2023-10-09T04:29:18.148951+0000","last_undegraded":"2023-10-09T04:29:18.148951+0000","last_fullsized":"2023-10-09T04:29:18.148951+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:21.515886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":575,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157091+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:29:18.157091+0000","last_peered":"2023-10-09T04:29:18.157091+0000","last_clean":"2023-10-09T04:29:18.157091+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:29:18.157091+0000","last_undegraded":"2023-10-09T04:29:18.157091+0000","last_fullsized":"2023-10-09T04:29:18.157091+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":155,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.162105+0000","last_change":"2023-10-09T04:24:56.170052+0000","last_active":"2023-10-09T04:29:18.162105+0000","last_peered":"2023-10-09T04:29:18.162105+0000","last_clean":"2023-10-09T04:29:18.162105+0000","last_became_active":"2023-10-09T04:24:56.169607+0000","last_became_peered":"2023-10-09T04:24:56.169607+0000","last_unstale":"2023-10-09T04:29:18.162105+0000","last_undegraded":"2023-10-09T04:29:18.162105+0000","last_fullsized":"2023-10-09T04:29:18.162105+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.849923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":72,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.155347+0000","last_change":"2023-10-09T04:26:21.345230+0000","last_active":"2023-10-09T04:29:18.155347+0000","last_peered":"2023-10-09T04:29:18.155347+0000","last_clean":"2023-10-09T04:29:18.155347+0000","last_became_active":"2023-10-09T04:26:21.344939+0000","last_became_peered":"2023-10-09T04:26:21.344939+0000","last_unstale":"2023-10-09T04:29:18.155347+0000","last_undegraded":"2023-10-09T04:29:18.155347+0000","last_fullsized":"2023-10-09T04:29:18.155347+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:14.495703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":606,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.150822+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:29:18.150822+0000","last_peered":"2023-10-09T04:29:18.150822+0000","last_clean":"2023-10-09T04:29:18.150822+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:29:18.150822+0000","last_undegraded":"2023-10-09T04:29:18.150822+0000","last_fullsized":"2023-10-09T04:29:18.150822+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":549,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157063+0000","last_change":"2023-10-09T04:24:56.181978+0000","last_active":"2023-10-09T04:29:18.157063+0000","last_peered":"2023-10-09T04:29:18.157063+0000","last_clean":"2023-10-09T04:29:18.157063+0000","last_became_active":"2023-10-09T04:24:56.164297+0000","last_became_peered":"2023-10-09T04:24:56.164297+0000","last_unstale":"2023-10-09T04:29:18.157063+0000","last_undegraded":"2023-10-09T04:29:18.157063+0000","last_fullsized":"2023-10-09T04:29:18.157063+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:15.703672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149437+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:29:18.149437+0000","last_peered":"2023-10-09T04:29:18.149437+0000","last_clean":"2023-10-09T04:29:18.149437+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:29:18.149437+0000","last_undegraded":"2023-10-09T04:29:18.149437+0000","last_fullsized":"2023-10-09T04:29:18.149437+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":560,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.157085+0000","last_change":"2023-10-09T04:24:56.202659+0000","last_active":"2023-10-09T04:29:18.157085+0000","last_peered":"2023-10-09T04:29:18.157085+0000","last_clean":"2023-10-09T04:29:18.157085+0000","last_became_active":"2023-10-09T04:24:56.166522+0000","last_became_peered":"2023-10-09T04:24:56.166522+0000","last_unstale":"2023-10-09T04:29:18.157085+0000","last_undegraded":"2023-10-09T04:29:18.157085+0000","last_fullsized":"2023-10-09T04:29:18.157085+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:15.858204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":34,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.148912+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:29:18.148912+0000","last_peered":"2023-10-09T04:29:18.148912+0000","last_clean":"2023-10-09T04:29:18.148912+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:29:18.148912+0000","last_undegraded":"2023-10-09T04:29:18.148912+0000","last_fullsized":"2023-10-09T04:29:18.148912+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":785,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.154669+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:29:18.154669+0000","last_peered":"2023-10-09T04:29:18.154669+0000","last_clean":"2023-10-09T04:29:18.154669+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:29:18.154669+0000","last_undegraded":"2023-10-09T04:29:18.154669+0000","last_fullsized":"2023-10-09T04:29:18.154669+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":266,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.160696+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:29:18.160696+0000","last_peered":"2023-10-09T04:29:18.160696+0000","last_clean":"2023-10-09T04:29:18.160696+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:29:18.160696+0000","last_undegraded":"2023-10-09T04:29:18.160696+0000","last_fullsized":"2023-10-09T04:29:18.160696+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":383,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.156941+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:29:18.156941+0000","last_peered":"2023-10-09T04:29:18.156941+0000","last_clean":"2023-10-09T04:29:18.156941+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:29:18.156941+0000","last_undegraded":"2023-10-09T04:29:18.156941+0000","last_fullsized":"2023-10-09T04:29:18.156941+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":643,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.149785+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:29:18.149785+0000","last_peered":"2023-10-09T04:29:18.149785+0000","last_clean":"2023-10-09T04:29:18.149785+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:29:18.149785+0000","last_undegraded":"2023-10-09T04:29:18.149785+0000","last_fullsized":"2023-10-09T04:29:18.149785+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":160,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:21.894397+0000","last_change":"2023-10-09T04:26:21.347607+0000","last_active":"2023-10-09T04:29:21.894397+0000","last_peered":"2023-10-09T04:29:21.894397+0000","last_clean":"2023-10-09T04:29:21.894397+0000","last_became_active":"2023-10-09T04:26:21.347136+0000","last_became_peered":"2023-10-09T04:26:21.347136+0000","last_unstale":"2023-10-09T04:29:21.894397+0000","last_undegraded":"2023-10-09T04:29:21.894397+0000","last_fullsized":"2023-10-09T04:29:21.894397+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:52.721389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-09T04:29:18.148829+0000","last_change":"2023-10-09T04:24:56.183973+0000","last_active":"2023-10-09T04:29:18.148829+0000","last_peered":"2023-10-09T04:29:18.148829+0000","last_clean":"2023-10-09T04:29:18.148829+0000","last_became_active":"2023-10-09T04:24:56.181453+0000","last_became_peered":"2023-10-09T04:24:56.181453+0000","last_unstale":"2023-10-09T04:29:18.148829+0000","last_undegraded":"2023-10-09T04:29:18.148829+0000","last_fullsized":"2023-10-09T04:29:18.148829+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:30:35.829983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5304877080,"num_objects":80947,"num_object_clones":0,"num_object_copies":242841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80947,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128607,"num_write_kb":6705665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15971528704,"data_stored":15971516488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108307,"ondisk_log_size":108307,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":11614,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1836096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201724,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2874568,"kb_used_data":1970560,"kb_used_omap":0,"kb_used_meta":904000,"kb_avail":90870584,"statfs":{"total":95995035648,"available":93051478016,"internally_reserved":0,"allocated":2017853440,"data_stored":2016350311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925696000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":6.1619999999999999,"5min":2.6819999999999999,"15min":2.758},"min":{"1min":0.313,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":59.780000000000001,"5min":70.614000000000004,"15min":148.65899999999999},"last":1.23},{"interface":"front","average":{"1min":5.5549999999999997,"5min":2.8180000000000001,"15min":2.7469999999999999},"min":{"1min":0.247,"5min":0.23000000000000001,"15min":0.221},"max":{"1min":59.709000000000003,"5min":70.721999999999994,"15min":148.642},"last":1.0669999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:29:25 2023","interfaces":[{"interface":"back","average":{"1min":18.831,"5min":5.3570000000000002,"15min":6.5460000000000003},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.22600000000000001},"max":{"1min":213.41800000000001,"5min":213.41800000000001,"15min":261.935},"last":1.4079999999999999},{"interface":"front","average":{"1min":17.637,"5min":5.0460000000000003,"15min":6.3090000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":213.352,"5min":213.352,"15min":262.07600000000002},"last":1.7070000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":2.024,"15min":2.4790000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":7.734,"5min":66.328999999999994,"15min":253.41200000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.9350000000000001,"15min":2.8050000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.246},"max":{"1min":5.9100000000000001,"5min":66.314999999999998,"15min":253.39400000000001},"last":1.0960000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:29:25 2023","interfaces":[{"interface":"back","average":{"1min":6.1459999999999999,"5min":2.6600000000000001,"15min":3.1640000000000001},"min":{"1min":0.35899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":73.701999999999998,"5min":73.701999999999998,"15min":255.97300000000001},"last":1.155},{"interface":"front","average":{"1min":5.649,"5min":2.641,"15min":3.363},"min":{"1min":0.39300000000000002,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":73.603999999999999,"5min":73.603999999999999,"15min":256.06099999999998},"last":1.1439999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":0.91500000000000004,"15min":0.86699999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.3480000000000008,"5min":9.3480000000000008,"15min":9.3480000000000008},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.2490000000000001,"5min":0.94899999999999995,"15min":1.1970000000000001},"min":{"1min":0.28699999999999998,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":6.1299999999999999,"5min":6.1299999999999999,"15min":12.566000000000001},"last":0.375}]},{"osd":5,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.367,"15min":1.7909999999999999},"min":{"1min":0.35099999999999998,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":6.1440000000000001,"5min":17.882999999999999,"15min":120.066},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.764,"5min":1.2430000000000001,"15min":1.579},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":7.8360000000000003,"5min":17.568999999999999,"15min":120.09699999999999},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.25,"15min":1.335},"min":{"1min":0.31,"5min":0.31,"15min":0.307},"max":{"1min":9.5069999999999997,"5min":9.5069999999999997,"15min":9.5069999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.601,"5min":1.1040000000000001,"15min":1.036},"min":{"1min":0.34699999999999998,"5min":0.33200000000000002,"15min":0.30299999999999999},"max":{"1min":7.7850000000000001,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.436}]}]},{"osd":6,"up_from":47,"seq":201863463360,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3216568,"kb_used_data":2618608,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":90528584,"statfs":{"total":95995035648,"available":92701270016,"internally_reserved":0,"allocated":2681454592,"data_stored":2679966858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:53 2023","interfaces":[{"interface":"back","average":{"1min":19.07,"5min":5.2350000000000003,"15min":4.4900000000000002},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.193},"max":{"1min":162.524,"5min":162.524,"15min":162.524},"last":0.35099999999999998},{"interface":"front","average":{"1min":18.472999999999999,"5min":5.1420000000000003,"15min":4.4640000000000004},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":161.73599999999999,"5min":161.73599999999999,"15min":161.73599999999999},"last":1.2629999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:29:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":1.7010000000000001,"15min":5.1369999999999996},"min":{"1min":0.253,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":9.3810000000000002,"5min":18.971,"15min":315.73599999999999},"last":1.288},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.6399999999999999,"15min":5.1509999999999998},"min":{"1min":0.26400000000000001,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":8.5549999999999997,"5min":18.943999999999999,"15min":316.91500000000002},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:28:53 2023","interfaces":[{"interface":"back","average":{"1min":8.1120000000000001,"5min":4.6950000000000003,"15min":3.0609999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.23899999999999999},"max":{"1min":116.084,"5min":116.084,"15min":139.06899999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":8.1560000000000006,"5min":4.7460000000000004,"15min":3.1040000000000001},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":116.11199999999999,"5min":116.11199999999999,"15min":139.173},"last":0.81799999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:29:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.484,"15min":1.9299999999999999},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.25600000000000001},"max":{"1min":5.6669999999999998,"5min":14.083,"15min":43.475999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.0760000000000001,"5min":1.5800000000000001,"15min":1.873},"min":{"1min":0.36099999999999999,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":7.3520000000000003,"5min":14.052,"15min":43.369},"last":0.67400000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:29:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.0229999999999999,"15min":1.5309999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":6.6040000000000001,"5min":17.545000000000002,"15min":161.27500000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.091,"15min":1.6399999999999999},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.22600000000000001},"max":{"1min":9.4199999999999999,"5min":15.212999999999999,"15min":160.32599999999999},"last":2.202}]},{"osd":5,"last update":"Mon Oct 9 04:29:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.679,"15min":1.3129999999999999},"min":{"1min":0.30499999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.6340000000000003,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.849,"5min":1.73,"15min":1.333},"min":{"1min":0.311,"5min":0.31,"15min":0.27900000000000003},"max":{"1min":10.021000000000001,"5min":74.073999999999998,"15min":74.073999999999998},"last":1.208}]},{"osd":7,"last update":"Mon Oct 9 04:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.93999999999999995,"15min":1.6639999999999999},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.6030000000000002,"5min":5.9050000000000002,"15min":161.39400000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.84899999999999998,"15min":1.653},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":5.6520000000000001,"5min":5.6520000000000001,"15min":161.41200000000001},"last":1.2410000000000001}]}]},{"osd":1,"up_from":89,"seq":382252089669,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3521068,"kb_used_data":2671588,"kb_used_omap":9,"kb_used_meta":849462,"kb_avail":90224084,"statfs":{"total":95995035648,"available":92389462016,"internally_reserved":0,"allocated":2735706112,"data_stored":2733421674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":869850089},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":2.3759999999999999,"15min":4.3419999999999996},"min":{"1min":0.28699999999999998,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":5.6260000000000003,"5min":86.963999999999999,"15min":259.26799999999997},"last":0.93799999999999994},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.629,"15min":5.25},"min":{"1min":0.32500000000000001,"5min":0.222,"15min":0.19400000000000001},"max":{"1min":10.948,"5min":83.213999999999999,"15min":258.46899999999999},"last":0.70299999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.929,"15min":2.96},"min":{"1min":0.247,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.0600000000000005,"5min":74.474000000000004,"15min":259.315},"last":0.95899999999999996},{"interface":"front","average":{"1min":2.085,"5min":2.1179999999999999,"15min":3.3679999999999999},"min":{"1min":0.28000000000000003,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.2249999999999996,"5min":75.653000000000006,"15min":259.733},"last":0.755}]},{"osd":3,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.8959999999999999,"15min":2.3090000000000002},"min":{"1min":0.318,"5min":0.24099999999999999,"15min":0.22600000000000001},"max":{"1min":12.615,"5min":71.745999999999995,"15min":71.745999999999995},"last":35.740000000000002},{"interface":"front","average":{"1min":2.645,"5min":2.0049999999999999,"15min":2.7330000000000001},"min":{"1min":0.29199999999999998,"5min":0.27300000000000002,"15min":0.24299999999999999},"max":{"1min":12.565,"5min":75.613,"15min":75.613},"last":35.761000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.617,"15min":2.9169999999999998},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":10.975,"5min":39.052,"15min":257.83300000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.3740000000000001,"5min":1.3069999999999999,"15min":2.6080000000000001},"min":{"1min":0.41699999999999998,"5min":0.29199999999999998,"15min":0.28499999999999998},"max":{"1min":6.1230000000000002,"5min":39.040999999999997,"15min":257.863},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.167,"15min":2.0649999999999999},"min":{"1min":0.38600000000000001,"5min":0.29599999999999999,"15min":0.29099999999999998},"max":{"1min":10.903,"5min":10.903,"15min":38.848999999999997},"last":0.72899999999999998},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.26,"15min":2.1080000000000001},"min":{"1min":0.54300000000000004,"5min":0.32400000000000001,"15min":0.29799999999999999},"max":{"1min":13.316000000000001,"5min":13.316000000000001,"15min":39.593000000000004},"last":0.41999999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.647,"15min":1.8580000000000001},"min":{"1min":0.52200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":13.353,"5min":35.045999999999999,"15min":35.045999999999999},"last":0.443},{"interface":"front","average":{"1min":1.6180000000000001,"5min":1.54,"15min":1.6359999999999999},"min":{"1min":0.40799999999999997,"5min":0.36699999999999999,"15min":0.29999999999999999},"max":{"1min":4.1699999999999999,"5min":34.960999999999999,"15min":34.960999999999999},"last":0.88300000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":1.742,"15min":3.7400000000000002},"min":{"1min":0.499,"5min":0.35899999999999999,"15min":0.34799999999999998},"max":{"1min":7.5149999999999997,"5min":36.765999999999998,"15min":259.08499999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.716,"15min":3.4820000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33100000000000002},"max":{"1min":4.2800000000000002,"5min":36.75,"15min":259.13299999999998},"last":0.91100000000000003}]}]},{"osd":0,"up_from":84,"seq":360777253195,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2441156,"kb_used_data":1627452,"kb_used_omap":28,"kb_used_meta":813347,"kb_avail":91303996,"statfs":{"total":95995035648,"available":93495291904,"internally_reserved":0,"allocated":1666510848,"data_stored":1664083521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":832868176},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:29:28 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.2649999999999999,"15min":4.3520000000000003},"min":{"1min":0.189,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":10.178000000000001,"5min":10.178000000000001,"15min":209.215},"last":0.67600000000000005},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.339,"15min":4.2060000000000004},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":9.0860000000000003,"5min":16.131,"15min":120.934},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 04:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.4970000000000001,"15min":2.508},"min":{"1min":0.218,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":16.096,"5min":17.562999999999999,"15min":88.088999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.5960000000000001,"5min":1.526,"15min":2.8319999999999999},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.9659999999999993,"5min":9.9659999999999993,"15min":88.004999999999995},"last":1.1299999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:29:28 2023","interfaces":[{"interface":"back","average":{"1min":2.819,"5min":1.278,"15min":2.556},"min":{"1min":0.315,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":12.807,"5min":12.807,"15min":99.709000000000003},"last":0.80900000000000005},{"interface":"front","average":{"1min":3.1110000000000002,"5min":1.591,"15min":2.7250000000000001},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.177,"5min":12.292,"15min":99.560000000000002},"last":1.446}]},{"osd":4,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2829999999999999,"5min":2,"15min":2.504},"min":{"1min":0.38900000000000001,"5min":0.29799999999999999,"15min":0.28899999999999998},"max":{"1min":11.153,"5min":86.906999999999996,"15min":88.058000000000007},"last":5.5300000000000002},{"interface":"front","average":{"1min":2.9089999999999998,"5min":2.1240000000000001,"15min":2.4540000000000002},"min":{"1min":0.49099999999999999,"5min":0.28299999999999997,"15min":0.24199999999999999},"max":{"1min":9.8810000000000002,"5min":86.891000000000005,"15min":86.891000000000005},"last":0.40400000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":7.7919999999999998,"5min":3.4929999999999999,"15min":3.774},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":84.983000000000004,"5min":134.93000000000001,"15min":156.96100000000001},"last":2.8149999999999999},{"interface":"front","average":{"1min":6.9589999999999996,"5min":3.246,"15min":3.2629999999999999},"min":{"1min":0.42399999999999999,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":81.691999999999993,"5min":135.04599999999999,"15min":156.97300000000001},"last":6.391}]},{"osd":6,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":2.641,"5min":1.212,"15min":1.758},"min":{"1min":0.39300000000000002,"5min":0.34699999999999998,"15min":0.33400000000000002},"max":{"1min":7.0469999999999997,"5min":7.0469999999999997,"15min":63.468000000000004},"last":2.3399999999999999},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.3899999999999999,"15min":2.1640000000000001},"min":{"1min":0.438,"5min":0.376,"15min":0.34100000000000003},"max":{"1min":11.128,"5min":12.26,"15min":88.180000000000007},"last":5.6029999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":7.3419999999999996,"5min":2.2999999999999998,"15min":2.2360000000000002},"min":{"1min":0.42499999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":78.307000000000002,"5min":78.307000000000002,"15min":78.307000000000002},"last":1.504},{"interface":"front","average":{"1min":7.194,"5min":2.3580000000000001,"15min":2.649},"min":{"1min":0.436,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":78.216999999999999,"5min":78.216999999999999,"15min":88.120000000000005},"last":5.5709999999999997}]}]},{"osd":2,"up_from":94,"seq":403726926143,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2744912,"kb_used_data":1960136,"kb_used_omap":81,"kb_used_meta":784622,"kb_avail":91000240,"statfs":{"total":95995035648,"available":93184245760,"internally_reserved":0,"allocated":2007179264,"data_stored":2004859007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":803453146},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":8.2379999999999995,"5min":3.5049999999999999,"15min":3.8769999999999998},"min":{"1min":0.25,"5min":0.183,"15min":0.17699999999999999},"max":{"1min":132.87299999999999,"5min":132.87299999999999,"15min":132.87299999999999},"last":4.4850000000000003},{"interface":"front","average":{"1min":9.3670000000000009,"5min":3.8580000000000001,"15min":3.8330000000000002},"min":{"1min":0.249,"5min":0.216,"15min":0.20799999999999999},"max":{"1min":130.398,"5min":130.398,"15min":130.398},"last":0.188}]},{"osd":1,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":8.5839999999999996,"5min":3.5510000000000002,"15min":4.7430000000000003},"min":{"1min":0.254,"5min":0.224,"15min":0.191},"max":{"1min":122.17100000000001,"5min":122.17100000000001,"15min":158.13399999999999},"last":0.442},{"interface":"front","average":{"1min":7.3179999999999996,"5min":3.3039999999999998,"15min":4.8360000000000003},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.20799999999999999},"max":{"1min":121.91800000000001,"5min":121.91800000000001,"15min":161.12},"last":0.55700000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:29:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7470000000000001,"15min":2.4620000000000002},"min":{"1min":0.40400000000000003,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.9560000000000004,"5min":45.023000000000003,"15min":58.997},"last":1.3100000000000001},{"interface":"front","average":{"1min":2.2360000000000002,"5min":1.863,"15min":2.5899999999999999},"min":{"1min":0.38600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.1959999999999997,"5min":44.923999999999999,"15min":66.135000000000005},"last":6.9139999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":11.311,"5min":4.0970000000000004,"15min":2.8410000000000002},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":196.57599999999999,"5min":196.57599999999999,"15min":196.57599999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":10.571,"5min":3.036,"15min":2.3380000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":201.81100000000001,"5min":201.81100000000001,"15min":201.81100000000001},"last":1.361}]},{"osd":5,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":10.795999999999999,"5min":3.073,"15min":2.9569999999999999},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":190.06899999999999,"5min":190.06899999999999,"15min":190.06899999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":11.347,"5min":3.9409999999999998,"15min":3.286},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":190.08500000000001,"5min":190.08500000000001,"15min":190.08500000000001},"last":0.38100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.607,"15min":1.903},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":2.0329999999999999,"5min":58.469999999999999,"15min":58.469999999999999},"last":1.3400000000000001},{"interface":"front","average":{"1min":1.6879999999999999,"5min":2.0619999999999998,"15min":2.1669999999999998},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.26800000000000002},"max":{"1min":5.9710000000000001,"5min":58.067999999999998,"15min":58.067999999999998},"last":6.9530000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:28:41 2023","interfaces":[{"interface":"back","average":{"1min":9.9800000000000004,"5min":2.7970000000000002,"15min":2.726},"min":{"1min":0.35099999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":191.792,"5min":191.792,"15min":191.792},"last":3.4079999999999999},{"interface":"front","average":{"1min":10.778,"5min":3.9470000000000001,"15min":3.266},"min":{"1min":0.40400000000000003,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":191.828,"5min":191.828,"15min":191.828},"last":7.0259999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762617,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3331972,"kb_used_data":2275004,"kb_used_omap":7,"kb_used_meta":1056952,"kb_avail":90413180,"statfs":{"total":95995035648,"available":92583096320,"internally_reserved":0,"allocated":2329604096,"data_stored":2327298502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1082319464},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":9.7129999999999992,"5min":6.2549999999999999,"15min":4.7190000000000003},"min":{"1min":0.26700000000000002,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":119.149,"5min":119.97799999999999,"15min":125.765},"last":0.30099999999999999},{"interface":"front","average":{"1min":10.815,"5min":6.5519999999999996,"15min":5.2320000000000002},"min":{"1min":0.249,"5min":0.223,"15min":0.185},"max":{"1min":119.251,"5min":120.307,"15min":125.46899999999999},"last":0.26600000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":8.8420000000000005,"5min":2.7519999999999998,"15min":4.9939999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":121.842,"5min":121.842,"15min":186.21299999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":8.6959999999999997,"5min":2.5310000000000001,"15min":4.8010000000000002},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.221},"max":{"1min":119.673,"5min":119.673,"15min":191.78},"last":0.60199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.1499999999999999,"15min":2.452},"min":{"1min":0.217,"5min":0.217,"15min":0.187},"max":{"1min":2.9630000000000001,"5min":7.0590000000000002,"15min":127.599},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.9940000000000002,"5min":1.413,"15min":3.4199999999999999},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":10.24,"5min":10.24,"15min":129.98599999999999},"last":1.7789999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":7.7199999999999998,"5min":3.1890000000000001,"15min":3.0009999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25700000000000001},"max":{"1min":83.010000000000005,"5min":83.010000000000005,"15min":211.84700000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":6.181,"5min":2.8250000000000002,"15min":3.0550000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":82.915000000000006,"5min":82.915000000000006,"15min":211.17500000000001},"last":3.1739999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":7.069,"5min":2.3380000000000001,"15min":2.3799999999999999},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":88.295000000000002,"5min":88.295000000000002,"15min":88.295000000000002},"last":3.1000000000000001},{"interface":"front","average":{"1min":6.9269999999999996,"5min":2.3450000000000002,"15min":1.8740000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":86.968999999999994,"5min":86.968999999999994,"15min":86.968999999999994},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8600000000000001,"15min":1.633},"min":{"1min":0.36599999999999999,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":2.5489999999999999,"5min":64.378,"15min":64.378},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.7509999999999999,"5min":1.9790000000000001,"15min":1.9930000000000001},"min":{"1min":0.36599999999999999,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":8.0690000000000008,"5min":64.353999999999999,"15min":64.353999999999999},"last":0.46999999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":9.6120000000000001,"5min":3.496,"15min":3.2890000000000001},"min":{"1min":0.39600000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":86.897000000000006,"5min":86.897000000000006,"15min":212.62700000000001},"last":0.442},{"interface":"front","average":{"1min":7.7160000000000002,"5min":3.0960000000000001,"15min":3.3679999999999999},"min":{"1min":0.41399999999999998,"5min":0.376,"15min":0.33300000000000002},"max":{"1min":87.105999999999995,"5min":87.105999999999995,"15min":212.04599999999999},"last":0.624}]}]},{"osd":4,"up_from":31,"seq":133143986633,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2216256,"kb_used_data":1306232,"kb_used_omap":0,"kb_used_meta":910016,"kb_avail":91528896,"statfs":{"total":95995035648,"available":93725589504,"internally_reserved":0,"allocated":1337581568,"data_stored":1336085776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931856384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":7.1310000000000002,"15min":6.7389999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":30.488,"5min":207.45699999999999,"15min":207.45699999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":3.0920000000000001,"5min":7.5090000000000003,"15min":6.9219999999999997},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":30.5,"5min":207.114,"15min":207.114},"last":4.8959999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.5249999999999999,"15min":5.0529999999999999},"min":{"1min":0.251,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":7.7880000000000003,"5min":33.433999999999997,"15min":135.25399999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.573,"5min":1.3029999999999999,"15min":4.8899999999999997},"min":{"1min":0.27700000000000002,"5min":0.252,"15min":0.251},"max":{"1min":6.8970000000000002,"5min":32.683,"15min":134.44300000000001},"last":0.51900000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":5.3819999999999997,"15min":3.976},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":7.6840000000000002,"5min":203.61500000000001,"15min":203.61500000000001},"last":4.9249999999999998},{"interface":"front","average":{"1min":1.4450000000000001,"5min":5.3399999999999999,"15min":3.9580000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.9320000000000004,"5min":203.601,"15min":203.601},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":5.4720000000000004,"5min":2.3940000000000001,"15min":2.6659999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":85.003,"5min":85.003,"15min":154.13200000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":5.4669999999999996,"5min":2.5249999999999999,"15min":2.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":84.962000000000003,"5min":84.962000000000003,"15min":154.077},"last":5.0640000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:29:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.02,"15min":1.7749999999999999},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":16.753,"5min":16.753,"15min":111.809},"last":4.8499999999999996},{"interface":"front","average":{"1min":1.7130000000000001,"5min":0.98599999999999999,"15min":1.9650000000000001},"min":{"1min":0.28799999999999998,"5min":0.253,"15min":0.252},"max":{"1min":17.792000000000002,"5min":17.792000000000002,"15min":112.002},"last":2.7130000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.83699999999999997,"15min":1.667},"min":{"1min":0.311,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2679999999999998,"5min":10.039999999999999,"15min":76.307000000000002},"last":0.5},{"interface":"front","average":{"1min":1.206,"5min":0.877,"15min":1.5229999999999999},"min":{"1min":0.34499999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.843,"5min":6.843,"15min":78.421999999999997},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":0.873,"15min":1.153},"min":{"1min":0.317,"5min":0.309,"15min":0.26800000000000002},"max":{"1min":4.8609999999999998,"5min":5.0510000000000002,"15min":34.039000000000001},"last":4.9560000000000004},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.0900000000000001,"15min":1.361},"min":{"1min":0.30199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.0679999999999996,"5min":12.061,"15min":33.433999999999997},"last":9.8610000000000007}]}]},{"osd":5,"up_from":39,"seq":167503724997,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2201840,"kb_used_data":1309288,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":91543312,"statfs":{"total":95995035648,"available":93740351488,"internally_reserved":0,"allocated":1340710912,"data_stored":1339227710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":8.1240000000000006,"5min":5.7590000000000003,"15min":4.5999999999999996},"min":{"1min":0.34300000000000003,"5min":0.22800000000000001,"15min":0.193},"max":{"1min":127.627,"5min":139.86799999999999,"15min":139.86799999999999},"last":41.090000000000003},{"interface":"front","average":{"1min":7.7599999999999998,"5min":5.6779999999999999,"15min":4.7530000000000001},"min":{"1min":0.24299999999999999,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":128.066,"5min":139.84899999999999,"15min":139.84899999999999},"last":41.329999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:29:17 2023","interfaces":[{"interface":"back","average":{"1min":13.365,"5min":3.6589999999999998,"15min":5.5309999999999997},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":131.86199999999999,"5min":131.86199999999999,"15min":296.23099999999999},"last":41.784999999999997},{"interface":"front","average":{"1min":12.981,"5min":3.7930000000000001,"15min":5.7649999999999997},"min":{"1min":0.29799999999999999,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":132.74600000000001,"5min":132.74600000000001,"15min":295.065},"last":41.816000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":7.0300000000000002,"5min":2.504,"15min":3.7160000000000002},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":115.66500000000001,"5min":115.66500000000001,"15min":168.779},"last":0.38700000000000001},{"interface":"front","average":{"1min":7.423,"5min":3.0270000000000001,"15min":3.5710000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":116.881,"5min":116.881,"15min":168.03100000000001},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:29:17 2023","interfaces":[{"interface":"back","average":{"1min":8.2810000000000006,"5min":3.278,"15min":2.5979999999999999},"min":{"1min":0.497,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":116.56,"5min":116.56,"15min":116.56},"last":0.45200000000000001},{"interface":"front","average":{"1min":8.3160000000000007,"5min":3.1970000000000001,"15min":2.625},"min":{"1min":0.44900000000000001,"5min":0.29699999999999999,"15min":0.23499999999999999},"max":{"1min":116.527,"5min":116.527,"15min":116.527},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.821,"15min":1.387},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":3.9670000000000001,"5min":95.668000000000006,"15min":95.668000000000006},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.022,"5min":1.752,"15min":1.3320000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4649999999999999,"5min":98.703999999999994,"15min":98.703999999999994},"last":0.311}]},{"osd":6,"last update":"Mon Oct 9 04:29:17 2023","interfaces":[{"interface":"back","average":{"1min":7.6920000000000002,"5min":2.093,"15min":1.4019999999999999},"min":{"1min":0.34499999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":145.054,"5min":145.054,"15min":145.054},"last":0.29699999999999999},{"interface":"front","average":{"1min":7.9930000000000003,"5min":2.2869999999999999,"15min":1.54},"min":{"1min":0.34599999999999997,"5min":0.32500000000000001,"15min":0.307},"max":{"1min":144.85900000000001,"5min":144.85900000000001,"15min":144.85900000000001},"last":0.34000000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":0.85199999999999998,"15min":1.079},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25900000000000001},"max":{"1min":9.1839999999999993,"5min":9.1839999999999993,"15min":12.096},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.905,"5min":1.147,"15min":1.26},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":9.1430000000000007,"5min":11.522,"15min":11.522},"last":0.36899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1642332160,"data_stored":1642332160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2707226624,"data_stored":2707226624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1997737984,"data_stored":1997733912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2320629760,"data_stored":2320629760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1329205248,"data_stored":1329201176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1329332224,"data_stored":1329332224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2655195136,"data_stored":2655191064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1989869568,"data_stored":1989869568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:29:34.557 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:29:34.557 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:29:34.558 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [1, 5, 7, 3, 6, 0, 4, 2] 2023-10-09T04:29:34.558 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T04:29:34.576 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:34.709 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:34.936 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:34.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:35.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:35.071 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:35.274 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:35.450 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:35.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:35.521 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:35.531 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:35.534 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:35.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:35.569 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:35.669 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:35.749 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:35.779 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:35.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:35.893 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:35.926 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:36.011 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:36.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:36.119 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:36.218 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:36.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:36 smithi163 ceph-mon[221067]: pgmap v1408: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T04:29:36.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:36 smithi163 ceph-mon[213841]: pgmap v1408: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T04:29:36.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:36 smithi181 ceph-mon[196979]: pgmap v1408: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-09T04:29:36.770 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:36.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:36.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:36.915 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:37.019 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:37.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:37.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:37.159 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:37.223 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:37.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:37.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:37.417 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:37.951 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:37.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:38.060 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:38.106 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:38.162 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:38.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:38.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:38.269 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:38.368 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:38.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:38.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:38.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:38 smithi163 ceph-mon[221067]: pgmap v1409: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T04:29:38.859 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3043670537' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:38 smithi163 ceph-mon[213841]: pgmap v1409: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T04:29:38.859 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3043670537' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:29:38.908 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:29:38.909 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:38.991 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:39.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:38 smithi181 ceph-mon[196979]: pgmap v1409: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T04:29:39.273 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3043670537' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:29:39.406 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:39.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:39.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:39.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:39.567 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:39.624 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:39.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:39.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:39.761 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:39.784 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:39.784 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[213841]: failed to encode map e533 with expected crc 2023-10-09T04:29:39.784 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3043670537' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:29:39.784 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[213841]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T04:29:39.784 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3589325992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:39.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:39.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[221067]: failed to encode map e533 with expected crc 2023-10-09T04:29:39.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3043670537' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:29:39.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[221067]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T04:29:39.785 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3589325992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:39.795 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:39.842 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:39.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:39 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:39 smithi181 ceph-mon[196979]: failed to encode map e533 with expected crc 2023-10-09T04:29:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3043670537' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:29:39.847 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:39 smithi181 ceph-mon[196979]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T04:29:39.848 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3589325992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:39.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:39.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:39.884 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:39.930 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:39.950 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:39.962 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:39.995 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:39.999 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:40.037 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:40.075 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.076 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:40.080 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:40.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:40.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:40.189 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:40.270 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:40.411 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:40.411 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:40.590 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:40.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:40.703 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:40.745 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:41.022 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:40 smithi163 ceph-mon[213841]: pgmap v1411: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T04:29:41.022 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:29:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:29:40] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-10-09T04:29:41.023 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:40 smithi163 ceph-mon[221067]: pgmap v1411: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T04:29:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:40 smithi181 ceph-mon[196979]: pgmap v1411: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T04:29:41.629 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:29:41.833 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:41.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:41.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:41.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:41.993 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:42.053 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:42.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:42.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:42.198 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:42.274 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:42.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:42.384 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:42.439 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:42.467 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:42.614 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:42.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:42.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:42.729 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:42 smithi163 conmon[213817]: 2023-10-09T04:29:42.571+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e533 definitely_dead 0 2023-10-09T04:29:42.765 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:42.849 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:42.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:42.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:42.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:43.025 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:43.154 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.5. 2023-10-09T04:29:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:43 smithi163 ceph-mon[221067]: pgmap v1412: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-10-09T04:29:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1720284663' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T04:29:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:43 smithi163 ceph-mon[221067]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T04:29:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:43 smithi163 ceph-mon[213841]: pgmap v1412: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-10-09T04:29:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1720284663' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T04:29:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:43 smithi163 ceph-mon[213841]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T04:29:43.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:43 smithi181 ceph-mon[196979]: pgmap v1412: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-10-09T04:29:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1720284663' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T04:29:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:43 smithi181 ceph-mon[196979]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T04:29:43.729 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:29:43.739 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:29:44.286 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:44.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:44.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:44.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1720284663' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T04:29:44.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[221067]: osdmap e534: 8 total, 8 up, 7 in 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[221067]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[221067]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[221067]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1720284663' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[213841]: osdmap e534: 8 total, 8 up, 7 in 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[213841]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[213841]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:44 smithi163 ceph-mon[213841]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.416 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:44.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1720284663' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T04:29:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:44 smithi181 ceph-mon[196979]: osdmap e534: 8 total, 8 up, 7 in 2023-10-09T04:29:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:44 smithi181 ceph-mon[196979]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:44 smithi181 ceph-mon[196979]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:44 smithi181 ceph-mon[196979]: failed to encode map e534 with expected crc 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:44.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:44.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:44.833 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:44.876 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:44.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:44.888 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:44.962 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:44.962 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:44.981 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:45.012 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:45.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: pgmap v1414: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:29:45.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: osdmap e535: 8 total, 8 up, 7 in 2023-10-09T04:29:45.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.277 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[221067]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: pgmap v1414: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: osdmap e535: 8 total, 8 up, 7 in 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.279 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:45 smithi163 ceph-mon[213841]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: pgmap v1414: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: osdmap e535: 8 total, 8 up, 7 in 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:45 smithi181 ceph-mon[196979]: failed to encode map e535 with expected crc 2023-10-09T04:29:45.953 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:45.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:46.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:46.094 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:46.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:46.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:46.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:46.254 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:46.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 533'8438 2023-10-09T04:29:46.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: osdmap e536: 8 total, 8 up, 7 in 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 533'8514 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: 11.7 starting backfill to osd.0 from (0'0,0'0] MAX to 533'8617 2023-10-09T04:29:46.316 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[213841]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 533'8660 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 533'8438 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: osdmap e536: 8 total, 8 up, 7 in 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 533'8514 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: 11.7 starting backfill to osd.0 from (0'0,0'0] MAX to 533'8617 2023-10-09T04:29:46.317 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:46 smithi163 ceph-mon[221067]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 533'8660 2023-10-09T04:29:46.323 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:46.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 533'8438 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: osdmap e536: 8 total, 8 up, 7 in 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 533'8514 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: failed to encode map e536 with expected crc 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: 11.7 starting backfill to osd.0 from (0'0,0'0] MAX to 533'8617 2023-10-09T04:29:46.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:46 smithi181 ceph-mon[196979]: 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 533'8660 2023-10-09T04:29:46.844 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:46.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:46.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:46.960 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:46.993 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:47.070 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:47.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:47.176 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:47.205 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: pgmap v1417: 119 pgs: 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 3 remapped+peering, 106 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 930 op/s; 17602/214578 objects misplaced (8.203%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:47 smithi181 ceph-mon[196979]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: pgmap v1417: 119 pgs: 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 3 remapped+peering, 106 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 930 op/s; 17602/214578 objects misplaced (8.203%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.507 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[221067]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: pgmap v1417: 119 pgs: 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 3 remapped+peering, 106 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 930 op/s; 17602/214578 objects misplaced (8.203%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.508 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:47 smithi163 ceph-mon[213841]: failed to encode map e537 with expected crc 2023-10-09T04:29:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:47.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:47.610 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:47.707 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:47.882 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:47.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:47.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.007 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:48.083 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:48.279 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:48.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:48.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T04:29:48.337 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[213841]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.338 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.339 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:48 smithi163 ceph-mon[221067]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.389 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.491 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T04:29:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:48 smithi181 ceph-mon[196979]: failed to encode map e538 with expected crc 2023-10-09T04:29:48.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:48.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:48.734 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:48.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:48.772 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:48.807 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:48.810 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:48.857 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:48.877 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:48.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:48.900 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:48.934 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:48.969 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:48.984 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:49.017 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:49.018 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.019 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:29:49.072 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:49.194 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:49.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:49.304 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[213841]: pgmap v1420: 119 pgs: 4 peering, 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 3 remapped+peering, 102 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 607 op/s; 17602/212889 objects misplaced (8.268%); 2.1 MiB/s, 35 objects/s recovering 2023-10-09T04:29:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/683053139' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/900561001' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:49.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[221067]: pgmap v1420: 119 pgs: 4 peering, 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 3 remapped+peering, 102 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 607 op/s; 17602/212889 objects misplaced (8.268%); 2.1 MiB/s, 35 objects/s recovering 2023-10-09T04:29:49.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/683053139' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:49.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/900561001' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:49.337 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:49 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:49.348 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:49.409 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:49.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:49.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:49 smithi181 ceph-mon[196979]: pgmap v1420: 119 pgs: 4 peering, 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 3 remapped+peering, 102 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 607 op/s; 17602/212889 objects misplaced (8.268%); 2.1 MiB/s, 35 objects/s recovering 2023-10-09T04:29:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/683053139' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/900561001' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:49 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:49.541 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:49.546 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 3, 6, 0, 4, 2] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:29:49.546 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:29:49.769 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:29:50.489 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: failed to encode map e539 with expected crc 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: failed to encode map e539 with expected crc 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T04:29:50.490 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[221067]: pgmap v1422: 119 pgs: 4 peering, 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 2 remapped+peering, 103 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 482 op/s; 17602/212346 objects misplaced (8.289%); 1.7 MiB/s, 29 objects/s recovering 2023-10-09T04:29:50.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:50.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:50.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:50.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: failed to encode map e539 with expected crc 2023-10-09T04:29:50.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: failed to encode map e539 with expected crc 2023-10-09T04:29:50.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:50.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T04:29:50.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:50 smithi163 ceph-mon[213841]: pgmap v1422: 119 pgs: 4 peering, 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 2 remapped+peering, 103 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 482 op/s; 17602/212346 objects misplaced (8.289%); 1.7 MiB/s, 29 objects/s recovering 2023-10-09T04:29:50.754 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:29:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:29:50] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.43.0" 2023-10-09T04:29:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: failed to encode map e539 with expected crc 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: failed to encode map e539 with expected crc 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T04:29:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:50 smithi181 ceph-mon[196979]: pgmap v1422: 119 pgs: 4 peering, 1 activating+remapped, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+undersized+remapped, 2 remapped+peering, 103 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 8.5 KiB/s rd, 22 KiB/s wr, 482 op/s; 17602/212346 objects misplaced (8.289%); 1.7 MiB/s, 29 objects/s recovering 2023-10-09T04:29:51.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:51.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:51.151 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:51.192 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:51.263 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:51.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:51.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:51.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:51.421 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:51.480 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:51.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:51.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:51.589 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:51.631 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:51.701 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:51.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:51.754 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:51.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:51.840 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:51.892 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:51.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:51.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:51.914 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:29:51.914 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:29:51.914 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:29:51.914 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:29:51.998 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:52.030 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:52.110 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:52.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:52.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:52 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:29:52.201 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:52 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:29:52.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:52 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:29:52.202 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:52 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:29:52.223 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:52.304 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:52 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:29:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:52 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:29:53.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:29:53.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:29:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:53 smithi181 ceph-mon[196979]: pgmap v1423: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 15663/202794 objects misplaced (7.724%); 12 MiB/s, 189 objects/s recovering 2023-10-09T04:29:53.536 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:53 smithi163 ceph-mon[213841]: pgmap v1423: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 15663/202794 objects misplaced (7.724%); 12 MiB/s, 189 objects/s recovering 2023-10-09T04:29:53.537 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:53 smithi163 ceph-mon[221067]: pgmap v1423: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 15663/202794 objects misplaced (7.724%); 12 MiB/s, 189 objects/s recovering 2023-10-09T04:29:53.899 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:29:54.031 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:54.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:54.174 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:54.233 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:54.339 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:54.341 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:54.403 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:54.417 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:54.618 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:54.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:54.786 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:54.840 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:54.949 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:55.036 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:55.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:55 smithi163 ceph-mon[213841]: pgmap v1424: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 15663/202794 objects misplaced (7.724%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T04:29:55.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:55 smithi163 ceph-mon[221067]: pgmap v1424: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 15663/202794 objects misplaced (7.724%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T04:29:55.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:55 smithi181 ceph-mon[196979]: pgmap v1424: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 15663/202794 objects misplaced (7.724%); 8.9 MiB/s, 143 objects/s recovering 2023-10-09T04:29:55.797 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:55.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:55.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:55.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:55.999 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:56.262 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:56.315 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:29:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:56.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:56.417 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:56.481 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3984797417' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:29:56.491 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3984797417' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:29:56.492 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3984797417' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:29:57.403 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:57.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:57.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:57.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[221067]: pgmap v1425: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 13785/194145 objects misplaced (7.100%); 15 MiB/s, 243 objects/s recovering 2023-10-09T04:29:57.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[221067]: failed to encode map e540 with expected crc 2023-10-09T04:29:57.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3984797417' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:29:57.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[221067]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T04:29:57.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[221067]: failed to encode map e540 with expected crc 2023-10-09T04:29:57.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[213841]: pgmap v1425: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 13785/194145 objects misplaced (7.100%); 15 MiB/s, 243 objects/s recovering 2023-10-09T04:29:57.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[213841]: failed to encode map e540 with expected crc 2023-10-09T04:29:57.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3984797417' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:29:57.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[213841]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T04:29:57.416 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:57 smithi163 ceph-mon[213841]: failed to encode map e540 with expected crc 2023-10-09T04:29:57.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:57.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:57 smithi181 ceph-mon[196979]: pgmap v1425: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 13785/194145 objects misplaced (7.100%); 15 MiB/s, 243 objects/s recovering 2023-10-09T04:29:57.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:57 smithi181 ceph-mon[196979]: failed to encode map e540 with expected crc 2023-10-09T04:29:57.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3984797417' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:29:57.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:57 smithi181 ceph-mon[196979]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T04:29:57.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:57 smithi181 ceph-mon[196979]: failed to encode map e540 with expected crc 2023-10-09T04:29:57.596 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:57.753 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:29:57.779 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:57.804 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:57.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:57.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:57.911 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:58.033 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:58.284 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:58.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:58.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:58.435 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:58.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:58.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:58.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:58.635 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:58.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:58.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:58.732 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:29:58.770 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:29:58.800 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:29:58.803 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:58.839 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:29:58.897 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:29:58.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:58 smithi163 ceph-mon[213841]: pgmap v1427: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 13785/191565 objects misplaced (7.196%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:29:58.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2487538135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:58.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:58 smithi163 ceph-mon[221067]: pgmap v1427: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 13785/191565 objects misplaced (7.196%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:29:58.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2487538135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:58.913 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:58.942 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:29:58.986 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:29:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:58 smithi181 ceph-mon[196979]: pgmap v1427: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 13785/191565 objects misplaced (7.196%); 16 MiB/s, 260 objects/s recovering 2023-10-09T04:29:59.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2487538135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:59.037 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:29:59.048 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.048 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.048 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.048 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.048 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.048 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.049 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.049 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:59.088 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:59.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:59.199 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:59.267 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:59.792 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:30:00.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:29:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3314837735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:00.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:29:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3314837735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:00.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:29:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3314837735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: failed to encode map e541 with expected crc 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: failed to encode map e541 with expected crc 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3314837735' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: overall HEALTH_WARN noscrub flag(s) set 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: pgmap v1429: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 13785/190947 objects misplaced (7.219%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:30:00.942 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: failed to encode map e541 with expected crc 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: failed to encode map e541 with expected crc 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3314837735' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: overall HEALTH_WARN noscrub flag(s) set 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: pgmap v1429: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 13785/190947 objects misplaced (7.219%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:30:00.943 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:00.944 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:30:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:30:00] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.43.0" 2023-10-09T04:30:01.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: failed to encode map e541 with expected crc 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: failed to encode map e541 with expected crc 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3314837735' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: overall HEALTH_WARN noscrub flag(s) set 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: pgmap v1429: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 13785/190947 objects misplaced (7.219%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:30:01.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:01.516 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:01.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:01.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:01.682 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:01.754 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:01.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:01.790 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:01.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:01.910 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:01.975 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:01.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:01.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:02.079 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:02.193 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:02.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:02.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:02.458 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:02.458 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:02.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:02.530 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:02.745 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:02.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:02.855 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:02.908 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:02.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:02.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:03.086 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:03.123 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.187 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:03.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:03 smithi181 ceph-mon[196979]: pgmap v1430: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 887 op/s; 12060/181482 objects misplaced (6.645%); 16 MiB/s, 259 objects/s recovering 2023-10-09T04:30:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:03.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:03.296 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:03.358 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.423 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:03 smithi163 ceph-mon[213841]: pgmap v1430: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 887 op/s; 12060/181482 objects misplaced (6.645%); 16 MiB/s, 259 objects/s recovering 2023-10-09T04:30:03.423 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:03 smithi163 ceph-mon[221067]: pgmap v1430: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 887 op/s; 12060/181482 objects misplaced (6.645%); 16 MiB/s, 259 objects/s recovering 2023-10-09T04:30:03.437 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:03.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:03.540 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:30:03.543 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:03.678 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:30:04.249 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:04.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:04.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:30:04.361 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:04.459 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:04.622 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:04.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:04.633 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:04.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:04.762 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:04.815 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:04.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:04.923 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:04.961 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:04.995 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:05.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:05.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:05.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:05.156 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:05.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:05.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:05.320 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:05.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:05 smithi163 ceph-mon[213841]: pgmap v1431: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 12060/181482 objects misplaced (6.645%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:30:05.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:05.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:05.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:05 smithi163 ceph-mon[221067]: pgmap v1431: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 12060/181482 objects misplaced (6.645%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:30:05.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:05.336 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:05.416 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:05.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:05 smithi181 ceph-mon[196979]: pgmap v1431: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 12060/181482 objects misplaced (6.645%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:30:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:06.189 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:06.227 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:30:06.251 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:06.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:06.255 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:06 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2912708825' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:30:06.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:06.255 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:06 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2912708825' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:30:06.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:06.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:06.357 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:06.429 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:06.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:06.522 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:06 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2912708825' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:30:07.292 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:07.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[221067]: pgmap v1432: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 10256/173046 objects misplaced (5.927%); 14 MiB/s, 218 objects/s recovering 2023-10-09T04:30:07.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[221067]: failed to encode map e542 with expected crc 2023-10-09T04:30:07.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2912708825' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:30:07.295 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[221067]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T04:30:07.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[213841]: pgmap v1432: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 10256/173046 objects misplaced (5.927%); 14 MiB/s, 218 objects/s recovering 2023-10-09T04:30:07.295 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[213841]: failed to encode map e542 with expected crc 2023-10-09T04:30:07.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2912708825' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:30:07.296 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:07 smithi163 ceph-mon[213841]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T04:30:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:07.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:07.397 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:07.480 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:07.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:07 smithi181 ceph-mon[196979]: pgmap v1432: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 10256/173046 objects misplaced (5.927%); 14 MiB/s, 218 objects/s recovering 2023-10-09T04:30:07.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:07 smithi181 ceph-mon[196979]: failed to encode map e542 with expected crc 2023-10-09T04:30:07.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2912708825' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:30:07.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:07 smithi181 ceph-mon[196979]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T04:30:07.935 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:07.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:08.040 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:08.164 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:08.314 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:30:08.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:08.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:08.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:08.467 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:08.607 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:09.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:08 smithi163 ceph-mon[213841]: pgmap v1434: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 10256/170223 objects misplaced (6.025%); 16 MiB/s, 259 objects/s recovering 2023-10-09T04:30:09.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:08 smithi163 ceph-mon[221067]: pgmap v1434: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 10256/170223 objects misplaced (6.025%); 16 MiB/s, 259 objects/s recovering 2023-10-09T04:30:09.275 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:08 smithi181 ceph-mon[196979]: pgmap v1434: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 10256/170223 objects misplaced (6.025%); 16 MiB/s, 259 objects/s recovering 2023-10-09T04:30:09.643 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:09.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:09.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:09.798 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:09.867 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:09.874 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:30:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:09.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:09.926 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:30:09.968 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:30:09.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:10.008 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:30:10.025 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:10.051 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:30:10.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:10.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:10.128 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:30:10.179 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:30:10.181 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:10.219 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:30:10.259 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:10.264 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.264 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.264 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.265 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:10.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:10.545 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:10.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/152616519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:10.546 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[213841]: pgmap v1435: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 10256/169515 objects misplaced (6.050%); 13 MiB/s, 214 objects/s recovering 2023-10-09T04:30:10.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:10.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:10.634 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:30:10.634 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:30:10.634 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:30:10.634 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:30:10.634 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.634 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18926 10.5 10:a0371204:::benchmark_data_smithi163_523108_object18925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e388)", 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 464.65333518300002, 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0791226709999999, 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18926 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.635 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977986+0000", 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.636 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.977985+0000", 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978010+0000", 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5281000000000001e-05 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978022+0000", 2023-10-09T04:30:10.637 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1691e-05 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978029+0000", 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.6140000000000003e-06 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978125+0000", 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5649000000000005e-05 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.638 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978259+0000", 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013442700000000001 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.978399+0000", 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013955200000000001 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.639 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.979678+0000", 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001279362 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.980540+0000", 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00086218799999999995 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.640 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.007748+0000", 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.027207293 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.007761+0000", 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3307e-05 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979759+0000", 2023-10-09T04:30:10.641 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97199803900000004 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979772+0000", 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2935999999999999e-05 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057024+0000", 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.07725206 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.642 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057109+0000", 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4544999999999998e-05 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18936 10.8 10:1447a75e:::benchmark_data_smithi163_523108_object18935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 464.63223540799999, 2023-10-09T04:30:10.643 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0663146870000002, 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18936 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:30:10.644 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999086+0000", 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999084+0000", 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999108+0000", 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4119000000000001e-05 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999119+0000", 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0609e-05 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:25.999129+0000", 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0173999999999999e-05 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.005396+0000", 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062664510000000001 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.005408+0000", 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2614e-05 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.978908+0000", 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.97349999099999995 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.978922+0000", 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3569e-05 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.058726+0000", 2023-10-09T04:30:10.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.079804481 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.059470+0000", 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00074404699999999996 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.059625+0000", 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015495099999999999 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060888+0000", 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0012629589999999999 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.061724+0000", 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00083528300000000003 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.064344+0000", 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0026200139999999999 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.064384+0000", 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0055999999999997e-05 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.065400+0000", 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0010168359999999999 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:18951 10.5 10:a56ce1b4:::benchmark_data_smithi163_523108_object18950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e389)", 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 464.613129399, 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0484555979999999, 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 18951 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018192+0000", 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018190+0000", 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018218+0000", 2023-10-09T04:30:10.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7906e-05 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018235+0000", 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7246999999999999e-05 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018245+0000", 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.465e-06 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018303+0000", 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8554999999999998e-05 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.018315+0000", 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2088e-05 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979816+0000", 2023-10-09T04:30:10.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96150034200000001 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:26.979836+0000", 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0537e-05 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057168+0000", 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.077331303 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057311+0000", 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014343400000000001 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.057491+0000", 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000179817 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060562+0000", 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030713649999999999 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.060602+0000", 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.9830999999999999e-05 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.065968+0000", 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053659409999999999 2023-10-09T04:30:10.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.066023+0000", 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5339e-05 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:28.066647+0000", 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00062404699999999997 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:38490 10.5 10:a06fa733:::benchmark_data_smithi163_523108_object38489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 445.562594891, 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0132891260000001, 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 38490 2023-10-09T04:30:10.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068726+0000", 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068746+0000", 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0443000000000001e-05 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068753+0000", 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9829999999999999e-06 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.068759+0000", 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5550000000000002e-06 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.077390+0000", 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086309060000000007 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:45.077400+0000", 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0161e-05 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075149+0000", 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.99774961600000001 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075307+0000", 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015781200000000001 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.075470+0000", 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016307 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080366+0000", 2023-10-09T04:30:10.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048961869999999998 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.080409+0000", 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2290999999999999e-05 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081931+0000", 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015225309999999999 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.081989+0000", 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7714999999999999e-05 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:22:46.082015+0000", 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6398999999999999e-05 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57312 10.8 10:143e0015:::benchmark_data_smithi163_523108_object57311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 424.29293590600003, 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.136295909, 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.669 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57312 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338385+0000", 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338384+0000", 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338414+0000", 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0681e-05 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338431+0000", 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6909999999999999e-05 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338443+0000", 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1537e-05 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342061+0000", 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0036182689999999999 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342072+0000", 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1219999999999999e-05 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.464257+0000", 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.122184904 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:30:10.673 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465066+0000", 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080893800000000004 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474628+0000", 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0095617159999999996 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474681+0000", 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3440000000000003e-05 2023-10-09T04:30:10.674 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57318 10.8 10:1fa756a5:::benchmark_data_smithi163_523108_object57317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 424.29286269300002, 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1361408630000001, 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.675 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57318 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338458+0000", 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338484+0000", 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6522000000000001e-05 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338493+0000", 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6600000000000001e-06 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338499+0000", 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.285e-06 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342095+0000", 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035956619999999999 2023-10-09T04:30:10.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.342105+0000", 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5959999999999993e-06 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465152+0000", 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.123046999 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for degraded object", 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.465958+0000", 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00080678800000000001 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474519+0000", 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0085601770000000004 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.474599+0000", 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0765000000000003e-05 2023-10-09T04:30:10.680 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57319 10.a 10:5ee00c90:::benchmark_data_smithi163_523108_object57318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 424.29281652399999, 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1486770740000001, 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.681 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57319 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338505+0000", 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338504+0000", 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338532+0000", 2023-10-09T04:30:10.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.7617999999999999e-05 2023-10-09T04:30:10.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338540+0000", 2023-10-09T04:30:10.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0949999999999996e-06 2023-10-09T04:30:10.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.338545+0000", 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4469999999999997e-06 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.339505+0000", 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00095972899999999999 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.339519+0000", 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3448000000000001e-05 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470780+0000", 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.131261257 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.470931+0000", 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015102200000000001 2023-10-09T04:30:10.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.471097+0000", 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016623699999999999 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.478095+0000", 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069980989999999998 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.478878+0000", 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00078291299999999999 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.481861+0000", 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029827709999999999 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.486996+0000", 2023-10-09T04:30:10.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051355569999999998 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.487059+0000", 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2792999999999993e-05 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.487182+0000", 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012257699999999999 2023-10-09T04:30:10.690 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:57338 10.5 10:a63760e3:::benchmark_data_smithi163_523108_object57337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 424.14610107999999, 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0231315560000001, 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.691 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57338 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485220+0000", 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485218+0000", 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485241+0000", 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2555e-05 2023-10-09T04:30:10.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485260+0000", 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8814e-05 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485267+0000", 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5599999999999996e-06 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485316+0000", 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8528e-05 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:06.485332+0000", 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6419e-05 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.495704+0000", 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.010371613 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.495862+0000", 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000157911 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.496027+0000", 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00016513000000000001 2023-10-09T04:30:10.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.505438+0000", 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0094115150000000005 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.507674+0000", 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0022358320000000001 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.507710+0000", 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5670999999999998e-05 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508174+0000", 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00046372900000000002 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508224+0000", 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0794000000000002e-05 2023-10-09T04:30:10.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:23:07.508352+0000", 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127293 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83363 10.5 10:a7ea1084:::benchmark_data_smithi163_523108_object9347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.361999+0000", 2023-10-09T04:30:10.699 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 349.26932203500002, 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1263849750000001, 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83363 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.361999+0000", 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.361999+0000", 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.361997+0000", 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362009+0000", 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1888e-05 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362032+0000", 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3439999999999999e-05 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362043+0000", 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0873000000000001e-05 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362103+0000", 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9784999999999998e-05 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362285+0000", 2023-10-09T04:30:10.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018241500000000001 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.362474+0000", 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000188996 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.364221+0000", 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001746538 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.364375+0000", 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015463299999999999 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.366914+0000", 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025380670000000002 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430708+0000", 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.063794540999999996 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430727+0000", 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9128e-05 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488297+0000", 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.057569824 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488384+0000", 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7097999999999999e-05 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83370 10.15 10:a9c25298:::benchmark_data_smithi163_523108_object9354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.383272+0000", 2023-10-09T04:30:10.707 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 349.24804934100001, 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.097316261, 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83370 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383272+0000", 2023-10-09T04:30:10.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383272+0000", 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383271+0000", 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383276+0000", 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.9679999999999997e-06 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383285+0000", 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8589999999999994e-06 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383290+0000", 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2809999999999999e-06 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383337+0000", 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7315999999999999e-05 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383467+0000", 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012938199999999999 2023-10-09T04:30:10.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.383564+0000", 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7003999999999995e-05 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.385231+0000", 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0016668589999999999 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.385448+0000", 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021762999999999999 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.385614+0000", 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165353 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430659+0000", 2023-10-09T04:30:10.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.045045307 2023-10-09T04:30:10.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.714 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:10.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.430673+0000", 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4246999999999999e-05 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.480490+0000", 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0498168329999999 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.480588+0000", 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8188000000000002e-05 2023-10-09T04:30:10.716 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83385 10.15 10:ac2c79ee:::benchmark_data_smithi163_523108_object9369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.444437+0000", 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 349.18688435799999, 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.099660825, 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.717 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83385 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444437+0000", 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.718 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444437+0000", 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444435+0000", 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444444+0000", 2023-10-09T04:30:10.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8049999999999996e-06 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444464+0000", 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0163999999999999e-05 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444475+0000", 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0655999999999999e-05 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444536+0000", 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1309000000000005e-05 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.444550+0000", 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4188e-05 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.480981+0000", 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.036431117 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.481113+0000", 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013171600000000001 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.481247+0000", 2023-10-09T04:30:10.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013369599999999999 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488039+0000", 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067923580000000001 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488120+0000", 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0903999999999997e-05 2023-10-09T04:30:10.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488205+0000", 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5637999999999996e-05 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.543917+0000", 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.055711636000000002 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.543976+0000", 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8547e-05 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.544098+0000", 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012199199999999999 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.725 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:83399 10.15 10:a8a18d93:::benchmark_data_smithi163_523108_object9383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:24:21.515179+0000", 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 349.11614202599998, 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.032424445, 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 83399 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515179+0000", 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515179+0000", 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.727 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515178+0000", 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515184+0000", 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.6899999999999997e-06 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515192+0000", 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.6840000000000002e-06 2023-10-09T04:30:10.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515199+0000", 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6610000000000003e-06 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515251+0000", 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2537999999999997e-05 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:21.515260+0000", 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4149999999999999e-06 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.482834+0000", 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.96757368700000002 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.482968+0000", 2023-10-09T04:30:10.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000134244 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.483092+0000", 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000124311 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488047+0000", 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049550740000000003 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488168+0000", 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000120597 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.488256+0000", 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8319999999999995e-05 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.547435+0000", 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.059178808999999999 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.547493+0000", 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8366000000000003e-05 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:24:22.547604+0000", 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000110287 2023-10-09T04:30:10.733 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116378 10.e 10:7321e9ff:::benchmark_data_smithi163_523108_object42362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 241.96487413400001, 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.099838952, 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.734 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116378 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666447+0000", 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666450+0000", 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.14e-06 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666456+0000", 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9830000000000001e-06 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.666460+0000", 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.668e-06 2023-10-09T04:30:10.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.672764+0000", 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.006303594 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.672913+0000", 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014891500000000001 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.673054+0000", 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014072499999999999 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.680949+0000", 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0078951679999999993 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.685946+0000", 2023-10-09T04:30:10.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0049966689999999996 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701182+0000", 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015236826 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701198+0000", 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5494000000000001e-05 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766190+0000", 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0649925410000001 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766286+0000", 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.5639999999999999e-05 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116382 10.e 10:746dc59f:::benchmark_data_smithi163_523108_object42366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.670407+0000", 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 241.96091442100001, 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0964046039999999, 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.742 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116382 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670407+0000", 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670407+0000", 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670405+0000", 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670415+0000", 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0526000000000001e-05 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670432+0000", 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6578999999999998e-05 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.670441+0000", 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4690000000000003e-06 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676668+0000", 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0062271150000000001 2023-10-09T04:30:10.745 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676858+0000", 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00018957699999999999 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.677005+0000", 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014689999999999999 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.681628+0000", 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0046230899999999998 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.685998+0000", 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00437039 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701238+0000", 2023-10-09T04:30:10.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.015239571 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701247+0000", 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4429999999999992e-06 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766739+0000", 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0654917580000001 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.766811+0000", 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2170999999999997e-05 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116384 10.8 10:10a2b316:::benchmark_data_smithi163_523108_object42368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.676857+0000", 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 241.954464616, 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0946857999999999, 2023-10-09T04:30:10.749 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116384 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676857+0000", 2023-10-09T04:30:10.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676857+0000", 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676855+0000", 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.751 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676860+0000", 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.51e-06 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676868+0000", 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.5439999999999999e-06 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676875+0000", 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4760000000000003e-06 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.676963+0000", 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7641999999999999e-05 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.677085+0000", 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012232700000000001 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.677206+0000", 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012082999999999999 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.679648+0000", 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024424469999999999 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.682902+0000", 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003254112 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701396+0000", 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.01849375 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.701407+0000", 2023-10-09T04:30:10.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0718e-05 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.771443+0000", 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0700367690000001 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.771542+0000", 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8873e-05 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.756 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116395 10.18 10:1a07f0f6:::benchmark_data_smithi163_523108_object42379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.718986+0000", 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 241.91233511600001, 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0974267150000001, 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.757 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116395 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718986+0000", 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718986+0000", 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718984+0000", 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.718995+0000", 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0963e-05 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719018+0000", 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3388999999999999e-05 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719029+0000", 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0668e-05 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719106+0000", 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7255999999999994e-05 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.719121+0000", 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.4649e-05 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779188+0000", 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0600670510000001 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779314+0000", 2023-10-09T04:30:10.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012573400000000001 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779443+0000", 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012905499999999999 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.815890+0000", 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.036447307999999998 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.815901+0000", 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0958000000000001e-05 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816203+0000", 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00030209200000000001 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816232+0000", 2023-10-09T04:30:10.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8938999999999999e-05 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816272+0000", 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0194000000000002e-05 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816413+0000", 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000140457 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.764 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116396 10.18 10:1aeefe62:::benchmark_data_smithi163_523108_object42380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.721996+0000", 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 241.90932544, 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0999008619999999, 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116396 2023-10-09T04:30:10.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.721996+0000", 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.721996+0000", 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.721994+0000", 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722004+0000", 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0215e-05 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722026+0000", 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1446000000000001e-05 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722034+0000", 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.4430000000000003e-06 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722094+0000", 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9790999999999997e-05 2023-10-09T04:30:10.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.722108+0000", 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.416e-05 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.781401+0000", 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0592935349999999 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.781533+0000", 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013104699999999999 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.781660+0000", 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012787000000000001 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.815940+0000", 2023-10-09T04:30:10.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.034279653 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816453+0000", 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00051292099999999995 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.816487+0000", 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3652000000000002e-05 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.821795+0000", 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0053085459999999999 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.821830+0000", 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4980000000000001e-05 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.821897+0000", 2023-10-09T04:30:10.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6478999999999998e-05 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:116399 10.8 10:127a1cac:::benchmark_data_smithi163_523108_object42383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:08.729156+0000", 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 241.90216529400001, 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.05177231, 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.773 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 116399 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729156+0000", 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.774 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729156+0000", 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729153+0000", 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999971 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.775 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729166+0000", 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.29e-05 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729191+0000", 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.5306000000000001e-05 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729203+0000", 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1882000000000001e-05 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.776 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729281+0000", 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.7956999999999999e-05 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:08.729299+0000", 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8009e-05 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.777 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.771878+0000", 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0425782130000001 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.772010+0000", 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013244900000000001 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.772131+0000", 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121487 2023-10-09T04:30:10.778 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779101+0000", 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0069697759999999996 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779135+0000", 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.4143999999999997e-05 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.779 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.779167+0000", 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1587000000000001e-05 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.780705+0000", 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0015378309999999999 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.780772+0000", 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.7465000000000005e-05 2023-10-09T04:30:10.780 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:09.780928+0000", 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015596100000000001 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:120723 10.8 10:160fe9bf:::benchmark_data_smithi163_523108_object46707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:20.337171+0000", 2023-10-09T04:30:10.781 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 230.29415041499999, 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0261010859999999, 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "reached pg", 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 120723 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.782 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337171+0000", 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337171+0000", 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337169+0000", 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.783 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337177+0000", 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5129999999999999e-06 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337190+0000", 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3644e-05 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.784 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337199+0000", 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.1110000000000008e-06 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337249+0000", 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0102000000000002e-05 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337420+0000", 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017113799999999999 2023-10-09T04:30:10.785 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.337547+0000", 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000127572 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.339814+0000", 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00226677 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.786 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.340029+0000", 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00021489799999999999 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553248+0000", 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.213219086 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.787 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553264+0000", 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.5498999999999998e-05 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.363189+0000", 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.80992576299999997 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.363272+0000", 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2527000000000005e-05 2023-10-09T04:30:10.788 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46073.0:120726 10.15 10:a8ac2898:::benchmark_data_smithi163_523108_object46710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:26:20.352749+0000", 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 230.27857224499999, 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0461038250000001, 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:10.789 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46073", 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 120726 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352749+0000", 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:10.790 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352749+0000", 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352747+0000", 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.352755+0000", 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.9699999999999999e-06 2023-10-09T04:30:10.791 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553214+0000", 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.20045837799999999 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.553225+0000", 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1096e-05 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.792 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.557694+0000", 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044695400000000001 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for peered", 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:20.557706+0000", 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.207e-05 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.378627+0000", 2023-10-09T04:30:10.793 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.82092094299999996 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.378725+0000", 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7714000000000007e-05 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,3,7", 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.378826+0000", 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000101026 2023-10-09T04:30:10.794 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.382312+0000", 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0034861559999999998 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.383401+0000", 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001088889 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.795 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.384130+0000", 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00072926299999999996 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.398699+0000", 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.014568129000000001 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:10.796 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.398758+0000", 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9379000000000002e-05 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:26:21.398853+0000", 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4778999999999995e-05 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:10.797 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:30:10.798 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:10.816 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:10.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:10.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:10.827 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:10.827 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:10 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:10.827 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/152616519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:10.828 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:10 smithi181 ceph-mon[196979]: pgmap v1435: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 10256/169515 objects misplaced (6.050%); 13 MiB/s, 214 objects/s recovering 2023-10-09T04:30:10.833 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:30:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:30:10] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.43.0" 2023-10-09T04:30:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/152616519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:10.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:10 smithi163 ceph-mon[221067]: pgmap v1435: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 10256/169515 objects misplaced (6.050%); 13 MiB/s, 214 objects/s recovering 2023-10-09T04:30:10.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:10.994 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:11.603 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:30:11.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4210922567' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:11.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:11 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:11.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4210922567' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:11.833 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:11 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4210922567' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:12.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:11 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:12.063 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:12.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:12.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:12.249 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:12.864 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:12.888 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:30:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[213841]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T04:30:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[213841]: failed to encode map e543 with expected crc 2023-10-09T04:30:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[213841]: failed to encode map e543 with expected crc 2023-10-09T04:30:12.889 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[213841]: pgmap v1437: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 8756/159561 objects misplaced (5.488%); 16 MiB/s, 251 objects/s recovering 2023-10-09T04:30:12.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:12.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:12.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[221067]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T04:30:12.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[221067]: failed to encode map e543 with expected crc 2023-10-09T04:30:12.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[221067]: failed to encode map e543 with expected crc 2023-10-09T04:30:12.890 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:12 smithi163 ceph-mon[221067]: pgmap v1437: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 8756/159561 objects misplaced (5.488%); 16 MiB/s, 251 objects/s recovering 2023-10-09T04:30:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:12 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:12 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:13.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:12 smithi181 ceph-mon[196979]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T04:30:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:12 smithi181 ceph-mon[196979]: failed to encode map e543 with expected crc 2023-10-09T04:30:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:12 smithi181 ceph-mon[196979]: failed to encode map e543 with expected crc 2023-10-09T04:30:13.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:12 smithi181 ceph-mon[196979]: pgmap v1437: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 8756/159561 objects misplaced (5.488%); 16 MiB/s, 251 objects/s recovering 2023-10-09T04:30:13.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:13.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:13.277 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:13.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:13.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:13.459 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:13.541 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:13.644 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:13.760 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:13.793 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:13.860 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:13.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:13.966 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:14.037 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:14.495 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:14.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:14.604 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:14.622 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:14.640 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:14.708 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:14.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:14.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:14.818 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:14.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:14.894 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:15.013 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:15.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:15.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:30:15.125 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:15.174 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:15 smithi181 ceph-mon[196979]: pgmap v1438: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8756/159561 objects misplaced (5.488%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T04:30:15.544 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:15.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:15.549 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:15 smithi163 ceph-mon[221067]: pgmap v1438: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8756/159561 objects misplaced (5.488%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T04:30:15.549 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:15 smithi163 ceph-mon[213841]: pgmap v1438: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 8756/159561 objects misplaced (5.488%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T04:30:15.649 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:15.693 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:15.767 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:15.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:15.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:15.930 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:15.999 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:16.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:16.132 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:16.155 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:30:16.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1888788688' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:30:16.176 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:16 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:30:16.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1888788688' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:30:16.177 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:16 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:30:16.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1888788688' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:30:16.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:16 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:30:16.580 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:16.684 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:16.714 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:16.787 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:16.902 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.937 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.044 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:17.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:17.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:17.171 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[221067]: pgmap v1439: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 7245/152058 objects misplaced (4.765%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[221067]: failed to encode map e544 with expected crc 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[221067]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[221067]: failed to encode map e544 with expected crc 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[213841]: pgmap v1439: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 7245/152058 objects misplaced (4.765%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:30:17.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[213841]: failed to encode map e544 with expected crc 2023-10-09T04:30:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:30:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[213841]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T04:30:17.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:17 smithi163 ceph-mon[213841]: failed to encode map e544 with expected crc 2023-10-09T04:30:17.227 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:17 smithi181 ceph-mon[196979]: pgmap v1439: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 7245/152058 objects misplaced (4.765%); 12 MiB/s, 191 objects/s recovering 2023-10-09T04:30:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:17 smithi181 ceph-mon[196979]: failed to encode map e544 with expected crc 2023-10-09T04:30:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:17 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:30:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:17 smithi181 ceph-mon[196979]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T04:30:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:17 smithi181 ceph-mon[196979]: failed to encode map e544 with expected crc 2023-10-09T04:30:18.342 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:18.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:18.421 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:18.446 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[213841]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[213841]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[213841]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[213841]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[213841]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[221067]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[221067]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[221067]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[221067]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.447 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:18 smithi163 ceph-mon[221067]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.448 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:18.456 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:30:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:18 smithi181 ceph-mon[196979]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:18 smithi181 ceph-mon[196979]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:18 smithi181 ceph-mon[196979]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T04:30:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:18 smithi181 ceph-mon[196979]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:18 smithi181 ceph-mon[196979]: failed to encode map e545 with expected crc 2023-10-09T04:30:18.563 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:161268 11.8 11:12e21800:::benchmark_data_smithi163_601242_object56489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:18.596522+0000", 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.043705239, 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.043740268999999998, 2023-10-09T04:30:18.645 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 161268 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596522+0000", 2023-10-09T04:30:18.646 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596522+0000", 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596522+0000", 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596525+0000", 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.8909999999999999e-06 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596531+0000", 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.1510000000000001e-06 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:18.648 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596536+0000", 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.524e-06 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.596910+0000", 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00037356999999999999 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.597031+0000", 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000121415 2023-10-09T04:30:18.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.597161+0000", 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000129281 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.625720+0000", 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.028559462000000001 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.630035+0000", 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0043147410000000004 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.630046+0000", 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1522000000000001e-05 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:18.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:161283 11.1 11:8fab9489:::benchmark_data_smithi163_601242_object56504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:18.621139+0000", 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.019088352999999999, 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.019252958000000001, 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 161283 2023-10-09T04:30:18.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621139+0000", 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621139+0000", 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621137+0000", 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621145+0000", 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.5109999999999997e-06 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621162+0000", 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7750000000000001e-05 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621171+0000", 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.14e-06 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:18.655 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621445+0000", 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00027417499999999998 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621593+0000", 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014815600000000001 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.621725+0000", 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000132273 2023-10-09T04:30:18.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.638626+0000", 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.016900273 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:18.640321+0000", 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001695895 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:18.657 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:18.658 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:18.658 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:30:18.658 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 2 2023-10-09T04:30:18.658 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:30:18.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:18.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:19.013 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:19.092 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:19.168 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:19 smithi163 ceph-mon[213841]: pgmap v1442: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 7245/149715 objects misplaced (4.839%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T04:30:19.171 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:19 smithi163 ceph-mon[221067]: pgmap v1442: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 7245/149715 objects misplaced (4.839%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T04:30:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:19 smithi181 ceph-mon[196979]: pgmap v1442: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 7245/149715 objects misplaced (4.839%); 9.6 MiB/s, 153 objects/s recovering 2023-10-09T04:30:20.188 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:20.445 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:20.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:20.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:30:20.715 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:20.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:20.727 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:30:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:30:20] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-10-09T04:30:20.821 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:20.987 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:21.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:21 smithi163 ceph-mon[213841]: pgmap v1443: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 7245/149184 objects misplaced (4.856%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:30:21.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:21 smithi163 ceph-mon[221067]: pgmap v1443: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 7245/149184 objects misplaced (4.856%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:30:21.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:21 smithi181 ceph-mon[196979]: pgmap v1443: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 2 active+remapped+backfill_wait, 115 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 432 op/s; 7245/149184 objects misplaced (4.856%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:30:21.729 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:21.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:21.834 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:21.841 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:21.959 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:22.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:22.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:22.299 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:30:22.335 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:22.341 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:30:22.366 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:22.408 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:30:22.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:22.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:22.473 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:30:22.511 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:30:22.536 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:22.548 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:30:22.583 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:30:22.626 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:30:22.638 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:22.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.654 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.655 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:30:22.738 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: pgmap v1444: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 6182/139932 objects misplaced (4.418%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:30:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1866002464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:30:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:30:22.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:22 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: pgmap v1444: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 6182/139932 objects misplaced (4.418%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1866002464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:30:22.868 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.870 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:22.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: pgmap v1444: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 6182/139932 objects misplaced (4.418%); 13 MiB/s, 206 objects/s recovering 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1866002464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:30:22.876 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:22 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:22.974 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:23.010 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:23.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:23.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:23.183 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:23.250 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:23.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:23.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:23.774 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:23.824 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:30:23.890 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1973077281' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1973077281' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1973077281' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:30:24.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:24.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[213841]: failed to encode map e546 with expected crc 2023-10-09T04:30:24.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:30:24.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[213841]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T04:30:24.791 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[213841]: failed to encode map e546 with expected crc 2023-10-09T04:30:24.903 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:24.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:25.009 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:25.035 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:25.076 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:25.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:24 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:25.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:24 smithi181 ceph-mon[196979]: failed to encode map e546 with expected crc 2023-10-09T04:30:25.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:30:25.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:24 smithi181 ceph-mon[196979]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T04:30:25.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:24 smithi181 ceph-mon[196979]: failed to encode map e546 with expected crc 2023-10-09T04:30:25.082 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:24 smithi181 ceph-mon[196979]: pgmap v1446: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 6182/139932 objects misplaced (4.418%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:30:25.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[213841]: pgmap v1446: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 6182/139932 objects misplaced (4.418%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:30:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:25.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[221067]: failed to encode map e546 with expected crc 2023-10-09T04:30:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:30:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[221067]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T04:30:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[221067]: failed to encode map e546 with expected crc 2023-10-09T04:30:25.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:24 smithi163 ceph-mon[221067]: pgmap v1446: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 2.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 6182/139932 objects misplaced (4.418%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:30:25.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:25.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:25.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:25.220 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:25.277 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:25.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:25.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:25.410 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:25.461 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:25.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:25.568 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:25.604 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[221067]: failed to encode map e547 with expected crc 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[221067]: failed to encode map e547 with expected crc 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[221067]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[221067]: failed to encode map e547 with expected crc 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[213841]: failed to encode map e547 with expected crc 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[213841]: failed to encode map e547 with expected crc 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[213841]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T04:30:25.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:25 smithi163 ceph-mon[213841]: failed to encode map e547 with expected crc 2023-10-09T04:30:26.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:30:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:25 smithi181 ceph-mon[196979]: failed to encode map e547 with expected crc 2023-10-09T04:30:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:25 smithi181 ceph-mon[196979]: failed to encode map e547 with expected crc 2023-10-09T04:30:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:25 smithi181 ceph-mon[196979]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T04:30:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:25 smithi181 ceph-mon[196979]: failed to encode map e547 with expected crc 2023-10-09T04:30:26.359 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:26.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:26.470 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:26.621 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:26.791 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:26.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:26.904 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:27.004 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:27.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[221067]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[221067]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[221067]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T04:30:27.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[221067]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.071 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[221067]: pgmap v1449: 119 pgs: 3 active+remapped+backfilling, 116 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 5344/132363 objects misplaced (4.037%); 11 MiB/s, 173 objects/s recovering 2023-10-09T04:30:27.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[213841]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[213841]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[213841]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T04:30:27.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[213841]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.072 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:26 smithi163 ceph-mon[213841]: pgmap v1449: 119 pgs: 3 active+remapped+backfilling, 116 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 5344/132363 objects misplaced (4.037%); 11 MiB/s, 173 objects/s recovering 2023-10-09T04:30:27.175 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:27.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:27.237 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:26 smithi181 ceph-mon[196979]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:26 smithi181 ceph-mon[196979]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:26 smithi181 ceph-mon[196979]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T04:30:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:26 smithi181 ceph-mon[196979]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:26 smithi181 ceph-mon[196979]: pgmap v1449: 119 pgs: 3 active+remapped+backfilling, 116 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 5344/132363 objects misplaced (4.037%); 11 MiB/s, 173 objects/s recovering 2023-10-09T04:30:27.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:27.360 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:27.360 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:27.401 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:27.702 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:27.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:27.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:27.809 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:27.876 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:30:27.894 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:27 smithi163 ceph-mon[213841]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.894 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3570506249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:30:27.894 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:30:27.894 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:27 smithi163 ceph-mon[221067]: failed to encode map e548 with expected crc 2023-10-09T04:30:27.894 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3570506249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:30:27.894 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:30:27.895 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:28.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:28.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:28.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:28.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:28.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:27 smithi181 ceph-mon[196979]: failed to encode map e548 with expected crc 2023-10-09T04:30:28.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3570506249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:30:28.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:30:28.171 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:28.184 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:30:28.214 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:28.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:28.321 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:28.352 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:28.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:28.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:28.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:28.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:28.553 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:28.630 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:28.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:28.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:28.737 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:28.778 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:28.843 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:28.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:28.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:28.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:30:28.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:28.988 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:29.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: 1.18 scrub starts 2023-10-09T04:30:29.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:30:29.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T04:30:29.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.021 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:28 smithi181 ceph-mon[196979]: pgmap v1451: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 5064/129108 objects misplaced (3.922%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:30:29.051 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:29.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:29.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:29.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: 1.18 scrub starts 2023-10-09T04:30:29.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:30:29.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T04:30:29.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.085 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[221067]: pgmap v1451: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 5064/129108 objects misplaced (3.922%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:30:29.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: 1.18 scrub starts 2023-10-09T04:30:29.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:30:29.086 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T04:30:29.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.087 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: failed to encode map e549 with expected crc 2023-10-09T04:30:29.088 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:28 smithi163 ceph-mon[213841]: pgmap v1451: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 5064/129108 objects misplaced (3.922%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:30:29.158 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:29.238 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:30.237 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:30:30.243 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:30.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:30.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:30.475 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:30.575 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:30.677 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:30.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:30.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:30.882 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:30.917 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:30:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:30:30] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-10-09T04:30:31.040 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:31.123 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:31.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:31.175 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:31 smithi163 ceph-mon[221067]: pgmap v1452: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 5064/128475 objects misplaced (3.942%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:30:31.176 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:31 smithi163 ceph-mon[213841]: pgmap v1452: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 5064/128475 objects misplaced (3.942%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:30:31.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:31.328 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:31.415 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:30:31.460 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:30:31.503 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:30:31.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:31.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:31 smithi181 ceph-mon[196979]: pgmap v1452: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 5064/128475 objects misplaced (3.942%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:30:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:31.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:31.538 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:30:31.581 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:30:31.621 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:30:31.628 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:31.660 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:30:31.695 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:30:31.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.712 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.713 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.713 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.713 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.713 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:31.731 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:32.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:32 smithi181 ceph-mon[196979]: 1.10 scrub starts 2023-10-09T04:30:32.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:32 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/81331791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:32.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:32 smithi163 ceph-mon[221067]: 1.10 scrub starts 2023-10-09T04:30:32.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:32 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/81331791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:32.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:32 smithi163 ceph-mon[213841]: 1.10 scrub starts 2023-10-09T04:30:32.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:32 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/81331791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:33.488 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:33.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:33.499 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:33 smithi163 ceph-mon[213841]: pgmap v1453: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 3440/118950 objects misplaced (2.892%); 17 MiB/s, 267 objects/s recovering 2023-10-09T04:30:33.499 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:33 smithi163 ceph-mon[221067]: pgmap v1453: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 3440/118950 objects misplaced (2.892%); 17 MiB/s, 267 objects/s recovering 2023-10-09T04:30:33.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:33 smithi181 ceph-mon[196979]: pgmap v1453: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 928 op/s; 3440/118950 objects misplaced (2.892%); 17 MiB/s, 267 objects/s recovering 2023-10-09T04:30:33.573 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:33.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:33.700 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:34.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:34.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:34.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:34.511 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:34.551 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:34.617 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:34.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:34.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:34.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:34.824 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:35.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:35.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:35.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:35.153 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:35.185 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:35.245 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:35.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:35.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:35.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:35 smithi163 ceph-mon[213841]: pgmap v1454: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 3440/118950 objects misplaced (2.892%); 14 MiB/s, 227 objects/s recovering 2023-10-09T04:30:35.336 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:35.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:35 smithi163 ceph-mon[221067]: pgmap v1454: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 3440/118950 objects misplaced (2.892%); 14 MiB/s, 227 objects/s recovering 2023-10-09T04:30:35.346 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:35.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:35.382 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:35.450 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:35.453 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:35 smithi181 ceph-mon[196979]: pgmap v1454: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 3440/118950 objects misplaced (2.892%); 14 MiB/s, 227 objects/s recovering 2023-10-09T04:30:35.453 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:35.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:35.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:35.606 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:35.647 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:30:35.647 INFO:teuthology.orchestra.run.smithi163.stdout: "size": 20, 2023-10-09T04:30:35.647 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 600, 2023-10-09T04:30:35.647 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:30:35.647 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:57099 11.1 11:865c1c69:::benchmark_data_smithi163_601242_object57098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:03.757721+0000", 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 151.81848022400001, 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.27673738199999998, 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57099 2023-10-09T04:30:35.648 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757721+0000", 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757721+0000", 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.649 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757719+0000", 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757744+0000", 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4459000000000001e-05 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.650 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757757+0000", 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2884000000000001e-05 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757765+0000", 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0320000000000003e-06 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757816+0000", 2023-10-09T04:30:35.651 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.1561000000000003e-05 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.757966+0000", 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014954399999999999 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.758116+0000", 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015058500000000001 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.652 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.761274+0000", 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003157694 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.766473+0000", 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0051985929999999996 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.653 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034353+0000", 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26787981500000002 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034412+0000", 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9691999999999997e-05 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034458+0000", 2023-10-09T04:30:35.654 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6224000000000003e-05 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:57110 11.8 e516/507 11:1c87aceb:::benchmark_data_smithi163_601242_object57109:head v 516'3457, mlcod=516'3454)", 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:03.772271+0000", 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 151.803930041, 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26188607200000003, 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.655 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772271+0000", 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772271+0000", 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.656 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772271+0000", 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772330+0000", 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9738999999999998e-05 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772339+0000", 2023-10-09T04:30:35.657 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7730000000000005e-06 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772344+0000", 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0529999999999997e-06 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772376+0000", 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.1341999999999999e-05 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.658 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772389+0000", 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3536999999999999e-05 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.775979+0000", 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0035895599999999999 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.659 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034157+0000", 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25817866499999997 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:57117 11.8 e516/507 11:12cc3834:::benchmark_data_smithi163_601242_object57116:head v 516'3458, mlcod=516'3455)", 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:03.772552+0000", 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 151.803648831, 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26175925900000002, 2023-10-09T04:30:35.660 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772552+0000", 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772552+0000", 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.661 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772552+0000", 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772634+0000", 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.2163000000000006e-05 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.662 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772644+0000", 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.7550000000000006e-06 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772650+0000", 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.9959999999999999e-06 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.663 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.774093+0000", 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001443253 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.774114+0000", 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0930000000000001e-05 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034175+0000", 2023-10-09T04:30:35.664 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26006089100000002 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034312+0000", 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013673099999999999 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.665 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:57123 11.1 11:87cdde8b:::benchmark_data_smithi163_601242_object57122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:03.772775+0000", 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 151.80342603, 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.26751641700000001, 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 57123 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.666 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772775+0000", 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772775+0000", 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.667 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772774+0000", 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772794+0000", 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.9463e-05 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.668 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772803+0000", 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8600000000000005e-06 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772809+0000", 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.169e-06 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772847+0000", 2023-10-09T04:30:35.669 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8257999999999999e-05 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.772959+0000", 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000111907 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.773053+0000", 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4747000000000006e-05 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.670 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.774891+0000", 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001837843 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034500+0000", 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25960886100000002 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.671 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.040194+0000", 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0056940280000000003 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.040250+0000", 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5346000000000002e-05 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.040292+0000", 2023-10-09T04:30:35.672 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2156000000000001e-05 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46577.0:57099 11.1 e516/507 ondisk, result = 0)", 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:03.776002+0000", 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 151.80019948, 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25846497899999998, 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:30:35.673 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.776002+0000", 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.776002+0000", 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.674 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.776001+0000", 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.776007+0000", 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.0079999999999999e-06 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.675 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034167+0000", 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25816025399999998 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034190+0000", 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2846000000000001e-05 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034285+0000", 2023-10-09T04:30:35.676 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4758000000000003e-05 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034309+0000", 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.429e-05 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034467+0000", 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00015787099999999999 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.677 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:57128 11.a e516/507 11:5e9a6111:::benchmark_data_smithi163_601242_object57127:head v 516'3530, mlcod=516'3528)", 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:03.779214+0000", 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 151.796987245, 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25720789599999999, 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.678 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.779214+0000", 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.779214+0000", 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.679 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.779213+0000", 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:03.779267+0000", 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3921000000000001e-05 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034165+0000", 2023-10-09T04:30:35.680 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25489810899999998 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034192+0000", 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6886e-05 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034310+0000", 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117484 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.681 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.034374+0000", 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.4598999999999996e-05 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.036380+0000", 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0020059330000000001 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.682 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:04.036422+0000", 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1658e-05 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:91530 11.1 11:8f19bf47:::benchmark_data_smithi163_601242_object91529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:40.784702+0000", 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 114.791499094, 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.232529507, 2023-10-09T04:30:35.683 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 91530 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.784702+0000", 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.684 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.784702+0000", 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.784701+0000", 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.685 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.784724+0000", 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.338e-05 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.784736+0000", 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2044e-05 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.686 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.784744+0000", 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8890000000000002e-06 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.788099+0000", 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003355355 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.788265+0000", 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000165935 2023-10-09T04:30:35.687 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.788441+0000", 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000175957 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.797041+0000", 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0086002200000000004 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.688 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798824+0000", 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0017824049999999999 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.803567+0000", 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047430069999999996 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.803619+0000", 2023-10-09T04:30:35.689 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2049000000000003e-05 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:41.017232+0000", 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.21361295899999999 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:91538 11.0 e522/507 11:00b6fc69:::benchmark_data_smithi163_601242_object91537:head v 522'5811, mlcod=522'5809)", 2023-10-09T04:30:35.690 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:40.798211+0000", 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 114.777990652, 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.233196137, 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798211+0000", 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.691 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798211+0000", 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798210+0000", 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798251+0000", 2023-10-09T04:30:35.692 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0847999999999999e-05 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798263+0000", 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2495e-05 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798272+0000", 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7339999999999997e-06 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.693 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798444+0000", 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00017267300000000001 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:40.798467+0000", 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3059000000000001e-05 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.694 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:41.031353+0000", 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.232886023 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:41.031407+0000", 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3309000000000001e-05 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.695 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:103609 11.1 11:8642cf26:::benchmark_data_smithi163_601242_object103608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:57.761489+0000", 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 97.814712103999994, 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.254886958, 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 103609 2023-10-09T04:30:35.696 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761489+0000", 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761489+0000", 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.697 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761487+0000", 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761511+0000", 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.376e-05 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.698 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761525+0000", 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3531e-05 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761545+0000", 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0174e-05 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.699 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761589+0000", 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.3951e-05 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761709+0000", 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012054599999999999 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.761807+0000", 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.8140000000000006e-05 2023-10-09T04:30:35.700 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.764562+0000", 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0027545080000000001 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.768096+0000", 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003534508 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.701 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:58.016263+0000", 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24816632999999999 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:58.016317+0000", 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.4716999999999998e-05 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.702 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:58.016376+0000", 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.8789000000000002e-05 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:103613 11.1 11:899f56f8:::benchmark_data_smithi163_601242_object103612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:28:57.765259+0000", 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 97.810942088999994, 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.251221688, 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.703 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 103613 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765259+0000", 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.704 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765259+0000", 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765258+0000", 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.705 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765284+0000", 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.6432999999999999e-05 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765294+0000", 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0077e-05 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765301+0000", 2023-10-09T04:30:35.706 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.8260000000000004e-06 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765347+0000", 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.5702999999999999e-05 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765464+0000", 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000117286 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.707 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.765571+0000", 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010641699999999999 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.768123+0000", 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0025528970000000002 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.708 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:57.768204+0000", 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0986e-05 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:58.016411+0000", 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24820694800000001 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:58.016452+0000", 2023-10-09T04:30:35.709 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0296999999999997e-05 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:28:58.016481+0000", 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.9251000000000002e-05 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:156508 11.8 11:11029075:::benchmark_data_smithi163_601242_object51729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-10-09T04:30:35.710 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.963471+0000", 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.612730783, 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25593604599999997, 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 156508 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.711 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963471+0000", 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963471+0000", 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963469+0000", 2023-10-09T04:30:35.712 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963477+0000", 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7129999999999994e-06 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963494+0000", 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.6656000000000001e-05 2023-10-09T04:30:35.713 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963503+0000", 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.7199999999999995e-06 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963555+0000", 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.2181000000000001e-05 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.714 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963682+0000", 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012740700000000001 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.964216+0000", 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00053347799999999999 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.973672+0000", 2023-10-09T04:30:35.715 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0094563970000000001 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.975415+0000", 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001743264 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.978384+0000", 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0029682580000000001 2023-10-09T04:30:35.716 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219240+0000", 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24085588599999999 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219290+0000", 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0562999999999997e-05 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.717 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219407+0000", 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00011650399999999999 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:156509 11.8 11:11858ce6:::benchmark_data_smithi163_601242_object51730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.963520+0000", 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.612681073000001, 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25605985199999998, 2023-10-09T04:30:35.718 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 156509 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963520+0000", 2023-10-09T04:30:35.719 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963520+0000", 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963520+0000", 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.720 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963523+0000", 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.5710000000000002e-06 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963531+0000", 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.3209999999999999e-06 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.721 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963537+0000", 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.6239999999999996e-06 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.970266+0000", 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0067292649999999999 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.722 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.970414+0000", 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000147126 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.970554+0000", 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00014014 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.975361+0000", 2023-10-09T04:30:35.723 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0048073630000000003 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216198+0000", 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24083686400000001 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219209+0000", 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0030107160000000001 2023-10-09T04:30:35.724 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219447+0000", 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00023801900000000001 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219486+0000", 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.8824e-05 2023-10-09T04:30:35.725 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219580+0000", 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.4581999999999997e-05 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.726 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:156510 11.1 11:80cb5e91:::benchmark_data_smithi163_601242_object51731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.963543+0000", 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.612658526000001, 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25276425200000002, 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 156510 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.727 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963543+0000", 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963543+0000", 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.728 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963542+0000", 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963545+0000", 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.159e-06 2023-10-09T04:30:35.729 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963550+0000", 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7030000000000003e-06 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963556+0000", 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3079999999999998e-06 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.730 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963669+0000", 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000113285 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963800+0000", 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013126300000000001 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.731 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.963907+0000", 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000107216 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974924+0000", 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.011016784 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.975291+0000", 2023-10-09T04:30:35.732 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00036698999999999998 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216219+0000", 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.240927366 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216265+0000", 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.6588999999999997e-05 2023-10-09T04:30:35.733 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216307+0000", 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1936e-05 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:156506 11.0 e542/507 11:0149a6db:::benchmark_data_smithi163_601242_object51727:head v 542'9905, mlcod=542'9901)", 2023-10-09T04:30:35.734 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.965542+0000", 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.610659018, 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25063216900000002, 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.965542+0000", 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.735 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.965542+0000", 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.965542+0000", 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.736 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.965549+0000", 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0890000000000002e-06 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.965559+0000", 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.026e-05 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.965566+0000", 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1250000000000004e-06 2023-10-09T04:30:35.737 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.967418+0000", 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0018518639999999999 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.967436+0000", 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7955e-05 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.738 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.978386+0000", 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.010949782999999999 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216174+0000", 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23778880799999999 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.739 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:156511 11.0 e542/507 11:05baa358:::benchmark_data_smithi163_601242_object51732:head v 542'9906, mlcod=542'9901)", 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.974923+0000", 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.601278743000002, 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24424398, 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974923+0000", 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.740 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974923+0000", 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974922+0000", 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.741 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974927+0000", 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7600000000000002e-06 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974933+0000", 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.2670000000000003e-06 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974938+0000", 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.8130000000000003e-06 2023-10-09T04:30:35.742 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.975387+0000", 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00044902800000000002 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.975405+0000", 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.8082999999999999e-05 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.743 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219127+0000", 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.243721403 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219167+0000", 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.0048e-05 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.744 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.745 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.745 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:156512 11.0 e542/507 11:0fc96b4b:::benchmark_data_smithi163_601242_object51733:head v 542'9907, mlcod=542'9901)", 2023-10-09T04:30:35.745 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.974942+0000", 2023-10-09T04:30:35.745 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.601259219999999, 2023-10-09T04:30:35.745 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25051148099999998, 2023-10-09T04:30:35.745 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974942+0000", 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974942+0000", 2023-10-09T04:30:35.746 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974942+0000", 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974945+0000", 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.467e-06 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.747 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974950+0000", 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.4050000000000002e-06 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974953+0000", 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.337e-06 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216233+0000", 2023-10-09T04:30:35.748 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24127997700000001 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216256+0000", 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.3220000000000001e-05 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.225417+0000", 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0091610130000000008 2023-10-09T04:30:35.749 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.225454+0000", 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6236999999999999e-05 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.750 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:156519 11.0 e542/507 11:0ea89e9b:::benchmark_data_smithi163_601242_object51740:head v 542'9908, mlcod=542'9901)", 2023-10-09T04:30:35.751 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.974956+0000", 2023-10-09T04:30:35.751 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.601244955999999, 2023-10-09T04:30:35.751 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.25053725599999999, 2023-10-09T04:30:35.751 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.751 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974956+0000", 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974956+0000", 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.752 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974956+0000", 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967296 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974963+0000", 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.3239999999999999e-06 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.753 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974967+0000", 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7929999999999997e-06 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.974971+0000", 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.7459999999999998e-06 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.754 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219874+0000", 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24490341199999999 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219895+0000", 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.0441e-05 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.225457+0000", 2023-10-09T04:30:35.755 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0055617349999999999 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.225494+0000", 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.6981999999999999e-05 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46577.0:156509 11.8 e542/535 ondisk, result = 0)", 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.977335+0000", 2023-10-09T04:30:35.756 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.598866228999999, 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24187614700000001, 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.977335+0000", 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.757 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.977335+0000", 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.977321+0000", 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999862 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.758 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.977343+0000", 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.1999000000000001e-05 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.977357+0000", 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3772e-05 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.977367+0000", 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.696e-06 2023-10-09T04:30:35.759 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219193+0000", 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.241826336 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219202+0000", 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.8060000000000001e-06 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.760 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.219211+0000", 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.414e-06 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:156521 11.1 11:87fc8ea1:::benchmark_data_smithi163_601242_object51742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.980476+0000", 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.595725408, 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.24434847200000001, 2023-10-09T04:30:35.761 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 156521 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980476+0000", 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.762 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980476+0000", 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980474+0000", 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.763 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980486+0000", 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1671999999999999e-05 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216146+0000", 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23566063800000001 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.764 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216160+0000", 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.3437e-05 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216348+0000", 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000187777 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216473+0000", 2023-10-09T04:30:35.765 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012597300000000001 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216567+0000", 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.3098999999999998e-05 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.220279+0000", 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.003712552 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.766 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.224725+0000", 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0044453940000000001 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.224733+0000", 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 8.0499999999999992e-06 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.767 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "commit_sent", 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.224783+0000", 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.0401000000000001e-05 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.224824+0000", 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.1389e-05 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.768 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop_reply(client.46577.0:156510 11.1 e542/507 ondisk, result = 0)", 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:30:10.980908+0000", 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 24.595293306999999, 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.235409071, 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.769 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980908+0000", 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980908+0000", 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980907+0000", 2023-10-09T04:30:35.770 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:10.980914+0000", 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.5549999999999999e-06 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216149+0000", 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.23523544699999999 2023-10-09T04:30:35.771 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216159+0000", 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 9.9790000000000005e-06 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216202+0000", 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2871999999999999e-05 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.772 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:30:35.773 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216212+0000", 2023-10-09T04:30:35.773 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0030000000000001e-05 2023-10-09T04:30:35.773 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:30:35.773 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:30:35.773 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "done", 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:30:11.216317+0000", 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010505600000000001 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:30:35.774 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:30:35.775 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:35.799 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:35.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:35.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:35.932 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:35.977 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:35.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:35.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:36.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:36.152 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:36.240 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:30:36.434 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2607826631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:36.434 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2607826631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:36.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2607826631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:37.043 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:37.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:37.158 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:37.195 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:37.238 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:37.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[221067]: pgmap v1455: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 2087/111222 objects misplaced (1.876%); 17 MiB/s, 264 objects/s recovering 2023-10-09T04:30:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:30:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[221067]: failed to encode map e550 with expected crc 2023-10-09T04:30:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2607826631' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:37.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[221067]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T04:30:37.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[213841]: pgmap v1455: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 2087/111222 objects misplaced (1.876%); 17 MiB/s, 264 objects/s recovering 2023-10-09T04:30:37.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:37.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:30:37.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[213841]: failed to encode map e550 with expected crc 2023-10-09T04:30:37.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2607826631' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:37.335 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:37 smithi163 ceph-mon[213841]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T04:30:37.343 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:37.388 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:37.454 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:37.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:37.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:37 smithi181 ceph-mon[196979]: pgmap v1455: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 2087/111222 objects misplaced (1.876%); 17 MiB/s, 264 objects/s recovering 2023-10-09T04:30:37.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:37 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:37.462 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:37 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:30:37.463 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:37 smithi181 ceph-mon[196979]: failed to encode map e550 with expected crc 2023-10-09T04:30:37.463 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2607826631' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:37.463 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:37 smithi181 ceph-mon[196979]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T04:30:37.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:37.626 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:37.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:30:37.945 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:37.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:38.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:38.126 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:38.208 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:38.209 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:38 smithi181 ceph-mon[196979]: failed to encode map e550 with expected crc 2023-10-09T04:30:38.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:38.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:38.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:38.404 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:38.414 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:38.532 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:38.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:38.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:38 smithi163 ceph-mon[221067]: failed to encode map e550 with expected crc 2023-10-09T04:30:38.588 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:38 smithi163 ceph-mon[213841]: failed to encode map e550 with expected crc 2023-10-09T04:30:38.642 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:38.684 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:38.746 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:38.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:38.851 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:38.887 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.016 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:39.121 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:39.189 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.309 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:39.322 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:39 smithi163 ceph-mon[213841]: pgmap v1457: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 2087/107409 objects misplaced (1.943%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:30:39.322 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2457506218' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:30:39.322 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:39 smithi163 ceph-mon[221067]: pgmap v1457: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 2087/107409 objects misplaced (1.943%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:30:39.323 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2457506218' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:30:39.422 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:39.506 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:39 smithi181 ceph-mon[196979]: pgmap v1457: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 2087/107409 objects misplaced (1.943%); 15 MiB/s, 238 objects/s recovering 2023-10-09T04:30:39.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2457506218' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:30:39.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:39.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:39.651 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:39.676 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.717 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:39.821 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:39.833 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:30:39.862 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.927 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:39.957 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:30:40.031 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:40.065 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.130 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:40.234 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:40.333 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.444 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:40.556 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:40.605 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.669 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:40.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:40.817 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.890 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[213841]: failed to encode map e551 with expected crc 2023-10-09T04:30:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2457506218' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:30:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[213841]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T04:30:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[213841]: failed to encode map e551 with expected crc 2023-10-09T04:30:40.900 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[213841]: pgmap v1459: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 2087/106584 objects misplaced (1.958%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:30:40.900 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:30:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:30:40] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.43.0" 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[221067]: failed to encode map e551 with expected crc 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2457506218' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[221067]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[221067]: failed to encode map e551 with expected crc 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:40 smithi163 ceph-mon[221067]: pgmap v1459: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 2087/106584 objects misplaced (1.958%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:40 smithi181 ceph-mon[196979]: failed to encode map e551 with expected crc 2023-10-09T04:30:40.901 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:40 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2457506218' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:30:40.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:40 smithi181 ceph-mon[196979]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T04:30:40.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:40 smithi181 ceph-mon[196979]: failed to encode map e551 with expected crc 2023-10-09T04:30:40.902 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:40 smithi181 ceph-mon[196979]: pgmap v1459: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 2087/106584 objects misplaced (1.958%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:30:40.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:40.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:30:41.003 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:41.033 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.094 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:41.197 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:41.297 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.373 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:30:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:41.440 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:30:41.484 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:30:41.514 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:41.546 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:30:41.584 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:30:41.606 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.618 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:30:41.657 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:30:41.696 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.723 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:30:41.875 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:41.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:41 smithi163 ceph-mon[221067]: 1.12 deep-scrub starts 2023-10-09T04:30:41.880 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:41 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1566696412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:41.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:41 smithi163 ceph-mon[213841]: 1.12 deep-scrub starts 2023-10-09T04:30:41.880 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:41 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1566696412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:41.981 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:42.017 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:42.046 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:42.064 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:42.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:42.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:42.198 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:41 smithi181 ceph-mon[196979]: 1.12 deep-scrub starts 2023-10-09T04:30:42.198 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:41 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1566696412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:42.200 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:42.255 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:42.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:42.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:42.362 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:42.388 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:42.429 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:42.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:42.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:42.533 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:42.564 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:43.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:42 smithi181 ceph-mon[196979]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:42 smithi181 ceph-mon[196979]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:42 smithi181 ceph-mon[196979]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T04:30:43.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:42 smithi181 ceph-mon[196979]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.156 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:42 smithi181 ceph-mon[196979]: pgmap v1461: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 782/96267 objects misplaced (0.812%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:30:43.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[221067]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[221067]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.297 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[221067]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T04:30:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[221067]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[221067]: pgmap v1461: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 782/96267 objects misplaced (0.812%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:30:43.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[213841]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[213841]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[213841]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T04:30:43.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[213841]: failed to encode map e552 with expected crc 2023-10-09T04:30:43.299 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:42 smithi163 ceph-mon[213841]: pgmap v1461: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 782/96267 objects misplaced (0.812%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:30:43.300 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:43.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:43.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:43.437 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:43.508 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:43.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:43.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:43.584 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:43.585 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:43.616 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:43.661 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:43.938 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[221067]: failed to encode map e553 with expected crc 2023-10-09T04:30:43.938 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[221067]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T04:30:43.938 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[221067]: failed to encode map e553 with expected crc 2023-10-09T04:30:43.938 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[221067]: failed to encode map e553 with expected crc 2023-10-09T04:30:43.938 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[213841]: failed to encode map e553 with expected crc 2023-10-09T04:30:43.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[213841]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T04:30:43.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[213841]: failed to encode map e553 with expected crc 2023-10-09T04:30:43.939 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:43 smithi163 ceph-mon[213841]: failed to encode map e553 with expected crc 2023-10-09T04:30:44.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:43 smithi181 ceph-mon[196979]: failed to encode map e553 with expected crc 2023-10-09T04:30:44.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:43 smithi181 ceph-mon[196979]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T04:30:44.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:43 smithi181 ceph-mon[196979]: failed to encode map e553 with expected crc 2023-10-09T04:30:44.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:43 smithi181 ceph-mon[196979]: failed to encode map e553 with expected crc 2023-10-09T04:30:44.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:44.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:44.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:44.183 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:44.305 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:44.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:44.398 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:30:44.410 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:44.441 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:44.587 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:44.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:44.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:44.717 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:44.739 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.222 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.223 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:44 smithi181 ceph-mon[196979]: pgmap v1464: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 782/96267 objects misplaced (0.812%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:30:45.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T04:30:45.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.321 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.322 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.322 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.322 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.322 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[221067]: pgmap v1464: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 782/96267 objects misplaced (0.812%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.325 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: failed to encode map e554 with expected crc 2023-10-09T04:30:45.326 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:44 smithi163 ceph-mon[213841]: pgmap v1464: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 782/96267 objects misplaced (0.812%); 16 MiB/s, 261 objects/s recovering 2023-10-09T04:30:45.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:45.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:45.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:45.571 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:45.659 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:45.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:45.762 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:45.798 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:46.046 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:30:46.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:45 smithi181 ceph-mon[196979]: failed to encode map e555 with expected crc 2023-10-09T04:30:46.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:45 smithi181 ceph-mon[196979]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T04:30:46.081 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:45 smithi181 ceph-mon[196979]: failed to encode map e555 with expected crc 2023-10-09T04:30:46.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:45 smithi163 ceph-mon[221067]: failed to encode map e555 with expected crc 2023-10-09T04:30:46.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:45 smithi163 ceph-mon[221067]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T04:30:46.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:45 smithi163 ceph-mon[221067]: failed to encode map e555 with expected crc 2023-10-09T04:30:46.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:45 smithi163 ceph-mon[213841]: failed to encode map e555 with expected crc 2023-10-09T04:30:46.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:45 smithi163 ceph-mon[213841]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T04:30:46.089 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:45 smithi163 ceph-mon[213841]: failed to encode map e555 with expected crc 2023-10-09T04:30:46.097 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:46.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:46.206 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:46.239 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:46.288 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:46.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:46.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:46.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:46.434 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:46.546 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:46.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:46.644 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:46.644 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:46.653 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:46.738 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:46.824 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:46.908 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:46.908 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:30:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:46.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:46.930 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:30:47.014 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:46 smithi181 ceph-mon[196979]: 1.18 scrub starts 2023-10-09T04:30:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:46 smithi181 ceph-mon[196979]: 1.18 scrub ok 2023-10-09T04:30:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:46 smithi181 ceph-mon[196979]: pgmap v1466: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 11 MiB/s, 180 objects/s recovering 2023-10-09T04:30:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/216579602' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:47.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:46 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:47.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3100223183' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:30:47.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[221067]: 1.18 scrub starts 2023-10-09T04:30:47.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[221067]: 1.18 scrub ok 2023-10-09T04:30:47.041 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[221067]: pgmap v1466: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 11 MiB/s, 180 objects/s recovering 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/216579602' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3100223183' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[213841]: 1.18 scrub starts 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[213841]: 1.18 scrub ok 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[213841]: pgmap v1466: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 11 MiB/s, 180 objects/s recovering 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/216579602' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:47.042 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3100223183' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:30:47.044 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:47.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:30:47.876 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:47.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:47.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:48.035 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:48.092 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:48.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:48.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:48.132 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:48.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: 1.10 scrub starts 2023-10-09T04:30:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: 1.10 scrub ok 2023-10-09T04:30:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: failed to encode map e556 with expected crc 2023-10-09T04:30:48.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3100223183' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[221067]: failed to encode map e556 with expected crc 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: 1.10 scrub starts 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: 1.10 scrub ok 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: failed to encode map e556 with expected crc 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:48.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3100223183' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:30:48.267 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T04:30:48.267 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:47 smithi163 ceph-mon[213841]: failed to encode map e556 with expected crc 2023-10-09T04:30:48.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: 1.10 scrub starts 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: 1.10 scrub ok 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: failed to encode map e556 with expected crc 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3100223183' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T04:30:48.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:47 smithi181 ceph-mon[196979]: failed to encode map e556 with expected crc 2023-10-09T04:30:48.270 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:48.630 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:48.633 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:30:48.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:48.735 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:48.802 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:48.908 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:48 smithi163 ceph-mon[213841]: pgmap v1468: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:30:48.924 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:48.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:49.031 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:49.057 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:48 smithi181 ceph-mon[196979]: pgmap v1468: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:30:49.335 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:48 smithi163 ceph-mon[221067]: pgmap v1468: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:30:49.446 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:49.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:49.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:49.552 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:49.587 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:49.658 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:49.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:49.713 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:49.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:49.850 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:49.909 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:50.079 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:50.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:50.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:50.185 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:50.187 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:50.193 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:50.210 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:50.211 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:50.287 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:50.346 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:30:50.409 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:30:50.468 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:50.469 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:30:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:50.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:50.513 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:30:50.540 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:30:50.566 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:30:50.581 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:50.585 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:30:50.609 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:50.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:50.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:50.679 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:30:50.737 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:30:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:30:50] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.43.0" 2023-10-09T04:30:50.751 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:30:50.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:50.771 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.771 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.771 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.771 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.771 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.772 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.773 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.773 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:50.798 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: pgmap v1469: 119 pgs: 119 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 7.7 MiB/s, 122 objects/s recovering 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/315468644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]: dispatch 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [4, 0]}]: dispatch 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2023-10-09T04:30:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 0]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: pgmap v1469: 119 pgs: 119 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 7.7 MiB/s, 122 objects/s recovering 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/315468644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [4, 0]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2023-10-09T04:30:51.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2023-10-09T04:30:51.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T04:30:51.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 0]}]: dispatch 2023-10-09T04:30:51.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]: dispatch 2023-10-09T04:30:51.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]: dispatch 2023-10-09T04:30:51.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: pgmap v1469: 119 pgs: 119 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 7.7 MiB/s, 122 objects/s recovering 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/315468644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]: dispatch 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [4, 0]}]: dispatch 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2023-10-09T04:30:51.129 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-09T04:30:51.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 0]}]: dispatch 2023-10-09T04:30:51.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]: dispatch 2023-10-09T04:30:51.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]: dispatch 2023-10-09T04:30:51.130 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T04:30:51.139 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:51.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:51.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:30:51.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:51.288 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:51.354 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:51.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:51.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:51.460 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:51.533 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:51.678 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:51.961 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:51.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:51.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: failed to encode map e557 with expected crc 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: failed to encode map e557 with expected crc 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]': finished 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [4, 0]}]': finished 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]': finished 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]': finished 2023-10-09T04:30:52.044 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T04:30:52.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 0]}]': finished 2023-10-09T04:30:52.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]': finished 2023-10-09T04:30:52.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2023-10-09T04:30:52.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T04:30:52.045 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[213841]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: failed to encode map e557 with expected crc 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: failed to encode map e557 with expected crc 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]': finished 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [4, 0]}]': finished 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]': finished 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]': finished 2023-10-09T04:30:52.046 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T04:30:52.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 0]}]': finished 2023-10-09T04:30:52.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]': finished 2023-10-09T04:30:52.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2023-10-09T04:30:52.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T04:30:52.047 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:51 smithi163 ceph-mon[221067]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T04:30:52.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:52.098 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:52.197 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: failed to encode map e557 with expected crc 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: failed to encode map e557 with expected crc 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2]}]': finished 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [4, 0]}]': finished 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]': finished 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]': finished 2023-10-09T04:30:52.199 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-09T04:30:52.200 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 0]}]': finished 2023-10-09T04:30:52.200 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 7]}]': finished 2023-10-09T04:30:52.200 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 6]}]': finished 2023-10-09T04:30:52.200 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T04:30:52.200 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:51 smithi181 ceph-mon[196979]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T04:30:52.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:52.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:52.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:52.340 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:52.375 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:52.381 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:52.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:52.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:52.485 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:52.557 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:52.661 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:52.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:52.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:52.841 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:53.041 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:53.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:53.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:53.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: pgmap v1472: 119 pgs: 3 activating, 1 peering, 115 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 757 op/s 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/633632159' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:30:53.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.086 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: pgmap v1472: 119 pgs: 3 activating, 1 peering, 115 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 757 op/s 2023-10-09T04:30:53.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/633632159' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:30:53.087 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:52 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:30:53.149 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:53.176 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:53.223 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:53.227 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T04:30:53.227 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.227 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: failed to encode map e558 with expected crc 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: pgmap v1472: 119 pgs: 3 activating, 1 peering, 115 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 757 op/s 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/633632159' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:30:53.228 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:52 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:30:53.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:53.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:53.330 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:53.356 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:53.679 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:53.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:53.750 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:30:53.750 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:30:53.750 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 0 2023-10-09T04:30:53.751 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:30:53.787 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:53.847 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:30:53.864 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:54.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:54.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:54.075 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:54.147 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:54.185 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:54.592 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:54.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:54.696 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:54.718 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:30:54.733 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:30:54.734 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:30:54.774 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[221067]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[221067]: failed to encode map e559 with expected crc 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[221067]: failed to encode map e559 with expected crc 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[221067]: pgmap v1474: 119 pgs: 3 activating, 1 peering, 115 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:30:54.846 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[213841]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T04:30:54.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[213841]: failed to encode map e559 with expected crc 2023-10-09T04:30:54.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[213841]: failed to encode map e559 with expected crc 2023-10-09T04:30:54.847 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:54 smithi163 ceph-mon[213841]: pgmap v1474: 119 pgs: 3 activating, 1 peering, 115 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T04:30:54.996 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:55.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:55.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:54 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:30:55.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:54 smithi181 ceph-mon[196979]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T04:30:55.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:54 smithi181 ceph-mon[196979]: failed to encode map e559 with expected crc 2023-10-09T04:30:55.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:54 smithi181 ceph-mon[196979]: failed to encode map e559 with expected crc 2023-10-09T04:30:55.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:54 smithi181 ceph-mon[196979]: pgmap v1474: 119 pgs: 3 activating, 1 peering, 115 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T04:30:55.099 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:55.111 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:55.146 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:55.161 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:55.203 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:55.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:55.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:55.313 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:55.412 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:55.511 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:55.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:55.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:55.622 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:55.657 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:56.043 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:56.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:56.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:56.188 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:56.251 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:30:56.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:56 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:56.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1841285181' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:56.536 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:56.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:56 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1841285181' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:56 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:30:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1841285181' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:56.642 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:56.653 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:56.737 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:57.197 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:30:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:30:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:30:57.197 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:30:57.197 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:30:57.289 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:57.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:57.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:57.433 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:57.496 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:57.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:57 smithi181 ceph-mon[196979]: pgmap v1475: 119 pgs: 3 activating, 116 active+clean; 1.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2023-10-09T04:30:57.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:57 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:57.497 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:57 smithi181 ceph-mon[196979]: failed to encode map e560 with expected crc 2023-10-09T04:30:57.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:57 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1841285181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:57.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:57 smithi181 ceph-mon[196979]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T04:30:57.498 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:57 smithi181 ceph-mon[196979]: failed to encode map e560 with expected crc 2023-10-09T04:30:57.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:57.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:57.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[213841]: pgmap v1475: 119 pgs: 3 activating, 116 active+clean; 1.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2023-10-09T04:30:57.576 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:57.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[213841]: failed to encode map e560 with expected crc 2023-10-09T04:30:57.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1841285181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:57.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[213841]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T04:30:57.577 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[213841]: failed to encode map e560 with expected crc 2023-10-09T04:30:57.577 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[221067]: pgmap v1475: 119 pgs: 3 activating, 116 active+clean; 1.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s 2023-10-09T04:30:57.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:57.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[221067]: failed to encode map e560 with expected crc 2023-10-09T04:30:57.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1841285181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:57.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[221067]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T04:30:57.578 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:57 smithi163 ceph-mon[221067]: failed to encode map e560 with expected crc 2023-10-09T04:30:57.600 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:57.681 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:57.963 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:57.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:58.076 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:58.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:30:58.143 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:58.229 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:58.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:58.267 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:30:58.309 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:30:58.337 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:58.356 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:30:58.388 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:30:58.427 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:30:58.433 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:58.460 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:30:58.494 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:30:58.546 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:30:58.547 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:58.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.582 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:30:58.609 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:30:58.658 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:58.713 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:58.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:58 smithi163 ceph-mon[213841]: pgmap v1477: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:30:58.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:30:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1595543627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:58.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:58 smithi163 ceph-mon[221067]: pgmap v1477: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:30:58.971 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:30:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1595543627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:58.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:59.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:59.101 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:59.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:58 smithi181 ceph-mon[196979]: pgmap v1477: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 772 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:30:59.113 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:30:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1595543627' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:59.181 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:59.481 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:30:59.653 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:59.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:59.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:59.764 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:59.799 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:59.857 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:59.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:59.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:59.961 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:59.990 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:00.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:00.218 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:00.277 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:00.356 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.594 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:00.601 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:31:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:00.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:00.634 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:31:00.706 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:00.770 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:00.770 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:00.781 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.862 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:31:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:31:00] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.43.0" 2023-10-09T04:31:01.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:01 smithi181 ceph-mon[196979]: pgmap v1478: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:01 smithi163 ceph-mon[213841]: pgmap v1478: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:01.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:01 smithi163 ceph-mon[221067]: pgmap v1478: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:01.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:31:01.803 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:01.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:01.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:01.914 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:01.976 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:02.111 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:02.250 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1478,"stamp":"2023-10-09T04:31:00.110195+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1424785547,"num_objects":21757,"num_object_clones":0,"num_object_copies":65271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21757,"num_whiteouts":0,"num_read":24,"num_read_kb":43,"num_write":187894,"num_write_kb":6705804,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12031,"num_bytes_recovered":783313479,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96976,"ondisk_log_size":96976,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10554132,"kb_used_data":4180444,"kb_used_omap":126,"kb_used_meta":6373441,"kb_avail":645661932,"statfs":{"total":671965249536,"available":661157818368,"internally_reserved":0,"allocated":4280774656,"data_stored":4266430837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6526404219},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,5,9,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-253165568,"num_objects":-3863,"num_object_clones":0,"num_object_copies":-11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3863,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004117"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":736,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.974894+0000","last_change":"2023-10-09T04:30:45.821591+0000","last_active":"2023-10-09T04:30:51.974894+0000","last_peered":"2023-10-09T04:30:51.974894+0000","last_clean":"2023-10-09T04:30:51.974894+0000","last_became_active":"2023-10-09T04:29:47.255157+0000","last_became_peered":"2023-10-09T04:29:47.255157+0000","last_unstale":"2023-10-09T04:30:51.974894+0000","last_undegraded":"2023-10-09T04:30:51.974894+0000","last_fullsized":"2023-10-09T04:30:51.974894+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:30:45.821476+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:30:45.821476+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:07.307678+0000","scrub_duration":0.010879602,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":144,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964826+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:30:51.964826+0000","last_peered":"2023-10-09T04:30:51.964826+0000","last_clean":"2023-10-09T04:30:51.964826+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:30:51.964826+0000","last_undegraded":"2023-10-09T04:30:51.964826+0000","last_fullsized":"2023-10-09T04:30:51.964826+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":121,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966244+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:30:51.966244+0000","last_peered":"2023-10-09T04:30:51.966244+0000","last_clean":"2023-10-09T04:30:51.966244+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:30:51.966244+0000","last_undegraded":"2023-10-09T04:30:51.966244+0000","last_fullsized":"2023-10-09T04:30:51.966244+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":673,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964520+0000","last_change":"2023-10-09T04:29:47.243471+0000","last_active":"2023-10-09T04:30:51.964520+0000","last_peered":"2023-10-09T04:30:51.964520+0000","last_clean":"2023-10-09T04:30:51.964520+0000","last_became_active":"2023-10-09T04:29:47.243158+0000","last_became_peered":"2023-10-09T04:29:47.243158+0000","last_unstale":"2023-10-09T04:30:51.964520+0000","last_undegraded":"2023-10-09T04:30:51.964520+0000","last_fullsized":"2023-10-09T04:30:51.964520+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:45.996992+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":577,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.829919+0000","last_change":"2023-10-09T04:30:51.983079+0000","last_active":"2023-10-09T04:30:53.829919+0000","last_peered":"2023-10-09T04:30:53.829919+0000","last_clean":"2023-10-09T04:30:53.829919+0000","last_became_active":"2023-10-09T04:30:51.982789+0000","last_became_peered":"2023-10-09T04:30:51.982789+0000","last_unstale":"2023-10-09T04:30:53.829919+0000","last_undegraded":"2023-10-09T04:30:53.829919+0000","last_fullsized":"2023-10-09T04:30:53.829919+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:47.753426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828380+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:30:53.828380+0000","last_peered":"2023-10-09T04:30:53.828380+0000","last_clean":"2023-10-09T04:30:53.828380+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:30:53.828380+0000","last_undegraded":"2023-10-09T04:30:53.828380+0000","last_fullsized":"2023-10-09T04:30:53.828380+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":551,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964659+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:30:51.964659+0000","last_peered":"2023-10-09T04:30:51.964659+0000","last_clean":"2023-10-09T04:30:51.964659+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:30:51.964659+0000","last_undegraded":"2023-10-09T04:30:51.964659+0000","last_fullsized":"2023-10-09T04:30:51.964659+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827775+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:30:53.827775+0000","last_peered":"2023-10-09T04:30:53.827775+0000","last_clean":"2023-10-09T04:30:53.827775+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:30:53.827775+0000","last_undegraded":"2023-10-09T04:30:53.827775+0000","last_fullsized":"2023-10-09T04:30:53.827775+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828418+0000","last_change":"2023-10-09T04:30:51.977588+0000","last_active":"2023-10-09T04:30:53.828418+0000","last_peered":"2023-10-09T04:30:53.828418+0000","last_clean":"2023-10-09T04:30:53.828418+0000","last_became_active":"2023-10-09T04:30:51.977346+0000","last_became_peered":"2023-10-09T04:30:51.977346+0000","last_unstale":"2023-10-09T04:30:53.828418+0000","last_undegraded":"2023-10-09T04:30:53.828418+0000","last_fullsized":"2023-10-09T04:30:53.828418+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.956710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":978,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976021+0000","last_change":"2023-10-09T04:29:47.255527+0000","last_active":"2023-10-09T04:30:51.976021+0000","last_peered":"2023-10-09T04:30:51.976021+0000","last_clean":"2023-10-09T04:30:51.976021+0000","last_became_active":"2023-10-09T04:29:47.255304+0000","last_became_peered":"2023-10-09T04:29:47.255304+0000","last_unstale":"2023-10-09T04:30:51.976021+0000","last_undegraded":"2023-10-09T04:30:51.976021+0000","last_fullsized":"2023-10-09T04:30:51.976021+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:12.642548+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828869+0000","last_change":"2023-10-09T04:29:44.187371+0000","last_active":"2023-10-09T04:30:53.828869+0000","last_peered":"2023-10-09T04:30:53.828869+0000","last_clean":"2023-10-09T04:30:53.828869+0000","last_became_active":"2023-10-09T04:29:44.186627+0000","last_became_peered":"2023-10-09T04:29:44.186627+0000","last_unstale":"2023-10-09T04:30:53.828869+0000","last_undegraded":"2023-10-09T04:30:53.828869+0000","last_fullsized":"2023-10-09T04:30:53.828869+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:29.638655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.981579+0000","last_change":"2023-10-09T04:30:51.977627+0000","last_active":"2023-10-09T04:30:51.981579+0000","last_peered":"2023-10-09T04:30:51.981579+0000","last_clean":"2023-10-09T04:30:51.981579+0000","last_became_active":"2023-10-09T04:30:51.977309+0000","last_became_peered":"2023-10-09T04:30:51.977309+0000","last_unstale":"2023-10-09T04:30:51.981579+0000","last_undegraded":"2023-10-09T04:30:51.981579+0000","last_fullsized":"2023-10-09T04:30:51.981579+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:30.619870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":865,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975876+0000","last_change":"2023-10-09T04:27:45.967966+0000","last_active":"2023-10-09T04:30:51.975876+0000","last_peered":"2023-10-09T04:30:51.975876+0000","last_clean":"2023-10-09T04:30:51.975876+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:30:51.975876+0000","last_undegraded":"2023-10-09T04:30:51.975876+0000","last_fullsized":"2023-10-09T04:30:51.975876+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:45.967906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:45.967906+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:06.528314+0000","scrub_duration":0.0012842089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.977673+0000","last_change":"2023-10-09T04:29:44.191334+0000","last_active":"2023-10-09T04:30:51.977673+0000","last_peered":"2023-10-09T04:30:51.977673+0000","last_clean":"2023-10-09T04:30:51.977673+0000","last_became_active":"2023-10-09T04:29:44.191013+0000","last_became_peered":"2023-10-09T04:29:44.191013+0000","last_unstale":"2023-10-09T04:30:51.977673+0000","last_undegraded":"2023-10-09T04:30:51.977673+0000","last_fullsized":"2023-10-09T04:30:51.977673+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:12.273980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":203,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.965236+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:30:51.965236+0000","last_peered":"2023-10-09T04:30:51.965236+0000","last_clean":"2023-10-09T04:30:51.965236+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:30:51.965236+0000","last_undegraded":"2023-10-09T04:30:51.965236+0000","last_fullsized":"2023-10-09T04:30:51.965236+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":641,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828514+0000","last_change":"2023-10-09T04:29:47.253301+0000","last_active":"2023-10-09T04:30:53.828514+0000","last_peered":"2023-10-09T04:30:53.828514+0000","last_clean":"2023-10-09T04:30:53.828514+0000","last_became_active":"2023-10-09T04:29:47.252838+0000","last_became_peered":"2023-10-09T04:29:47.252838+0000","last_unstale":"2023-10-09T04:30:53.828514+0000","last_undegraded":"2023-10-09T04:30:53.828514+0000","last_fullsized":"2023-10-09T04:30:53.828514+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:45.161327+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":633,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976398+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:30:51.976398+0000","last_peered":"2023-10-09T04:30:51.976398+0000","last_clean":"2023-10-09T04:30:51.976398+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:30:51.976398+0000","last_undegraded":"2023-10-09T04:30:51.976398+0000","last_fullsized":"2023-10-09T04:30:51.976398+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":195,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.969567+0000","last_change":"2023-10-09T04:29:44.190663+0000","last_active":"2023-10-09T04:30:51.969567+0000","last_peered":"2023-10-09T04:30:51.969567+0000","last_clean":"2023-10-09T04:30:51.969567+0000","last_became_active":"2023-10-09T04:29:44.190245+0000","last_became_peered":"2023-10-09T04:29:44.190245+0000","last_unstale":"2023-10-09T04:30:51.969567+0000","last_undegraded":"2023-10-09T04:30:51.969567+0000","last_fullsized":"2023-10-09T04:30:51.969567+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:00:16.922638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":740,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827726+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:30:53.827726+0000","last_peered":"2023-10-09T04:30:53.827726+0000","last_clean":"2023-10-09T04:30:53.827726+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:30:53.827726+0000","last_undegraded":"2023-10-09T04:30:53.827726+0000","last_fullsized":"2023-10-09T04:30:53.827726+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":134,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964573+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:30:51.964573+0000","last_peered":"2023-10-09T04:30:51.964573+0000","last_clean":"2023-10-09T04:30:51.964573+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:30:51.964573+0000","last_undegraded":"2023-10-09T04:30:51.964573+0000","last_fullsized":"2023-10-09T04:30:51.964573+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964716+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:30:51.964716+0000","last_peered":"2023-10-09T04:30:51.964716+0000","last_clean":"2023-10-09T04:30:51.964716+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:30:51.964716+0000","last_undegraded":"2023-10-09T04:30:51.964716+0000","last_fullsized":"2023-10-09T04:30:51.964716+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":610,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832835+0000","last_change":"2023-10-09T04:29:47.263524+0000","last_active":"2023-10-09T04:30:53.832835+0000","last_peered":"2023-10-09T04:30:53.832835+0000","last_clean":"2023-10-09T04:30:53.832835+0000","last_became_active":"2023-10-09T04:29:47.263039+0000","last_became_peered":"2023-10-09T04:29:47.263039+0000","last_unstale":"2023-10-09T04:30:53.832835+0000","last_undegraded":"2023-10-09T04:30:53.832835+0000","last_fullsized":"2023-10-09T04:30:53.832835+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:02.486982+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":602,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975274+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:30:51.975274+0000","last_peered":"2023-10-09T04:30:51.975274+0000","last_clean":"2023-10-09T04:30:51.975274+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:30:51.975274+0000","last_undegraded":"2023-10-09T04:30:51.975274+0000","last_fullsized":"2023-10-09T04:30:51.975274+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":334,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975698+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:30:51.975698+0000","last_peered":"2023-10-09T04:30:51.975698+0000","last_clean":"2023-10-09T04:30:51.975698+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:30:51.975698+0000","last_undegraded":"2023-10-09T04:30:51.975698+0000","last_fullsized":"2023-10-09T04:30:51.975698+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":722,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.974985+0000","last_change":"2023-10-09T04:30:46.778471+0000","last_active":"2023-10-09T04:30:51.974985+0000","last_peered":"2023-10-09T04:30:51.974985+0000","last_clean":"2023-10-09T04:30:51.974985+0000","last_became_active":"2023-10-09T04:29:47.194746+0000","last_became_peered":"2023-10-09T04:29:47.194746+0000","last_unstale":"2023-10-09T04:30:51.974985+0000","last_undegraded":"2023-10-09T04:30:51.974985+0000","last_fullsized":"2023-10-09T04:30:51.974985+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:30:46.778406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:30:46.778406+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:51.879595+0000","scrub_duration":0.001341243,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828743+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:30:53.828743+0000","last_peered":"2023-10-09T04:30:53.828743+0000","last_clean":"2023-10-09T04:30:53.828743+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:30:53.828743+0000","last_undegraded":"2023-10-09T04:30:53.828743+0000","last_fullsized":"2023-10-09T04:30:53.828743+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":317,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976475+0000","last_change":"2023-10-09T04:29:44.196614+0000","last_active":"2023-10-09T04:30:51.976475+0000","last_peered":"2023-10-09T04:30:51.976475+0000","last_clean":"2023-10-09T04:30:51.976475+0000","last_became_active":"2023-10-09T04:29:44.196013+0000","last_became_peered":"2023-10-09T04:29:44.196013+0000","last_unstale":"2023-10-09T04:30:51.976475+0000","last_undegraded":"2023-10-09T04:30:51.976475+0000","last_fullsized":"2023-10-09T04:30:51.976475+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:38.381032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":705,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.965843+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:30:51.965843+0000","last_peered":"2023-10-09T04:30:51.965843+0000","last_clean":"2023-10-09T04:30:51.965843+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:30:51.965843+0000","last_undegraded":"2023-10-09T04:30:51.965843+0000","last_fullsized":"2023-10-09T04:30:51.965843+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964611+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:30:51.964611+0000","last_peered":"2023-10-09T04:30:51.964611+0000","last_clean":"2023-10-09T04:30:51.964611+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:30:51.964611+0000","last_undegraded":"2023-10-09T04:30:51.964611+0000","last_fullsized":"2023-10-09T04:30:51.964611+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828371+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:30:53.828371+0000","last_peered":"2023-10-09T04:30:53.828371+0000","last_clean":"2023-10-09T04:30:53.828371+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:30:53.828371+0000","last_undegraded":"2023-10-09T04:30:53.828371+0000","last_fullsized":"2023-10-09T04:30:53.828371+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":446,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975299+0000","last_change":"2023-10-09T04:30:40.819676+0000","last_active":"2023-10-09T04:30:51.975299+0000","last_peered":"2023-10-09T04:30:51.975299+0000","last_clean":"2023-10-09T04:30:51.975299+0000","last_became_active":"2023-10-09T04:29:47.249760+0000","last_became_peered":"2023-10-09T04:29:47.249760+0000","last_unstale":"2023-10-09T04:30:51.975299+0000","last_undegraded":"2023-10-09T04:30:51.975299+0000","last_fullsized":"2023-10-09T04:30:51.975299+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":195,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828115+0000","last_change":"2023-10-09T04:29:44.182868+0000","last_active":"2023-10-09T04:30:53.828115+0000","last_peered":"2023-10-09T04:30:53.828115+0000","last_clean":"2023-10-09T04:30:53.828115+0000","last_became_active":"2023-10-09T04:29:44.177350+0000","last_became_peered":"2023-10-09T04:29:44.177350+0000","last_unstale":"2023-10-09T04:30:53.828115+0000","last_undegraded":"2023-10-09T04:30:53.828115+0000","last_fullsized":"2023-10-09T04:30:53.828115+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:52.638746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":225,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964118+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:30:51.964118+0000","last_peered":"2023-10-09T04:30:51.964118+0000","last_clean":"2023-10-09T04:30:51.964118+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:30:51.964118+0000","last_undegraded":"2023-10-09T04:30:51.964118+0000","last_fullsized":"2023-10-09T04:30:51.964118+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"559'11659","reported_seq":16601,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.204563+0000","last_change":"2023-10-09T04:30:42.940632+0000","last_active":"2023-10-09T04:30:55.204563+0000","last_peered":"2023-10-09T04:30:55.204563+0000","last_clean":"2023-10-09T04:30:55.204563+0000","last_became_active":"2023-10-09T04:30:42.903426+0000","last_became_peered":"2023-10-09T04:30:42.903426+0000","last_unstale":"2023-10-09T04:30:55.204563+0000","last_undegraded":"2023-10-09T04:30:55.204563+0000","last_fullsized":"2023-10-09T04:30:55.204563+0000","mapping_epoch":552,"log_start":"523'5857","ondisk_log_start":"523'5857","created":507,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:04.590139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93913088,"num_objects":1433,"num_object_clones":0,"num_object_copies":4299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11659,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2429,"num_bytes_recovered":159186944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":1001,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976079+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:30:51.976079+0000","last_peered":"2023-10-09T04:30:51.976079+0000","last_clean":"2023-10-09T04:30:51.976079+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:30:51.976079+0000","last_undegraded":"2023-10-09T04:30:51.976079+0000","last_fullsized":"2023-10-09T04:30:51.976079+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":603,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976613+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:30:51.976613+0000","last_peered":"2023-10-09T04:30:51.976613+0000","last_clean":"2023-10-09T04:30:51.976613+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:30:51.976613+0000","last_undegraded":"2023-10-09T04:30:51.976613+0000","last_fullsized":"2023-10-09T04:30:51.976613+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":373,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975225+0000","last_change":"2023-10-09T04:29:44.198437+0000","last_active":"2023-10-09T04:30:51.975225+0000","last_peered":"2023-10-09T04:30:51.975225+0000","last_clean":"2023-10-09T04:30:51.975225+0000","last_became_active":"2023-10-09T04:29:44.198097+0000","last_became_peered":"2023-10-09T04:29:44.198097+0000","last_unstale":"2023-10-09T04:30:51.975225+0000","last_undegraded":"2023-10-09T04:30:51.975225+0000","last_fullsized":"2023-10-09T04:30:51.975225+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:36.976087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"560'11837","reported_seq":11899,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.974801+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:30:56.974801+0000","last_peered":"2023-10-09T04:30:56.974801+0000","last_clean":"2023-10-09T04:30:56.974801+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:30:56.974801+0000","last_undegraded":"2023-10-09T04:30:56.974801+0000","last_fullsized":"2023-10-09T04:30:56.974801+0000","mapping_epoch":507,"log_start":"523'5923","ondisk_log_start":"523'5923","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5914,"log_dups_size":0,"ondisk_log_size":5914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91947008,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11837,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":793,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.965165+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:30:51.965165+0000","last_peered":"2023-10-09T04:30:51.965165+0000","last_clean":"2023-10-09T04:30:51.965165+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:30:51.965165+0000","last_undegraded":"2023-10-09T04:30:51.965165+0000","last_fullsized":"2023-10-09T04:30:51.965165+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":593,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:52.027681+0000","last_change":"2023-10-09T04:30:52.027413+0000","last_active":"2023-10-09T04:30:52.027681+0000","last_peered":"2023-10-09T04:30:52.027681+0000","last_clean":"2023-10-09T04:30:52.027681+0000","last_became_active":"2023-10-09T04:30:51.982673+0000","last_became_peered":"2023-10-09T04:30:51.982673+0000","last_unstale":"2023-10-09T04:30:52.027681+0000","last_undegraded":"2023-10-09T04:30:52.027681+0000","last_fullsized":"2023-10-09T04:30:52.027681+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:22.580030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964767+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:30:51.964767+0000","last_peered":"2023-10-09T04:30:51.964767+0000","last_clean":"2023-10-09T04:30:51.964767+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:30:51.964767+0000","last_undegraded":"2023-10-09T04:30:51.964767+0000","last_fullsized":"2023-10-09T04:30:51.964767+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"559'11683","reported_seq":19410,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.703158+0000","last_change":"2023-10-09T04:30:17.183742+0000","last_active":"2023-10-09T04:30:55.703158+0000","last_peered":"2023-10-09T04:30:55.703158+0000","last_clean":"2023-10-09T04:30:55.703158+0000","last_became_active":"2023-10-09T04:30:17.165879+0000","last_became_peered":"2023-10-09T04:30:17.165879+0000","last_unstale":"2023-10-09T04:30:55.703158+0000","last_undegraded":"2023-10-09T04:30:55.703158+0000","last_fullsized":"2023-10-09T04:30:55.703158+0000","mapping_epoch":544,"log_start":"523'5948","ondisk_log_start":"523'5948","created":507,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:46.438208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92733440,"num_objects":1415,"num_object_clones":0,"num_object_copies":4245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11683,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3821,"num_bytes_recovered":250413056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":680,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966761+0000","last_change":"2023-10-09T04:29:47.252712+0000","last_active":"2023-10-09T04:30:51.966761+0000","last_peered":"2023-10-09T04:30:51.966761+0000","last_clean":"2023-10-09T04:30:51.966761+0000","last_became_active":"2023-10-09T04:29:47.252283+0000","last_became_peered":"2023-10-09T04:29:47.252283+0000","last_unstale":"2023-10-09T04:30:51.966761+0000","last_undegraded":"2023-10-09T04:30:51.966761+0000","last_fullsized":"2023-10-09T04:30:51.966761+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:09.123986+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":544,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976233+0000","last_change":"2023-10-09T04:29:44.201736+0000","last_active":"2023-10-09T04:30:51.976233+0000","last_peered":"2023-10-09T04:30:51.976233+0000","last_clean":"2023-10-09T04:30:51.976233+0000","last_became_active":"2023-10-09T04:29:44.196504+0000","last_became_peered":"2023-10-09T04:29:44.196504+0000","last_unstale":"2023-10-09T04:30:51.976233+0000","last_undegraded":"2023-10-09T04:30:51.976233+0000","last_fullsized":"2023-10-09T04:30:51.976233+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:53.990636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":185,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.969417+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:30:51.969417+0000","last_peered":"2023-10-09T04:30:51.969417+0000","last_clean":"2023-10-09T04:30:51.969417+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:30:51.969417+0000","last_undegraded":"2023-10-09T04:30:51.969417+0000","last_fullsized":"2023-10-09T04:30:51.969417+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"559'11523","reported_seq":11583,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.708905+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:30:55.708905+0000","last_peered":"2023-10-09T04:30:55.708905+0000","last_clean":"2023-10-09T04:30:55.708905+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:30:55.708905+0000","last_undegraded":"2023-10-09T04:30:55.708905+0000","last_fullsized":"2023-10-09T04:30:55.708905+0000","mapping_epoch":507,"log_start":"523'5799","ondisk_log_start":"523'5799","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87097344,"num_objects":1329,"num_object_clones":0,"num_object_copies":3987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11523,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":815,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966067+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:30:51.966067+0000","last_peered":"2023-10-09T04:30:51.966067+0000","last_clean":"2023-10-09T04:30:51.966067+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:30:51.966067+0000","last_undegraded":"2023-10-09T04:30:51.966067+0000","last_fullsized":"2023-10-09T04:30:51.966067+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":410,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964388+0000","last_change":"2023-10-09T04:29:44.187314+0000","last_active":"2023-10-09T04:30:51.964388+0000","last_peered":"2023-10-09T04:30:51.964388+0000","last_clean":"2023-10-09T04:30:51.964388+0000","last_became_active":"2023-10-09T04:29:44.186972+0000","last_became_peered":"2023-10-09T04:29:44.186972+0000","last_unstale":"2023-10-09T04:30:51.964388+0000","last_undegraded":"2023-10-09T04:30:51.964388+0000","last_fullsized":"2023-10-09T04:30:51.964388+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:36.570707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":184,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827760+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:30:53.827760+0000","last_peered":"2023-10-09T04:30:53.827760+0000","last_clean":"2023-10-09T04:30:53.827760+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:30:53.827760+0000","last_undegraded":"2023-10-09T04:30:53.827760+0000","last_fullsized":"2023-10-09T04:30:53.827760+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"559'11819","reported_seq":11879,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.702873+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:30:55.702873+0000","last_peered":"2023-10-09T04:30:55.702873+0000","last_clean":"2023-10-09T04:30:55.702873+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:30:55.702873+0000","last_undegraded":"2023-10-09T04:30:55.702873+0000","last_fullsized":"2023-10-09T04:30:55.702873+0000","mapping_epoch":507,"log_start":"524'6107","ondisk_log_start":"524'6107","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92340224,"num_objects":1409,"num_object_clones":0,"num_object_copies":4227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11819,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1022,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975028+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:30:51.975028+0000","last_peered":"2023-10-09T04:30:51.975028+0000","last_clean":"2023-10-09T04:30:51.975028+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:30:51.975028+0000","last_undegraded":"2023-10-09T04:30:51.975028+0000","last_fullsized":"2023-10-09T04:30:51.975028+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":196,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827719+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:30:53.827719+0000","last_peered":"2023-10-09T04:30:53.827719+0000","last_clean":"2023-10-09T04:30:53.827719+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:30:53.827719+0000","last_undegraded":"2023-10-09T04:30:53.827719+0000","last_fullsized":"2023-10-09T04:30:53.827719+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964538+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:30:51.964538+0000","last_peered":"2023-10-09T04:30:51.964538+0000","last_clean":"2023-10-09T04:30:51.964538+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:30:51.964538+0000","last_undegraded":"2023-10-09T04:30:51.964538+0000","last_fullsized":"2023-10-09T04:30:51.964538+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"560'11776","reported_seq":11839,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.867046+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:30:56.867046+0000","last_peered":"2023-10-09T04:30:56.867046+0000","last_clean":"2023-10-09T04:30:56.867046+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:30:56.867046+0000","last_undegraded":"2023-10-09T04:30:56.867046+0000","last_fullsized":"2023-10-09T04:30:56.867046+0000","mapping_epoch":507,"log_start":"523'5866","ondisk_log_start":"523'5866","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84934656,"num_objects":1296,"num_object_clones":0,"num_object_copies":3888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11776,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":710,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.822477+0000","last_change":"2023-10-09T04:29:47.267181+0000","last_active":"2023-10-09T04:30:53.822477+0000","last_peered":"2023-10-09T04:30:53.822477+0000","last_clean":"2023-10-09T04:30:53.822477+0000","last_became_active":"2023-10-09T04:29:47.266850+0000","last_became_peered":"2023-10-09T04:29:47.266850+0000","last_unstale":"2023-10-09T04:30:53.822477+0000","last_undegraded":"2023-10-09T04:30:53.822477+0000","last_fullsized":"2023-10-09T04:30:53.822477+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:20.803798+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964741+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:30:51.964741+0000","last_peered":"2023-10-09T04:30:51.964741+0000","last_clean":"2023-10-09T04:30:51.964741+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:30:51.964741+0000","last_undegraded":"2023-10-09T04:30:51.964741+0000","last_fullsized":"2023-10-09T04:30:51.964741+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":141,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966165+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:30:51.966165+0000","last_peered":"2023-10-09T04:30:51.966165+0000","last_clean":"2023-10-09T04:30:51.966165+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:30:51.966165+0000","last_undegraded":"2023-10-09T04:30:51.966165+0000","last_fullsized":"2023-10-09T04:30:51.966165+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"559'11775","reported_seq":11837,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.023086+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:30:56.023086+0000","last_peered":"2023-10-09T04:30:56.023086+0000","last_clean":"2023-10-09T04:30:56.023086+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:30:56.023086+0000","last_undegraded":"2023-10-09T04:30:56.023086+0000","last_fullsized":"2023-10-09T04:30:56.023086+0000","mapping_epoch":507,"log_start":"526'6491","ondisk_log_start":"526'6491","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5284,"log_dups_size":0,"ondisk_log_size":5284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87228416,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11775,"num_write_kb":419392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":684,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827805+0000","last_change":"2023-10-09T04:29:47.259563+0000","last_active":"2023-10-09T04:30:53.827805+0000","last_peered":"2023-10-09T04:30:53.827805+0000","last_clean":"2023-10-09T04:30:53.827805+0000","last_became_active":"2023-10-09T04:29:47.259210+0000","last_became_peered":"2023-10-09T04:29:47.259210+0000","last_unstale":"2023-10-09T04:30:53.827805+0000","last_undegraded":"2023-10-09T04:30:53.827805+0000","last_fullsized":"2023-10-09T04:30:53.827805+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:16.297729+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":144,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964840+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:30:51.964840+0000","last_peered":"2023-10-09T04:30:51.964840+0000","last_clean":"2023-10-09T04:30:51.964840+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:30:51.964840+0000","last_undegraded":"2023-10-09T04:30:51.964840+0000","last_fullsized":"2023-10-09T04:30:51.964840+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966140+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:30:51.966140+0000","last_peered":"2023-10-09T04:30:51.966140+0000","last_clean":"2023-10-09T04:30:51.966140+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:30:51.966140+0000","last_undegraded":"2023-10-09T04:30:51.966140+0000","last_fullsized":"2023-10-09T04:30:51.966140+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"560'12070","reported_seq":12133,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:59.334978+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:30:59.334978+0000","last_peered":"2023-10-09T04:30:59.334978+0000","last_clean":"2023-10-09T04:30:59.334978+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:30:59.334978+0000","last_undegraded":"2023-10-09T04:30:59.334978+0000","last_fullsized":"2023-10-09T04:30:59.334978+0000","mapping_epoch":507,"log_start":"526'6551","ondisk_log_start":"526'6551","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79298560,"num_objects":1210,"num_object_clones":0,"num_object_copies":3630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12070,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":698,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964347+0000","last_change":"2023-10-09T04:29:47.234552+0000","last_active":"2023-10-09T04:30:51.964347+0000","last_peered":"2023-10-09T04:30:51.964347+0000","last_clean":"2023-10-09T04:30:51.964347+0000","last_became_active":"2023-10-09T04:29:47.234172+0000","last_became_peered":"2023-10-09T04:29:47.234172+0000","last_unstale":"2023-10-09T04:30:51.964347+0000","last_undegraded":"2023-10-09T04:30:51.964347+0000","last_fullsized":"2023-10-09T04:30:51.964347+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:35.812833+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.829149+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:30:53.829149+0000","last_peered":"2023-10-09T04:30:53.829149+0000","last_clean":"2023-10-09T04:30:53.829149+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:30:53.829149+0000","last_undegraded":"2023-10-09T04:30:53.829149+0000","last_fullsized":"2023-10-09T04:30:53.829149+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":144,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.977724+0000","last_change":"2023-10-09T04:30:51.977312+0000","last_active":"2023-10-09T04:30:51.977724+0000","last_peered":"2023-10-09T04:30:51.977724+0000","last_clean":"2023-10-09T04:30:51.977724+0000","last_became_active":"2023-10-09T04:30:51.976877+0000","last_became_peered":"2023-10-09T04:30:51.976877+0000","last_unstale":"2023-10-09T04:30:51.977724+0000","last_undegraded":"2023-10-09T04:30:51.977724+0000","last_fullsized":"2023-10-09T04:30:51.977724+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:10.085422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"559'11728","reported_seq":11789,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.183088+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:30:55.183088+0000","last_peered":"2023-10-09T04:30:55.183088+0000","last_clean":"2023-10-09T04:30:55.183088+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:30:55.183088+0000","last_undegraded":"2023-10-09T04:30:55.183088+0000","last_fullsized":"2023-10-09T04:30:55.183088+0000","mapping_epoch":507,"log_start":"523'5843","ondisk_log_start":"523'5843","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89587736,"num_objects":1368,"num_object_clones":0,"num_object_copies":4104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1368,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11728,"num_write_kb":419009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":685,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.969775+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:30:51.969775+0000","last_peered":"2023-10-09T04:30:51.969775+0000","last_clean":"2023-10-09T04:30:51.969775+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:30:51.969775+0000","last_undegraded":"2023-10-09T04:30:51.969775+0000","last_fullsized":"2023-10-09T04:30:51.969775+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":164,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828102+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:30:53.828102+0000","last_peered":"2023-10-09T04:30:53.828102+0000","last_clean":"2023-10-09T04:30:53.828102+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:30:53.828102+0000","last_undegraded":"2023-10-09T04:30:53.828102+0000","last_fullsized":"2023-10-09T04:30:53.828102+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":515,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966192+0000","last_change":"2023-10-09T04:29:44.187226+0000","last_active":"2023-10-09T04:30:51.966192+0000","last_peered":"2023-10-09T04:30:51.966192+0000","last_clean":"2023-10-09T04:30:51.966192+0000","last_became_active":"2023-10-09T04:29:44.186899+0000","last_became_peered":"2023-10-09T04:29:44.186899+0000","last_unstale":"2023-10-09T04:30:51.966192+0000","last_undegraded":"2023-10-09T04:30:51.966192+0000","last_fullsized":"2023-10-09T04:30:51.966192+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:21.406947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":373,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964572+0000","last_change":"2023-10-09T04:29:44.180975+0000","last_active":"2023-10-09T04:30:51.964572+0000","last_peered":"2023-10-09T04:30:51.964572+0000","last_clean":"2023-10-09T04:30:51.964572+0000","last_became_active":"2023-10-09T04:29:44.180615+0000","last_became_peered":"2023-10-09T04:29:44.180615+0000","last_unstale":"2023-10-09T04:30:51.964572+0000","last_undegraded":"2023-10-09T04:30:51.964572+0000","last_fullsized":"2023-10-09T04:30:51.964572+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:00.181777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"560'12050","reported_seq":12112,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.966830+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:30:56.966830+0000","last_peered":"2023-10-09T04:30:56.966830+0000","last_clean":"2023-10-09T04:30:56.966830+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:30:56.966830+0000","last_undegraded":"2023-10-09T04:30:56.966830+0000","last_fullsized":"2023-10-09T04:30:56.966830+0000","mapping_epoch":507,"log_start":"524'6151","ondisk_log_start":"524'6151","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89915392,"num_objects":1372,"num_object_clones":0,"num_object_copies":4116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12050,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":604,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.829147+0000","last_change":"2023-10-09T04:29:47.257461+0000","last_active":"2023-10-09T04:30:53.829147+0000","last_peered":"2023-10-09T04:30:53.829147+0000","last_clean":"2023-10-09T04:30:53.829147+0000","last_became_active":"2023-10-09T04:29:47.257014+0000","last_became_peered":"2023-10-09T04:29:47.257014+0000","last_unstale":"2023-10-09T04:30:53.829147+0000","last_undegraded":"2023-10-09T04:30:53.829147+0000","last_fullsized":"2023-10-09T04:30:53.829147+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:52.834528+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":36,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828740+0000","last_change":"2023-10-09T04:29:44.191398+0000","last_active":"2023-10-09T04:30:53.828740+0000","last_peered":"2023-10-09T04:30:53.828740+0000","last_clean":"2023-10-09T04:30:53.828740+0000","last_became_active":"2023-10-09T04:29:44.191063+0000","last_became_peered":"2023-10-09T04:29:44.191063+0000","last_unstale":"2023-10-09T04:30:53.828740+0000","last_undegraded":"2023-10-09T04:30:53.828740+0000","last_fullsized":"2023-10-09T04:30:53.828740+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:56.910914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964753+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:30:51.964753+0000","last_peered":"2023-10-09T04:30:51.964753+0000","last_clean":"2023-10-09T04:30:51.964753+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:30:51.964753+0000","last_undegraded":"2023-10-09T04:30:51.964753+0000","last_fullsized":"2023-10-09T04:30:51.964753+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827952+0000","last_change":"2023-10-09T04:30:51.977416+0000","last_active":"2023-10-09T04:30:53.827952+0000","last_peered":"2023-10-09T04:30:53.827952+0000","last_clean":"2023-10-09T04:30:53.827952+0000","last_became_active":"2023-10-09T04:30:51.977079+0000","last_became_peered":"2023-10-09T04:30:51.977079+0000","last_unstale":"2023-10-09T04:30:53.827952+0000","last_undegraded":"2023-10-09T04:30:53.827952+0000","last_fullsized":"2023-10-09T04:30:53.827952+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:06.715973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"560'11494","reported_seq":18476,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.975734+0000","last_change":"2023-10-09T04:30:26.112157+0000","last_active":"2023-10-09T04:30:56.975734+0000","last_peered":"2023-10-09T04:30:56.975734+0000","last_clean":"2023-10-09T04:30:56.975734+0000","last_became_active":"2023-10-09T04:30:26.074223+0000","last_became_peered":"2023-10-09T04:30:26.074223+0000","last_unstale":"2023-10-09T04:30:56.975734+0000","last_undegraded":"2023-10-09T04:30:56.975734+0000","last_fullsized":"2023-10-09T04:30:56.975734+0000","mapping_epoch":547,"log_start":"522'5584","ondisk_log_start":"522'5584","created":507,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:14.191269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89128960,"num_objects":1360,"num_object_clones":0,"num_object_copies":4080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11494,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3451,"num_bytes_recovered":226164736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":735,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828657+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:30:53.828657+0000","last_peered":"2023-10-09T04:30:53.828657+0000","last_clean":"2023-10-09T04:30:53.828657+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:30:53.828657+0000","last_undegraded":"2023-10-09T04:30:53.828657+0000","last_fullsized":"2023-10-09T04:30:53.828657+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":412,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832682+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:30:53.832682+0000","last_peered":"2023-10-09T04:30:53.832682+0000","last_clean":"2023-10-09T04:30:53.832682+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:30:53.832682+0000","last_undegraded":"2023-10-09T04:30:53.832682+0000","last_fullsized":"2023-10-09T04:30:53.832682+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":501,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827264+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:30:53.827264+0000","last_peered":"2023-10-09T04:30:53.827264+0000","last_clean":"2023-10-09T04:30:53.827264+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:30:53.827264+0000","last_undegraded":"2023-10-09T04:30:53.827264+0000","last_fullsized":"2023-10-09T04:30:53.827264+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":154,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828784+0000","last_change":"2023-10-09T04:29:44.189319+0000","last_active":"2023-10-09T04:30:53.828784+0000","last_peered":"2023-10-09T04:30:53.828784+0000","last_clean":"2023-10-09T04:30:53.828784+0000","last_became_active":"2023-10-09T04:29:44.188987+0000","last_became_peered":"2023-10-09T04:29:44.188987+0000","last_unstale":"2023-10-09T04:30:53.828784+0000","last_undegraded":"2023-10-09T04:30:53.828784+0000","last_fullsized":"2023-10-09T04:30:53.828784+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:02.006532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"559'11598","reported_seq":16140,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.040883+0000","last_change":"2023-10-09T04:30:44.959704+0000","last_active":"2023-10-09T04:30:56.040883+0000","last_peered":"2023-10-09T04:30:56.040883+0000","last_clean":"2023-10-09T04:30:56.040883+0000","last_became_active":"2023-10-09T04:30:44.926653+0000","last_became_peered":"2023-10-09T04:30:44.926653+0000","last_unstale":"2023-10-09T04:30:56.040883+0000","last_undegraded":"2023-10-09T04:30:56.040883+0000","last_fullsized":"2023-10-09T04:30:56.040883+0000","mapping_epoch":554,"log_start":"525'6242","ondisk_log_start":"525'6242","created":507,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5356,"log_dups_size":0,"ondisk_log_size":5356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.880322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89522176,"num_objects":1366,"num_object_clones":0,"num_object_copies":4098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11598,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2230,"num_bytes_recovered":146145280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":503,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975360+0000","last_change":"2023-10-09T04:29:44.188178+0000","last_active":"2023-10-09T04:30:51.975360+0000","last_peered":"2023-10-09T04:30:51.975360+0000","last_clean":"2023-10-09T04:30:51.975360+0000","last_became_active":"2023-10-09T04:29:44.187626+0000","last_became_peered":"2023-10-09T04:29:44.187626+0000","last_unstale":"2023-10-09T04:30:51.975360+0000","last_undegraded":"2023-10-09T04:30:51.975360+0000","last_fullsized":"2023-10-09T04:30:51.975360+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:01.695892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":1013,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975145+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:30:51.975145+0000","last_peered":"2023-10-09T04:30:51.975145+0000","last_clean":"2023-10-09T04:30:51.975145+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:30:51.975145+0000","last_undegraded":"2023-10-09T04:30:51.975145+0000","last_fullsized":"2023-10-09T04:30:51.975145+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":34,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964258+0000","last_change":"2023-10-09T04:29:44.195422+0000","last_active":"2023-10-09T04:30:51.964258+0000","last_peered":"2023-10-09T04:30:51.964258+0000","last_clean":"2023-10-09T04:30:51.964258+0000","last_became_active":"2023-10-09T04:29:44.194489+0000","last_became_peered":"2023-10-09T04:29:44.194489+0000","last_unstale":"2023-10-09T04:30:51.964258+0000","last_undegraded":"2023-10-09T04:30:51.964258+0000","last_fullsized":"2023-10-09T04:30:51.964258+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:33.263534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":121,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966008+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:30:51.966008+0000","last_peered":"2023-10-09T04:30:51.966008+0000","last_clean":"2023-10-09T04:30:51.966008+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:30:51.966008+0000","last_undegraded":"2023-10-09T04:30:51.966008+0000","last_fullsized":"2023-10-09T04:30:51.966008+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"560'11726","reported_seq":11789,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.837461+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:30:56.837461+0000","last_peered":"2023-10-09T04:30:56.837461+0000","last_clean":"2023-10-09T04:30:56.837461+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:30:56.837461+0000","last_undegraded":"2023-10-09T04:30:56.837461+0000","last_fullsized":"2023-10-09T04:30:56.837461+0000","mapping_epoch":507,"log_start":"523'5834","ondisk_log_start":"523'5834","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721088,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11726,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":654,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964796+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:30:51.964796+0000","last_peered":"2023-10-09T04:30:51.964796+0000","last_clean":"2023-10-09T04:30:51.964796+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:30:51.964796+0000","last_undegraded":"2023-10-09T04:30:51.964796+0000","last_fullsized":"2023-10-09T04:30:51.964796+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":559,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975065+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:30:51.975065+0000","last_peered":"2023-10-09T04:30:51.975065+0000","last_clean":"2023-10-09T04:30:51.975065+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:30:51.975065+0000","last_undegraded":"2023-10-09T04:30:51.975065+0000","last_fullsized":"2023-10-09T04:30:51.975065+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":202,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832991+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:30:53.832991+0000","last_peered":"2023-10-09T04:30:53.832991+0000","last_clean":"2023-10-09T04:30:53.832991+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:30:53.832991+0000","last_undegraded":"2023-10-09T04:30:53.832991+0000","last_fullsized":"2023-10-09T04:30:53.832991+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":184,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828292+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:30:53.828292+0000","last_peered":"2023-10-09T04:30:53.828292+0000","last_clean":"2023-10-09T04:30:53.828292+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:30:53.828292+0000","last_undegraded":"2023-10-09T04:30:53.828292+0000","last_fullsized":"2023-10-09T04:30:53.828292+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"559'11789","reported_seq":11849,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.218477+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:30:55.218477+0000","last_peered":"2023-10-09T04:30:55.218477+0000","last_clean":"2023-10-09T04:30:55.218477+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:30:55.218477+0000","last_undegraded":"2023-10-09T04:30:55.218477+0000","last_fullsized":"2023-10-09T04:30:55.218477+0000","mapping_epoch":507,"log_start":"523'5991","ondisk_log_start":"523'5991","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90505216,"num_objects":1381,"num_object_clones":0,"num_object_copies":4143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11789,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":677,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.969554+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:30:51.969554+0000","last_peered":"2023-10-09T04:30:51.969554+0000","last_clean":"2023-10-09T04:30:51.969554+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:30:51.969554+0000","last_undegraded":"2023-10-09T04:30:51.969554+0000","last_fullsized":"2023-10-09T04:30:51.969554+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":582,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976307+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:30:51.976307+0000","last_peered":"2023-10-09T04:30:51.976307+0000","last_clean":"2023-10-09T04:30:51.976307+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:30:51.976307+0000","last_undegraded":"2023-10-09T04:30:51.976307+0000","last_fullsized":"2023-10-09T04:30:51.976307+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":36,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828005+0000","last_change":"2023-10-09T04:29:44.178854+0000","last_active":"2023-10-09T04:30:53.828005+0000","last_peered":"2023-10-09T04:30:53.828005+0000","last_clean":"2023-10-09T04:30:53.828005+0000","last_became_active":"2023-10-09T04:29:44.178554+0000","last_became_peered":"2023-10-09T04:29:44.178554+0000","last_unstale":"2023-10-09T04:30:53.828005+0000","last_undegraded":"2023-10-09T04:30:53.828005+0000","last_fullsized":"2023-10-09T04:30:53.828005+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:56.730403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827836+0000","last_change":"2023-10-09T04:29:44.189508+0000","last_active":"2023-10-09T04:30:53.827836+0000","last_peered":"2023-10-09T04:30:53.827836+0000","last_clean":"2023-10-09T04:30:53.827836+0000","last_became_active":"2023-10-09T04:29:44.189002+0000","last_became_peered":"2023-10-09T04:29:44.189002+0000","last_unstale":"2023-10-09T04:30:53.827836+0000","last_undegraded":"2023-10-09T04:30:53.827836+0000","last_fullsized":"2023-10-09T04:30:53.827836+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:44.869908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"559'11755","reported_seq":11815,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.678745+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:30:55.678745+0000","last_peered":"2023-10-09T04:30:55.678745+0000","last_clean":"2023-10-09T04:30:55.678745+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:30:55.678745+0000","last_undegraded":"2023-10-09T04:30:55.678745+0000","last_fullsized":"2023-10-09T04:30:55.678745+0000","mapping_epoch":507,"log_start":"524'6045","ondisk_log_start":"524'6045","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5710,"log_dups_size":0,"ondisk_log_size":5710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89849856,"num_objects":1371,"num_object_clones":0,"num_object_copies":4113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11755,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":616,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966086+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:30:51.966086+0000","last_peered":"2023-10-09T04:30:51.966086+0000","last_clean":"2023-10-09T04:30:51.966086+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:30:51.966086+0000","last_undegraded":"2023-10-09T04:30:51.966086+0000","last_fullsized":"2023-10-09T04:30:51.966086+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":519,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976794+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:30:51.976794+0000","last_peered":"2023-10-09T04:30:51.976794+0000","last_clean":"2023-10-09T04:30:51.976794+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:30:51.976794+0000","last_undegraded":"2023-10-09T04:30:51.976794+0000","last_fullsized":"2023-10-09T04:30:51.976794+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.966208+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:30:51.966208+0000","last_peered":"2023-10-09T04:30:51.966208+0000","last_clean":"2023-10-09T04:30:51.966208+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:30:51.966208+0000","last_undegraded":"2023-10-09T04:30:51.966208+0000","last_fullsized":"2023-10-09T04:30:51.966208+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":467,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.976435+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:30:51.976435+0000","last_peered":"2023-10-09T04:30:51.976435+0000","last_clean":"2023-10-09T04:30:51.976435+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:30:51.976435+0000","last_undegraded":"2023-10-09T04:30:51.976435+0000","last_fullsized":"2023-10-09T04:30:51.976435+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"559'11540","reported_seq":11600,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:55.204407+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:30:55.204407+0000","last_peered":"2023-10-09T04:30:55.204407+0000","last_clean":"2023-10-09T04:30:55.204407+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:30:55.204407+0000","last_undegraded":"2023-10-09T04:30:55.204407+0000","last_fullsized":"2023-10-09T04:30:55.204407+0000","mapping_epoch":507,"log_start":"523'5660","ondisk_log_start":"523'5660","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5880,"log_dups_size":0,"ondisk_log_size":5880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90439680,"num_objects":1380,"num_object_clones":0,"num_object_copies":4140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11540,"num_write_kb":413440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":753,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.965078+0000","last_change":"2023-10-09T04:29:47.253692+0000","last_active":"2023-10-09T04:30:51.965078+0000","last_peered":"2023-10-09T04:30:51.965078+0000","last_clean":"2023-10-09T04:30:51.965078+0000","last_became_active":"2023-10-09T04:29:47.253340+0000","last_became_peered":"2023-10-09T04:29:47.253340+0000","last_unstale":"2023-10-09T04:30:51.965078+0000","last_undegraded":"2023-10-09T04:30:51.965078+0000","last_fullsized":"2023-10-09T04:30:51.965078+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:17.491055+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":358,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.969903+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:30:51.969903+0000","last_peered":"2023-10-09T04:30:51.969903+0000","last_clean":"2023-10-09T04:30:51.969903+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:30:51.969903+0000","last_undegraded":"2023-10-09T04:30:51.969903+0000","last_fullsized":"2023-10-09T04:30:51.969903+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":196,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828477+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:30:53.828477+0000","last_peered":"2023-10-09T04:30:53.828477+0000","last_clean":"2023-10-09T04:30:53.828477+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:30:53.828477+0000","last_undegraded":"2023-10-09T04:30:53.828477+0000","last_fullsized":"2023-10-09T04:30:53.828477+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964695+0000","last_change":"2023-10-09T04:29:44.183037+0000","last_active":"2023-10-09T04:30:51.964695+0000","last_peered":"2023-10-09T04:30:51.964695+0000","last_clean":"2023-10-09T04:30:51.964695+0000","last_became_active":"2023-10-09T04:29:44.182667+0000","last_became_peered":"2023-10-09T04:29:44.182667+0000","last_unstale":"2023-10-09T04:30:51.964695+0000","last_undegraded":"2023-10-09T04:30:51.964695+0000","last_fullsized":"2023-10-09T04:30:51.964695+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:35.337400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":604,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832756+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:30:53.832756+0000","last_peered":"2023-10-09T04:30:53.832756+0000","last_clean":"2023-10-09T04:30:53.832756+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:30:53.832756+0000","last_undegraded":"2023-10-09T04:30:53.832756+0000","last_fullsized":"2023-10-09T04:30:53.832756+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":193,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827920+0000","last_change":"2023-10-09T04:30:51.979500+0000","last_active":"2023-10-09T04:30:53.827920+0000","last_peered":"2023-10-09T04:30:53.827920+0000","last_clean":"2023-10-09T04:30:53.827920+0000","last_became_active":"2023-10-09T04:30:51.979149+0000","last_became_peered":"2023-10-09T04:30:51.979149+0000","last_unstale":"2023-10-09T04:30:53.827920+0000","last_undegraded":"2023-10-09T04:30:53.827920+0000","last_fullsized":"2023-10-09T04:30:53.827920+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:52.702260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.823737+0000","last_change":"2023-10-09T04:30:51.973822+0000","last_active":"2023-10-09T04:30:53.823737+0000","last_peered":"2023-10-09T04:30:53.823737+0000","last_clean":"2023-10-09T04:30:53.823737+0000","last_became_active":"2023-10-09T04:30:51.973546+0000","last_became_peered":"2023-10-09T04:30:51.973546+0000","last_unstale":"2023-10-09T04:30:53.823737+0000","last_undegraded":"2023-10-09T04:30:53.823737+0000","last_fullsized":"2023-10-09T04:30:53.823737+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:42.932321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":633,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.969464+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:30:51.969464+0000","last_peered":"2023-10-09T04:30:51.969464+0000","last_clean":"2023-10-09T04:30:51.969464+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:30:51.969464+0000","last_undegraded":"2023-10-09T04:30:51.969464+0000","last_fullsized":"2023-10-09T04:30:51.969464+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":490,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964636+0000","last_change":"2023-10-09T04:29:44.199915+0000","last_active":"2023-10-09T04:30:51.964636+0000","last_peered":"2023-10-09T04:30:51.964636+0000","last_clean":"2023-10-09T04:30:51.964636+0000","last_became_active":"2023-10-09T04:29:44.188601+0000","last_became_peered":"2023-10-09T04:29:44.188601+0000","last_unstale":"2023-10-09T04:30:51.964636+0000","last_undegraded":"2023-10-09T04:30:51.964636+0000","last_fullsized":"2023-10-09T04:30:51.964636+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:10.093968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":124,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964678+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:30:51.964678+0000","last_peered":"2023-10-09T04:30:51.964678+0000","last_clean":"2023-10-09T04:30:51.964678+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:30:51.964678+0000","last_undegraded":"2023-10-09T04:30:51.964678+0000","last_fullsized":"2023-10-09T04:30:51.964678+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":501,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.965840+0000","last_change":"2023-10-09T04:29:44.203641+0000","last_active":"2023-10-09T04:30:51.965840+0000","last_peered":"2023-10-09T04:30:51.965840+0000","last_clean":"2023-10-09T04:30:51.965840+0000","last_became_active":"2023-10-09T04:29:44.194615+0000","last_became_peered":"2023-10-09T04:29:44.194615+0000","last_unstale":"2023-10-09T04:30:51.965840+0000","last_undegraded":"2023-10-09T04:30:51.965840+0000","last_fullsized":"2023-10-09T04:30:51.965840+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:30.344525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":61,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964426+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:30:51.964426+0000","last_peered":"2023-10-09T04:30:51.964426+0000","last_clean":"2023-10-09T04:30:51.964426+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:30:51.964426+0000","last_undegraded":"2023-10-09T04:30:51.964426+0000","last_fullsized":"2023-10-09T04:30:51.964426+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":812,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.975628+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:30:51.975628+0000","last_peered":"2023-10-09T04:30:51.975628+0000","last_clean":"2023-10-09T04:30:51.975628+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:30:51.975628+0000","last_undegraded":"2023-10-09T04:30:51.975628+0000","last_fullsized":"2023-10-09T04:30:51.975628+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":294,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.964777+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:30:51.964777+0000","last_peered":"2023-10-09T04:30:51.964777+0000","last_clean":"2023-10-09T04:30:51.964777+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:30:51.964777+0000","last_undegraded":"2023-10-09T04:30:51.964777+0000","last_fullsized":"2023-10-09T04:30:51.964777+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":411,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832609+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:30:53.832609+0000","last_peered":"2023-10-09T04:30:53.832609+0000","last_clean":"2023-10-09T04:30:53.832609+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:30:53.832609+0000","last_undegraded":"2023-10-09T04:30:53.832609+0000","last_fullsized":"2023-10-09T04:30:53.832609+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":670,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.965929+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:30:51.965929+0000","last_peered":"2023-10-09T04:30:51.965929+0000","last_clean":"2023-10-09T04:30:51.965929+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:30:51.965929+0000","last_undegraded":"2023-10-09T04:30:51.965929+0000","last_fullsized":"2023-10-09T04:30:51.965929+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":203,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828963+0000","last_change":"2023-10-09T04:29:44.196276+0000","last_active":"2023-10-09T04:30:53.828963+0000","last_peered":"2023-10-09T04:30:53.828963+0000","last_clean":"2023-10-09T04:30:53.828963+0000","last_became_active":"2023-10-09T04:29:44.187018+0000","last_became_peered":"2023-10-09T04:29:44.187018+0000","last_unstale":"2023-10-09T04:30:53.828963+0000","last_undegraded":"2023-10-09T04:30:53.828963+0000","last_fullsized":"2023-10-09T04:30:53.828963+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:41.215750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":134,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-09T04:30:51.979310+0000","last_change":"2023-10-09T04:30:51.978882+0000","last_active":"2023-10-09T04:30:51.979310+0000","last_peered":"2023-10-09T04:30:51.979310+0000","last_clean":"2023-10-09T04:30:51.979310+0000","last_became_active":"2023-10-09T04:30:51.977708+0000","last_became_peered":"2023-10-09T04:30:51.977708+0000","last_unstale":"2023-10-09T04:30:51.979310+0000","last_undegraded":"2023-10-09T04:30:51.979310+0000","last_fullsized":"2023-10-09T04:30:51.979310+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:47.210981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1424162840,"num_objects":21732,"num_object_clones":0,"num_object_copies":65196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21732,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187822,"num_write_kb":6705665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11931,"num_bytes_recovered":781910016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4600967168,"data_stored":4600954952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91930,"ondisk_log_size":91930,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":12238,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1391225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201741,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1457232,"kb_used_data":553224,"kb_used_omap":0,"kb_used_meta":904000,"kb_avail":92287920,"statfs":{"total":95995035648,"available":94502830080,"internally_reserved":0,"allocated":566501376,"data_stored":564949268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925696000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:57 2023","interfaces":[{"interface":"back","average":{"1min":7.524,"5min":3.246,"15min":3.1840000000000002},"min":{"1min":0.23899999999999999,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":72.108999999999995,"5min":72.108999999999995,"15min":148.65899999999999},"last":2.2829999999999999},{"interface":"front","average":{"1min":6.6120000000000001,"5min":2.9590000000000001,"15min":3.1309999999999998},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.221},"max":{"1min":72.161000000000001,"5min":72.161000000000001,"15min":148.642},"last":2.9769999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":5.6619999999999999,"15min":6.5970000000000004},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.22600000000000001},"max":{"1min":16.129000000000001,"5min":213.41800000000001,"15min":261.935},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.873,"5min":5.2249999999999996,"15min":6.3550000000000004},"min":{"1min":0.29699999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.2969999999999997,"5min":213.352,"15min":262.07600000000002},"last":1.431}]},{"osd":2,"last update":"Mon Oct 9 04:29:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8980000000000001,"5min":1.706,"15min":2.6890000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":37.131999999999998,"5min":37.131999999999998,"15min":253.41200000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.839,"5min":1.615,"15min":3.008},"min":{"1min":0.35899999999999999,"5min":0.26800000000000002,"15min":0.246},"max":{"1min":37.145000000000003,"5min":37.145000000000003,"15min":253.39400000000001},"last":0.504}]},{"osd":3,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":5.7809999999999997,"5min":3.6659999999999999,"15min":3.464},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":77.245000000000005,"5min":77.245000000000005,"15min":255.97300000000001},"last":1.145},{"interface":"front","average":{"1min":5.3540000000000001,"5min":3.5489999999999999,"15min":3.6339999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":77.263999999999996,"5min":77.263999999999996,"15min":256.06099999999998},"last":1.109}]},{"osd":4,"last update":"Mon Oct 9 04:29:57 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.98299999999999998,"15min":0.88700000000000001},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8380000000000001,"5min":9.3480000000000008,"15min":9.3480000000000008},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.1040000000000001,"15min":1.2410000000000001},"min":{"1min":0.29699999999999999,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":3.915,"5min":6.1299999999999999,"15min":12.566000000000001},"last":3.0190000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:29:57 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":1.2290000000000001,"15min":1.804},"min":{"1min":0.28899999999999998,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":3.3700000000000001,"5min":7.4820000000000002,"15min":120.066},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.1200000000000001,"15min":1.5920000000000001},"min":{"1min":0.31900000000000001,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":2.9100000000000001,"5min":7.8360000000000003,"15min":120.09699999999999},"last":2.3959999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:29:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.3660000000000001,"15min":1.3700000000000001},"min":{"1min":0.318,"5min":0.31,"15min":0.307},"max":{"1min":5.2800000000000002,"5min":9.5069999999999997,"15min":9.5069999999999997},"last":0.437},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.2410000000000001,"15min":1.081},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":4.7380000000000004,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.45200000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463377,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1644676,"kb_used_data":745148,"kb_used_omap":0,"kb_used_meta":899520,"kb_avail":92100476,"statfs":{"total":95995035648,"available":94310887424,"internally_reserved":0,"allocated":763031552,"data_stored":761491297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921108480},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:29:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":5.6029999999999998,"15min":4.5979999999999999},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":21.004999999999999,"5min":162.524,"15min":162.524},"last":0.28199999999999997},{"interface":"front","average":{"1min":3.0459999999999998,"5min":5.577,"15min":4.601},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.218},"max":{"1min":20.649999999999999,"5min":161.73599999999999,"15min":161.73599999999999},"last":0.26500000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":1.972,"15min":4.4710000000000001},"min":{"1min":0.24299999999999999,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":10.055,"5min":18.971,"15min":315.73599999999999},"last":2.2930000000000001},{"interface":"front","average":{"1min":3.1269999999999998,"5min":2.004,"15min":4.5510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.603999999999999,"5min":18.943999999999999,"15min":316.91500000000002},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 04:29:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":4.2080000000000002,"15min":2.6509999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.74,"5min":116.084,"15min":116.084},"last":3.339},{"interface":"front","average":{"1min":2.6200000000000001,"5min":4.258,"15min":2.694},"min":{"1min":0.33100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.772,"5min":116.11199999999999,"15min":116.11199999999999},"last":3.3730000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":8.1199999999999992,"5min":2.8100000000000001,"15min":2.331},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.25600000000000001},"max":{"1min":127.44199999999999,"5min":127.44199999999999,"15min":127.44199999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":9.3219999999999992,"5min":3.0379999999999998,"15min":2.3969999999999998},"min":{"1min":0.34200000000000003,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":127.51900000000001,"5min":127.51900000000001,"15min":127.51900000000001},"last":4.2149999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.0029999999999999,"15min":1.532},"min":{"1min":0.27600000000000002,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":5.5659999999999998,"5min":17.545000000000002,"15min":161.27500000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.1439999999999999,"15min":1.637},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.22600000000000001},"max":{"1min":3.0699999999999998,"5min":15.212999999999999,"15min":160.32599999999999},"last":0.33200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.6839999999999999,"15min":1.306},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26100000000000001},"max":{"1min":2.7959999999999998,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.441},{"interface":"front","average":{"1min":0.875,"5min":1.7989999999999999,"15min":1.3280000000000001},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.27900000000000003},"max":{"1min":2.8180000000000001,"5min":74.073999999999998,"15min":74.073999999999998},"last":0.35999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.92600000000000005,"15min":1.6319999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7480000000000002,"5min":4.6189999999999998,"15min":161.39400000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.234,"15min":1.722},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":2.6520000000000001,"5min":10.77,"15min":161.41200000000001},"last":0.42099999999999999}]}]},{"osd":1,"up_from":89,"seq":382252089687,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621040,"kb_used_data":725992,"kb_used_omap":9,"kb_used_meta":895030,"kb_avail":92124112,"statfs":{"total":95995035648,"available":94335090688,"internally_reserved":0,"allocated":743415808,"data_stored":741035033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":916511721},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":3.5609999999999999,"15min":4.3239999999999998},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":33.351999999999997,"5min":86.963999999999999,"15min":259.26799999999997},"last":0.70999999999999996},{"interface":"front","average":{"1min":5.4699999999999998,"5min":3.988,"15min":5.3090000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":35.271000000000001,"5min":83.213999999999999,"15min":258.46899999999999},"last":8.8480000000000008}]},{"osd":2,"last update":"Mon Oct 9 04:30:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":2.1659999999999999,"15min":3.1219999999999999},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.919,"5min":74.474000000000004,"15min":259.315},"last":7.0919999999999996},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.3719999999999999,"15min":3.484},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.718,"5min":75.653000000000006,"15min":259.733},"last":5.8029999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":14.018000000000001,"5min":4.8899999999999997,"15min":3.242},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.22600000000000001},"max":{"1min":156.261,"5min":156.261,"15min":156.261},"last":0.77100000000000002},{"interface":"front","average":{"1min":12.929,"5min":5.1210000000000004,"15min":3.6000000000000001},"min":{"1min":0.36299999999999999,"5min":0.27700000000000002,"15min":0.24299999999999999},"max":{"1min":155.684,"5min":155.684,"15min":155.684},"last":7.8869999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":2.028,"15min":2.9860000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26900000000000002},"max":{"1min":4.8170000000000002,"5min":39.052,"15min":257.83300000000003},"last":2.4849999999999999},{"interface":"front","average":{"1min":2.778,"5min":2.0880000000000001,"15min":2.7839999999999998},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":14.32,"5min":39.040999999999997,"15min":257.863},"last":4.8970000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.6619999999999999,"15min":2.1419999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":4.8330000000000002,"5min":17.751999999999999,"15min":38.848999999999997},"last":2.5179999999999998},{"interface":"front","average":{"1min":2.694,"5min":1.99,"15min":2.2799999999999998},"min":{"1min":0.44400000000000001,"5min":0.35699999999999998,"15min":0.29799999999999999},"max":{"1min":14.356999999999999,"5min":14.356999999999999,"15min":39.593000000000004},"last":4.944}]},{"osd":6,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9740000000000002,"5min":2.8180000000000001,"15min":2.1659999999999999},"min":{"1min":0.46800000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.406000000000001,"5min":50.814999999999998,"15min":50.814999999999998},"last":4.851},{"interface":"front","average":{"1min":3.161,"5min":2.46,"15min":1.915},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29999999999999999},"max":{"1min":13.026,"5min":50.749000000000002,"15min":50.749000000000002},"last":0.745}]},{"osd":7,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":2.988,"5min":2.4180000000000001,"15min":3.887},"min":{"1min":0.499,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":14.382999999999999,"5min":36.765999999999998,"15min":259.08499999999998},"last":4.9219999999999997},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.1000000000000001,"15min":3.5990000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33100000000000002},"max":{"1min":12.994,"5min":36.75,"15min":259.13299999999998},"last":0.79200000000000004}]}]},{"osd":0,"up_from":84,"seq":360777253212,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1486604,"kb_used_data":623172,"kb_used_omap":28,"kb_used_meta":863395,"kb_avail":92258548,"statfs":{"total":95995035648,"available":94472753152,"internally_reserved":0,"allocated":638128128,"data_stored":635600134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":884117328},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:30:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.6930000000000001,"15min":4.4560000000000004},"min":{"1min":0.193,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":12.641999999999999,"5min":12.641999999999999,"15min":209.215},"last":1.448},{"interface":"front","average":{"1min":2.141,"5min":1.615,"15min":4.157},"min":{"1min":0.20599999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.7220000000000004,"5min":16.131,"15min":120.934},"last":4.5179999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":1.5580000000000001,"15min":2.5699999999999998},"min":{"1min":0.247,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":12.036,"5min":17.562999999999999,"15min":88.088999999999999},"last":0.217},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.653,"15min":2.9449999999999998},"min":{"1min":0.32000000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.8539999999999992,"5min":9.9659999999999993,"15min":88.004999999999995},"last":4.923}]},{"osd":3,"last update":"Mon Oct 9 04:30:29 2023","interfaces":[{"interface":"back","average":{"1min":4.327,"5min":2.0019999999999998,"15min":2.7149999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":19.507999999999999,"5min":19.507999999999999,"15min":99.709000000000003},"last":0.60299999999999998},{"interface":"front","average":{"1min":4.1769999999999996,"5min":2.286,"15min":2.8109999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":19.741,"5min":19.741,"15min":99.560000000000002},"last":0.253}]},{"osd":4,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.1320000000000001,"15min":2.5089999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28899999999999998},"max":{"1min":7.9770000000000003,"5min":86.906999999999996,"15min":88.058000000000007},"last":0.308},{"interface":"front","average":{"1min":1.8700000000000001,"5min":2.2519999999999998,"15min":2.4830000000000001},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.24199999999999999},"max":{"1min":5.625,"5min":86.891000000000005,"15min":86.891000000000005},"last":3.371}]},{"osd":5,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":3.7869999999999999,"15min":3.855},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":13.198,"5min":134.93000000000001,"15min":156.96100000000001},"last":3.258},{"interface":"front","average":{"1min":2.9009999999999998,"5min":3.5960000000000001,"15min":3.3879999999999999},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":8.5399999999999991,"5min":135.04599999999999,"15min":156.97300000000001},"last":0.442}]},{"osd":6,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":2.698,"5min":1.5169999999999999,"15min":1.8759999999999999},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.34300000000000003},"max":{"1min":8.4920000000000009,"5min":8.4920000000000009,"15min":63.468000000000004},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.4910000000000001,"15min":2.1749999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34100000000000003},"max":{"1min":11.052,"5min":12.26,"15min":88.180000000000007},"last":0.34100000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.5670000000000002,"15min":2.3170000000000002},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":8.4870000000000001,"5min":78.307000000000002,"15min":78.307000000000002},"last":1.478},{"interface":"front","average":{"1min":1.748,"5min":2.4369999999999998,"15min":2.645},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.548999999999999,"5min":78.216999999999999,"15min":88.120000000000005},"last":0.38900000000000001}]}]},{"osd":2,"up_from":94,"seq":403726926161,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1279744,"kb_used_data":458872,"kb_used_omap":81,"kb_used_meta":820718,"kb_avail":92465408,"statfs":{"total":95995035648,"available":94684577792,"internally_reserved":0,"allocated":469884928,"data_stored":467486312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":840415450},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":3.8530000000000002,"15min":3.5830000000000002},"min":{"1min":0.19900000000000001,"5min":0.18099999999999999,"15min":0.17699999999999999},"max":{"1min":12.644,"5min":132.87299999999999,"15min":132.87299999999999},"last":0.251},{"interface":"front","average":{"1min":2.819,"5min":4.2480000000000002,"15min":3.6619999999999999},"min":{"1min":0.23400000000000001,"5min":0.188,"15min":0.188},"max":{"1min":12.710000000000001,"5min":130.398,"15min":130.398},"last":0.23400000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":4.109,"5min":3.3090000000000002,"15min":4.7800000000000002},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.191},"max":{"1min":14.109999999999999,"5min":122.17100000000001,"15min":158.13399999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":4.2169999999999996,"5min":3.1680000000000001,"15min":4.8209999999999997},"min":{"1min":0.21299999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":17.061,"5min":121.91800000000001,"15min":161.12},"last":0.99099999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":3.9510000000000001,"5min":2.3490000000000002,"15min":2.5179999999999998},"min":{"1min":0.27300000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":43.707999999999998,"5min":45.023000000000003,"15min":58.997},"last":0.40500000000000003},{"interface":"front","average":{"1min":5.641,"5min":2.7200000000000002,"15min":2.8679999999999999},"min":{"1min":0.26600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":43.427999999999997,"5min":44.923999999999999,"15min":66.135000000000005},"last":1.1890000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":4.3099999999999996,"15min":2.9609999999999999},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.4149999999999991,"5min":196.57599999999999,"15min":196.57599999999999},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.956,"5min":3.1619999999999999,"15min":2.3450000000000002},"min":{"1min":0.312,"5min":0.27900000000000003,"15min":0.26800000000000002},"max":{"1min":9.0289999999999999,"5min":201.81100000000001,"15min":201.81100000000001},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":3.3999999999999999,"15min":3.0870000000000002},"min":{"1min":0.379,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":8.6699999999999999,"5min":190.06899999999999,"15min":190.06899999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.7370000000000001,"5min":4.4509999999999996,"15min":3.4119999999999999},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":9.8949999999999996,"5min":190.08500000000001,"15min":190.08500000000001},"last":1.0629999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.7989999999999999,"15min":1.853},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":9.0779999999999994,"5min":58.469999999999999,"15min":58.469999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.2469999999999999,"5min":2.4489999999999998,"15min":2.3159999999999998},"min":{"1min":0.40699999999999997,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":8.6989999999999998,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.43099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":3.2250000000000001,"15min":2.7589999999999999},"min":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.32800000000000001},"max":{"1min":9.1120000000000001,"5min":191.792,"15min":191.792},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.585,"5min":4.391,"15min":3.4529999999999998},"min":{"1min":0.42099999999999999,"5min":0.372,"15min":0.34399999999999997},"max":{"1min":8.843,"5min":191.828,"15min":191.828},"last":0.46899999999999997}]}]},{"osd":3,"up_from":99,"seq":425201762634,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804156,"kb_used_data":711092,"kb_used_omap":7,"kb_used_meta":1093048,"kb_avail":91940996,"statfs":{"total":95995035648,"available":94147579904,"internally_reserved":0,"allocated":728158208,"data_stored":725758908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1119281768},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":3.7530000000000001,"5min":5.319,"15min":4.0540000000000003},"min":{"1min":0.189,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":29.254999999999999,"5min":119.97799999999999,"15min":119.97799999999999},"last":9.9039999999999999},{"interface":"front","average":{"1min":7.0410000000000004,"5min":6.3849999999999998,"15min":4.7759999999999998},"min":{"1min":0.26600000000000001,"5min":0.223,"15min":0.185},"max":{"1min":29.506,"5min":120.307,"15min":120.307},"last":4.0279999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":4.2320000000000002,"5min":3.3580000000000001,"15min":4.4299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":16.427,"5min":121.842,"15min":170.196},"last":2.5680000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.6779999999999999,"15min":4.1299999999999999},"min":{"1min":0.22900000000000001,"5min":0.224,"15min":0.221},"max":{"1min":9.1519999999999992,"5min":119.673,"15min":166.953},"last":1.466}]},{"osd":2,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.103,"15min":2.2959999999999998},"min":{"1min":0.25800000000000001,"5min":0.217,"15min":0.187},"max":{"1min":4.1929999999999996,"5min":5.8300000000000001,"15min":127.599},"last":9.9510000000000005},{"interface":"front","average":{"1min":5.1680000000000001,"5min":2.2930000000000001,"15min":3.452},"min":{"1min":0.34399999999999997,"5min":0.23799999999999999,"15min":0.221},"max":{"1min":17.463000000000001,"5min":17.463000000000001,"15min":129.98599999999999},"last":3.9119999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":4.2830000000000004,"5min":3.8210000000000002,"15min":3.1800000000000002},"min":{"1min":0.38800000000000001,"5min":0.26900000000000002,"15min":0.25700000000000001},"max":{"1min":16.588999999999999,"5min":83.010000000000005,"15min":211.84700000000001},"last":4.0549999999999997},{"interface":"front","average":{"1min":3.7810000000000001,"5min":3.3290000000000002,"15min":3.1579999999999999},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":16.492000000000001,"5min":82.915000000000006,"15min":211.17500000000001},"last":2.6520000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":3.6000000000000001,"5min":2.802,"15min":2.4710000000000001},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":16.52,"5min":88.295000000000002,"15min":88.295000000000002},"last":2.625},{"interface":"front","average":{"1min":0.81000000000000005,"5min":2.222,"15min":1.825},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":4.1580000000000004,"5min":86.968999999999994,"15min":86.968999999999994},"last":1.579}]},{"osd":6,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.75,"15min":1.5860000000000001},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":4.1180000000000003,"5min":64.378,"15min":64.378},"last":1.5509999999999999},{"interface":"front","average":{"1min":4.0830000000000002,"5min":2.641,"15min":2.1600000000000001},"min":{"1min":0.45000000000000001,"5min":0.36599999999999999,"15min":0.29999999999999999},"max":{"1min":16.632000000000001,"5min":64.353999999999999,"15min":64.353999999999999},"last":4.101}]},{"osd":7,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":4.4089999999999998,"5min":4.2149999999999999,"15min":3.4620000000000002},"min":{"1min":0.442,"5min":0.38,"15min":0.34799999999999998},"max":{"1min":16.661000000000001,"5min":86.897000000000006,"15min":212.62700000000001},"last":4.1399999999999997},{"interface":"front","average":{"1min":4.0430000000000001,"5min":3.6419999999999999,"15min":3.488},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":16.553000000000001,"5min":87.105999999999995,"15min":212.04599999999999},"last":9.9939999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986650,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1260680,"kb_used_data":362944,"kb_used_omap":0,"kb_used_meta":897728,"kb_avail":92484472,"statfs":{"total":95995035648,"available":94704099328,"internally_reserved":0,"allocated":371654656,"data_stored":370109885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919273472},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:41 2023","interfaces":[{"interface":"back","average":{"1min":8.0099999999999998,"5min":6.5099999999999998,"15min":6.665},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20799999999999999},"max":{"1min":110.95,"5min":207.45699999999999,"15min":207.45699999999999},"last":3.0129999999999999},{"interface":"front","average":{"1min":7.2160000000000002,"5min":6.4530000000000003,"15min":6.7549999999999999},"min":{"1min":0.23200000000000001,"5min":0.223,"15min":0.223},"max":{"1min":111.057,"5min":207.114,"15min":207.114},"last":3.5470000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:30:28 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":1.8340000000000001,"15min":4.6180000000000003},"min":{"1min":0.35099999999999998,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":8.1839999999999993,"5min":33.433999999999997,"15min":135.25399999999999},"last":7.4029999999999996},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.661,"15min":4.4950000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.251},"max":{"1min":13.685,"5min":32.683,"15min":134.44300000000001},"last":3.3519999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:29:59 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":5.4969999999999999,"15min":4.0620000000000003},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":9.5809999999999995,"5min":203.61500000000001,"15min":203.61500000000001},"last":7.1020000000000003},{"interface":"front","average":{"1min":1.732,"5min":5.359,"15min":4.0149999999999997},"min":{"1min":0.316,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.2599999999999998,"5min":203.601,"15min":203.601},"last":7.1180000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:30:28 2023","interfaces":[{"interface":"back","average":{"1min":16.184000000000001,"5min":5.3200000000000003,"15min":3.5539999999999998},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":135.97800000000001,"5min":135.97800000000001,"15min":154.13200000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":15.048,"5min":5.218,"15min":3.5},"min":{"1min":0.309,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":135.899,"5min":135.899,"15min":154.077},"last":3.758}]},{"osd":5,"last update":"Mon Oct 9 04:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.2090000000000001,"15min":1.8220000000000001},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.7469999999999999,"5min":16.753,"15min":111.809},"last":3.794},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1120000000000001,"15min":1.9890000000000001},"min":{"1min":0.27600000000000002,"5min":0.253,"15min":0.252},"max":{"1min":6.6449999999999996,"5min":17.792000000000002,"15min":112.002},"last":1.222}]},{"osd":6,"last update":"Mon Oct 9 04:30:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.0720000000000001,"15min":1.7270000000000001},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.4120000000000008,"5min":10.039999999999999,"15min":76.307000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.296,"5min":1.0449999999999999,"15min":1.5700000000000001},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.1760000000000002,"5min":9.1760000000000002,"15min":78.421999999999997},"last":3.3140000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.3089999999999999,"15min":1.169},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":5.1550000000000002,"5min":12.587999999999999,"15min":12.587999999999999},"last":6.0869999999999997},{"interface":"front","average":{"1min":1.085,"5min":1.486,"15min":1.359},"min":{"1min":0.32300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.6710000000000003,"5min":12.863,"15min":12.863},"last":0.41599999999999998}]}]},{"osd":5,"up_from":0,"seq":167503725014,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":614268928,"data_stored":614268928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":712179712,"data_stored":712179712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":456003584,"data_stored":455999512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":720699392,"data_stored":720699392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":364449792,"data_stored":364445720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":455147520,"data_stored":455147520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":734793728,"data_stored":734789656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":543424512,"data_stored":543424512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:31:02.259 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:31:02.435 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:02.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:02 smithi163 ceph-mon[213841]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:31:02.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:02.444 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:02 smithi163 ceph-mon[221067]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:31:02.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:02.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:02 smithi181 ceph-mon[196979]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:31:02.545 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:02.580 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:02.752 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:02.796 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:02.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:02.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:02.940 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:02.984 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:02.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:03.088 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:03.113 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:03.167 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:03.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:03.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:03.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:03 smithi163 ceph-mon[221067]: pgmap v1479: 119 pgs: 119 active+clean; 1.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:03.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:03 smithi163 ceph-mon[213841]: pgmap v1479: 119 pgs: 119 active+clean; 1.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:03.335 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:03.414 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:03 smithi181 ceph-mon[196979]: pgmap v1479: 119 pgs: 119 active+clean; 1.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 766 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:03.549 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:03.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:03.658 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:03.712 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:03.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:03.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:03.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:04.001 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:04.180 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:04.180 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:04.257 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:31:04.266 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:04.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2783535290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:04.276 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:04 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:04.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:04.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2783535290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:04.278 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:04 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:04.378 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:04.409 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:04.472 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:04.474 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2783535290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:04.474 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:04 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:04.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:04.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:04.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:04.611 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:04.664 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:04.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:04.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:04.769 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:04.795 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:05.048 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:05.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:05.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:31:05.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:05.220 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:05.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:05.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:05.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: pgmap v1480: 119 pgs: 119 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: failed to encode map e561 with expected crc 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: failed to encode map e561 with expected crc 2023-10-09T04:31:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: pgmap v1480: 119 pgs: 119 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: failed to encode map e561 with expected crc 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: failed to encode map e561 with expected crc 2023-10-09T04:31:05.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:05.400 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:05.460 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:05.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: pgmap v1480: 119 pgs: 119 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: failed to encode map e561 with expected crc 2023-10-09T04:31:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T04:31:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: failed to encode map e561 with expected crc 2023-10-09T04:31:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:05.599 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:06.066 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:06.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:06.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:06.215 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:06.326 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:06.431 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:06.455 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:31:06.457 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:06.474 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:31:06.492 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:31:06.498 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:06.500 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:06 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:06.512 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:31:06.528 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:06 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:06.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:06 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:06.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:06.642 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:06.651 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:31:06.692 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:31:06.707 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:06.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:06.738 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:31:06.786 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.810 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.811 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:31:06.813 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:06.913 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.088 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:07.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:07.156 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:31:07.184 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:31:07.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:07.234 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.278 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:07.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:07.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:07.424 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.501 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:07.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:07.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:07 smithi181 ceph-mon[196979]: pgmap v1482: 119 pgs: 119 active+clean; 957 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 789 op/s 2023-10-09T04:31:07.510 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2968905932' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:07.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:07 smithi163 ceph-mon[213841]: pgmap v1482: 119 pgs: 119 active+clean; 957 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 789 op/s 2023-10-09T04:31:07.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2968905932' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:07.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:07 smithi163 ceph-mon[221067]: pgmap v1482: 119 pgs: 119 active+clean; 957 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 789 op/s 2023-10-09T04:31:07.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2968905932' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:07.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:07.659 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:31:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [], 2023-10-09T04:31:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "complaint_time": 30, 2023-10-09T04:31:07.659 INFO:teuthology.orchestra.run.smithi181.stdout: "num_blocked_ops": 0 2023-10-09T04:31:07.659 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:31:07.680 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.938 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:31:08.392 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:08.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:08.423 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1482,"stamp":"2023-10-09T04:31:06.111870+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1003257995,"num_objects":15325,"num_object_clones":0,"num_object_copies":45975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15325,"num_whiteouts":0,"num_read":24,"num_read_kb":43,"num_write":194326,"num_write_kb":6705804,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12031,"num_bytes_recovered":783313479,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98008,"ondisk_log_size":98008,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9412484,"kb_used_data":3038732,"kb_used_omap":126,"kb_used_meta":6373441,"kb_avail":646803580,"statfs":{"total":671965249536,"available":662326865920,"internally_reserved":0,"allocated":3111661568,"data_stored":3097278382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":6526404219},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-511180800,"num_objects":-7800,"num_object_clones":0,"num_object_copies":-23400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7800,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.883411"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":738,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234343+0000","last_change":"2023-10-09T04:30:45.821591+0000","last_active":"2023-10-09T04:30:56.234343+0000","last_peered":"2023-10-09T04:30:56.234343+0000","last_clean":"2023-10-09T04:30:56.234343+0000","last_became_active":"2023-10-09T04:29:47.255157+0000","last_became_peered":"2023-10-09T04:29:47.255157+0000","last_unstale":"2023-10-09T04:30:56.234343+0000","last_undegraded":"2023-10-09T04:30:56.234343+0000","last_fullsized":"2023-10-09T04:30:56.234343+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:30:45.821476+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:30:45.821476+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:07.307678+0000","scrub_duration":0.010879602,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":146,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242616+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:30:56.242616+0000","last_peered":"2023-10-09T04:30:56.242616+0000","last_clean":"2023-10-09T04:30:56.242616+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:30:56.242616+0000","last_undegraded":"2023-10-09T04:30:56.242616+0000","last_fullsized":"2023-10-09T04:30:56.242616+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":123,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.244008+0000","last_change":"2023-10-09T04:24:56.167559+0000","last_active":"2023-10-09T04:30:56.244008+0000","last_peered":"2023-10-09T04:30:56.244008+0000","last_clean":"2023-10-09T04:30:56.244008+0000","last_became_active":"2023-10-09T04:24:56.164533+0000","last_became_peered":"2023-10-09T04:24:56.164533+0000","last_unstale":"2023-10-09T04:30:56.244008+0000","last_undegraded":"2023-10-09T04:30:56.244008+0000","last_fullsized":"2023-10-09T04:30:56.244008+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:53.593094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":675,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242705+0000","last_change":"2023-10-09T04:29:47.243471+0000","last_active":"2023-10-09T04:30:56.242705+0000","last_peered":"2023-10-09T04:30:56.242705+0000","last_clean":"2023-10-09T04:30:56.242705+0000","last_became_active":"2023-10-09T04:29:47.243158+0000","last_became_peered":"2023-10-09T04:29:47.243158+0000","last_unstale":"2023-10-09T04:30:56.242705+0000","last_undegraded":"2023-10-09T04:30:56.242705+0000","last_fullsized":"2023-10-09T04:30:56.242705+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:45.996992+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":579,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.247313+0000","last_change":"2023-10-09T04:30:51.983079+0000","last_active":"2023-10-09T04:30:56.247313+0000","last_peered":"2023-10-09T04:30:56.247313+0000","last_clean":"2023-10-09T04:30:56.247313+0000","last_became_active":"2023-10-09T04:30:51.982789+0000","last_became_peered":"2023-10-09T04:30:51.982789+0000","last_unstale":"2023-10-09T04:30:56.247313+0000","last_undegraded":"2023-10-09T04:30:56.247313+0000","last_fullsized":"2023-10-09T04:30:56.247313+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:47.753426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828380+0000","last_change":"2023-10-09T04:23:06.350437+0000","last_active":"2023-10-09T04:30:53.828380+0000","last_peered":"2023-10-09T04:30:53.828380+0000","last_clean":"2023-10-09T04:30:53.828380+0000","last_became_active":"2023-10-09T04:23:06.349756+0000","last_became_peered":"2023-10-09T04:23:06.349756+0000","last_unstale":"2023-10-09T04:30:53.828380+0000","last_undegraded":"2023-10-09T04:30:53.828380+0000","last_fullsized":"2023-10-09T04:30:53.828380+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:14.219839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":553,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242536+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:30:56.242536+0000","last_peered":"2023-10-09T04:30:56.242536+0000","last_clean":"2023-10-09T04:30:56.242536+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:30:56.242536+0000","last_undegraded":"2023-10-09T04:30:56.242536+0000","last_fullsized":"2023-10-09T04:30:56.242536+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827775+0000","last_change":"2023-10-09T04:23:06.349423+0000","last_active":"2023-10-09T04:30:53.827775+0000","last_peered":"2023-10-09T04:30:53.827775+0000","last_clean":"2023-10-09T04:30:53.827775+0000","last_became_active":"2023-10-09T04:23:06.349038+0000","last_became_peered":"2023-10-09T04:23:06.349038+0000","last_unstale":"2023-10-09T04:30:53.827775+0000","last_undegraded":"2023-10-09T04:30:53.827775+0000","last_fullsized":"2023-10-09T04:30:53.827775+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:28.805841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828418+0000","last_change":"2023-10-09T04:30:51.977588+0000","last_active":"2023-10-09T04:30:53.828418+0000","last_peered":"2023-10-09T04:30:53.828418+0000","last_clean":"2023-10-09T04:30:53.828418+0000","last_became_active":"2023-10-09T04:30:51.977346+0000","last_became_peered":"2023-10-09T04:30:51.977346+0000","last_unstale":"2023-10-09T04:30:53.828418+0000","last_undegraded":"2023-10-09T04:30:53.828418+0000","last_fullsized":"2023-10-09T04:30:53.828418+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:18.956710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":980,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238358+0000","last_change":"2023-10-09T04:29:47.255527+0000","last_active":"2023-10-09T04:30:56.238358+0000","last_peered":"2023-10-09T04:30:56.238358+0000","last_clean":"2023-10-09T04:30:56.238358+0000","last_became_active":"2023-10-09T04:29:47.255304+0000","last_became_peered":"2023-10-09T04:29:47.255304+0000","last_unstale":"2023-10-09T04:30:56.238358+0000","last_undegraded":"2023-10-09T04:30:56.238358+0000","last_fullsized":"2023-10-09T04:30:56.238358+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:12.642548+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828869+0000","last_change":"2023-10-09T04:29:44.187371+0000","last_active":"2023-10-09T04:30:53.828869+0000","last_peered":"2023-10-09T04:30:53.828869+0000","last_clean":"2023-10-09T04:30:53.828869+0000","last_became_active":"2023-10-09T04:29:44.186627+0000","last_became_peered":"2023-10-09T04:29:44.186627+0000","last_unstale":"2023-10-09T04:30:53.828869+0000","last_undegraded":"2023-10-09T04:30:53.828869+0000","last_fullsized":"2023-10-09T04:30:53.828869+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:29.638655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":262,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.244276+0000","last_change":"2023-10-09T04:30:51.977627+0000","last_active":"2023-10-09T04:30:56.244276+0000","last_peered":"2023-10-09T04:30:56.244276+0000","last_clean":"2023-10-09T04:30:56.244276+0000","last_became_active":"2023-10-09T04:30:51.977309+0000","last_became_peered":"2023-10-09T04:30:51.977309+0000","last_unstale":"2023-10-09T04:30:56.244276+0000","last_undegraded":"2023-10-09T04:30:56.244276+0000","last_fullsized":"2023-10-09T04:30:56.244276+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:00:30.619870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":867,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.235018+0000","last_change":"2023-10-09T04:27:45.967966+0000","last_active":"2023-10-09T04:30:56.235018+0000","last_peered":"2023-10-09T04:30:56.235018+0000","last_clean":"2023-10-09T04:30:56.235018+0000","last_became_active":"2023-10-09T04:26:23.674512+0000","last_became_peered":"2023-10-09T04:26:23.674512+0000","last_unstale":"2023-10-09T04:30:56.235018+0000","last_undegraded":"2023-10-09T04:30:56.235018+0000","last_fullsized":"2023-10-09T04:30:56.235018+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:45.967906+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:45.967906+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:06.528314+0000","scrub_duration":0.0012842089999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":80,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238634+0000","last_change":"2023-10-09T04:29:44.191334+0000","last_active":"2023-10-09T04:30:56.238634+0000","last_peered":"2023-10-09T04:30:56.238634+0000","last_clean":"2023-10-09T04:30:56.238634+0000","last_became_active":"2023-10-09T04:29:44.191013+0000","last_became_peered":"2023-10-09T04:29:44.191013+0000","last_unstale":"2023-10-09T04:30:56.238634+0000","last_undegraded":"2023-10-09T04:30:56.238634+0000","last_fullsized":"2023-10-09T04:30:56.238634+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:12.273980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":205,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.247389+0000","last_change":"2023-10-09T04:22:13.627987+0000","last_active":"2023-10-09T04:30:56.247389+0000","last_peered":"2023-10-09T04:30:56.247389+0000","last_clean":"2023-10-09T04:30:56.247389+0000","last_became_active":"2023-10-09T04:22:13.627617+0000","last_became_peered":"2023-10-09T04:22:13.627617+0000","last_unstale":"2023-10-09T04:30:56.247389+0000","last_undegraded":"2023-10-09T04:30:56.247389+0000","last_fullsized":"2023-10-09T04:30:56.247389+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:42.464486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":641,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828514+0000","last_change":"2023-10-09T04:29:47.253301+0000","last_active":"2023-10-09T04:30:53.828514+0000","last_peered":"2023-10-09T04:30:53.828514+0000","last_clean":"2023-10-09T04:30:53.828514+0000","last_became_active":"2023-10-09T04:29:47.252838+0000","last_became_peered":"2023-10-09T04:29:47.252838+0000","last_unstale":"2023-10-09T04:30:53.828514+0000","last_undegraded":"2023-10-09T04:30:53.828514+0000","last_fullsized":"2023-10-09T04:30:53.828514+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:52:45.161327+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":635,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238586+0000","last_change":"2023-10-09T04:24:56.216021+0000","last_active":"2023-10-09T04:30:56.238586+0000","last_peered":"2023-10-09T04:30:56.238586+0000","last_clean":"2023-10-09T04:30:56.238586+0000","last_became_active":"2023-10-09T04:24:56.184515+0000","last_became_peered":"2023-10-09T04:24:56.184515+0000","last_unstale":"2023-10-09T04:30:56.238586+0000","last_undegraded":"2023-10-09T04:30:56.238586+0000","last_fullsized":"2023-10-09T04:30:56.238586+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:11.130212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":197,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.245836+0000","last_change":"2023-10-09T04:29:44.190663+0000","last_active":"2023-10-09T04:30:56.245836+0000","last_peered":"2023-10-09T04:30:56.245836+0000","last_clean":"2023-10-09T04:30:56.245836+0000","last_became_active":"2023-10-09T04:29:44.190245+0000","last_became_peered":"2023-10-09T04:29:44.190245+0000","last_unstale":"2023-10-09T04:30:56.245836+0000","last_undegraded":"2023-10-09T04:30:56.245836+0000","last_fullsized":"2023-10-09T04:30:56.245836+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:00:16.922638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":741,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.249182+0000","last_change":"2023-10-09T04:23:08.625236+0000","last_active":"2023-10-09T04:30:56.249182+0000","last_peered":"2023-10-09T04:30:56.249182+0000","last_clean":"2023-10-09T04:30:56.249182+0000","last_became_active":"2023-10-09T04:23:08.624916+0000","last_became_peered":"2023-10-09T04:23:08.624916+0000","last_unstale":"2023-10-09T04:30:56.249182+0000","last_undegraded":"2023-10-09T04:30:56.249182+0000","last_fullsized":"2023-10-09T04:30:56.249182+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:47:24.532473+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":136,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.244740+0000","last_change":"2023-10-09T04:24:56.201189+0000","last_active":"2023-10-09T04:30:56.244740+0000","last_peered":"2023-10-09T04:30:56.244740+0000","last_clean":"2023-10-09T04:30:56.244740+0000","last_became_active":"2023-10-09T04:24:56.175176+0000","last_became_peered":"2023-10-09T04:24:56.175176+0000","last_unstale":"2023-10-09T04:30:56.244740+0000","last_undegraded":"2023-10-09T04:30:56.244740+0000","last_fullsized":"2023-10-09T04:30:56.244740+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:51.573341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242825+0000","last_change":"2023-10-09T04:24:56.169656+0000","last_active":"2023-10-09T04:30:56.242825+0000","last_peered":"2023-10-09T04:30:56.242825+0000","last_clean":"2023-10-09T04:30:56.242825+0000","last_became_active":"2023-10-09T04:24:56.168905+0000","last_became_peered":"2023-10-09T04:24:56.168905+0000","last_unstale":"2023-10-09T04:30:56.242825+0000","last_undegraded":"2023-10-09T04:30:56.242825+0000","last_fullsized":"2023-10-09T04:30:56.242825+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:20.583237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":610,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832835+0000","last_change":"2023-10-09T04:29:47.263524+0000","last_active":"2023-10-09T04:30:53.832835+0000","last_peered":"2023-10-09T04:30:53.832835+0000","last_clean":"2023-10-09T04:30:53.832835+0000","last_became_active":"2023-10-09T04:29:47.263039+0000","last_became_peered":"2023-10-09T04:29:47.263039+0000","last_unstale":"2023-10-09T04:30:53.832835+0000","last_undegraded":"2023-10-09T04:30:53.832835+0000","last_fullsized":"2023-10-09T04:30:53.832835+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:02.486982+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":604,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.235049+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:30:56.235049+0000","last_peered":"2023-10-09T04:30:56.235049+0000","last_clean":"2023-10-09T04:30:56.235049+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:30:56.235049+0000","last_undegraded":"2023-10-09T04:30:56.235049+0000","last_fullsized":"2023-10-09T04:30:56.235049+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":336,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234831+0000","last_change":"2023-10-09T04:22:02.699452+0000","last_active":"2023-10-09T04:30:56.234831+0000","last_peered":"2023-10-09T04:30:56.234831+0000","last_clean":"2023-10-09T04:30:56.234831+0000","last_became_active":"2023-10-09T04:22:02.699116+0000","last_became_peered":"2023-10-09T04:22:02.699116+0000","last_unstale":"2023-10-09T04:30:56.234831+0000","last_undegraded":"2023-10-09T04:30:56.234831+0000","last_fullsized":"2023-10-09T04:30:56.234831+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:43.682723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":724,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234457+0000","last_change":"2023-10-09T04:30:46.778471+0000","last_active":"2023-10-09T04:30:56.234457+0000","last_peered":"2023-10-09T04:30:56.234457+0000","last_clean":"2023-10-09T04:30:56.234457+0000","last_became_active":"2023-10-09T04:29:47.194746+0000","last_became_peered":"2023-10-09T04:29:47.194746+0000","last_unstale":"2023-10-09T04:30:56.234457+0000","last_undegraded":"2023-10-09T04:30:56.234457+0000","last_fullsized":"2023-10-09T04:30:56.234457+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:30:46.778406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:30:46.778406+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:51.879595+0000","scrub_duration":0.001341243,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828743+0000","last_change":"2023-10-09T04:23:06.350578+0000","last_active":"2023-10-09T04:30:53.828743+0000","last_peered":"2023-10-09T04:30:53.828743+0000","last_clean":"2023-10-09T04:30:53.828743+0000","last_became_active":"2023-10-09T04:23:06.350299+0000","last_became_peered":"2023-10-09T04:23:06.350299+0000","last_unstale":"2023-10-09T04:30:53.828743+0000","last_undegraded":"2023-10-09T04:30:53.828743+0000","last_fullsized":"2023-10-09T04:30:53.828743+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:51.383259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":319,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238669+0000","last_change":"2023-10-09T04:29:44.196614+0000","last_active":"2023-10-09T04:30:56.238669+0000","last_peered":"2023-10-09T04:30:56.238669+0000","last_clean":"2023-10-09T04:30:56.238669+0000","last_became_active":"2023-10-09T04:29:44.196013+0000","last_became_peered":"2023-10-09T04:29:44.196013+0000","last_unstale":"2023-10-09T04:30:56.238669+0000","last_undegraded":"2023-10-09T04:30:56.238669+0000","last_fullsized":"2023-10-09T04:30:56.238669+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:38.381032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":707,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.248415+0000","last_change":"2023-10-09T04:24:58.646677+0000","last_active":"2023-10-09T04:30:56.248415+0000","last_peered":"2023-10-09T04:30:56.248415+0000","last_clean":"2023-10-09T04:30:56.248415+0000","last_became_active":"2023-10-09T04:24:58.646341+0000","last_became_peered":"2023-10-09T04:24:58.646341+0000","last_unstale":"2023-10-09T04:30:56.248415+0000","last_undegraded":"2023-10-09T04:30:56.248415+0000","last_fullsized":"2023-10-09T04:30:56.248415+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:51.993485+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243343+0000","last_change":"2023-10-09T04:24:56.179989+0000","last_active":"2023-10-09T04:30:56.243343+0000","last_peered":"2023-10-09T04:30:56.243343+0000","last_clean":"2023-10-09T04:30:56.243343+0000","last_became_active":"2023-10-09T04:24:56.170855+0000","last_became_peered":"2023-10-09T04:24:56.170855+0000","last_unstale":"2023-10-09T04:30:56.243343+0000","last_undegraded":"2023-10-09T04:30:56.243343+0000","last_fullsized":"2023-10-09T04:30:56.243343+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:12:47.991338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828371+0000","last_change":"2023-10-09T04:23:06.350602+0000","last_active":"2023-10-09T04:30:53.828371+0000","last_peered":"2023-10-09T04:30:53.828371+0000","last_clean":"2023-10-09T04:30:53.828371+0000","last_became_active":"2023-10-09T04:23:06.350167+0000","last_became_peered":"2023-10-09T04:23:06.350167+0000","last_unstale":"2023-10-09T04:30:53.828371+0000","last_undegraded":"2023-10-09T04:30:53.828371+0000","last_fullsized":"2023-10-09T04:30:53.828371+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:45.644274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":448,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234502+0000","last_change":"2023-10-09T04:30:40.819676+0000","last_active":"2023-10-09T04:30:56.234502+0000","last_peered":"2023-10-09T04:30:56.234502+0000","last_clean":"2023-10-09T04:30:56.234502+0000","last_became_active":"2023-10-09T04:29:47.249760+0000","last_became_peered":"2023-10-09T04:29:47.249760+0000","last_unstale":"2023-10-09T04:30:56.234502+0000","last_undegraded":"2023-10-09T04:30:56.234502+0000","last_fullsized":"2023-10-09T04:30:56.234502+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:19.353766+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:19.353766+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":195,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828115+0000","last_change":"2023-10-09T04:29:44.182868+0000","last_active":"2023-10-09T04:30:53.828115+0000","last_peered":"2023-10-09T04:30:53.828115+0000","last_clean":"2023-10-09T04:30:53.828115+0000","last_became_active":"2023-10-09T04:29:44.177350+0000","last_became_peered":"2023-10-09T04:29:44.177350+0000","last_unstale":"2023-10-09T04:30:53.828115+0000","last_undegraded":"2023-10-09T04:30:53.828115+0000","last_fullsized":"2023-10-09T04:30:53.828115+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:52.638746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":227,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.247998+0000","last_change":"2023-10-09T04:20:02.106203+0000","last_active":"2023-10-09T04:30:56.247998+0000","last_peered":"2023-10-09T04:30:56.247998+0000","last_clean":"2023-10-09T04:30:56.247998+0000","last_became_active":"2023-10-09T04:20:02.105886+0000","last_became_peered":"2023-10-09T04:20:02.105886+0000","last_unstale":"2023-10-09T04:30:56.247998+0000","last_undegraded":"2023-10-09T04:30:56.247998+0000","last_fullsized":"2023-10-09T04:30:56.247998+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:18.972464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"561'12141","reported_seq":17085,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.199936+0000","last_change":"2023-10-09T04:30:42.940632+0000","last_active":"2023-10-09T04:31:05.199936+0000","last_peered":"2023-10-09T04:31:05.199936+0000","last_clean":"2023-10-09T04:31:05.199936+0000","last_became_active":"2023-10-09T04:30:42.903426+0000","last_became_peered":"2023-10-09T04:30:42.903426+0000","last_unstale":"2023-10-09T04:31:05.199936+0000","last_undegraded":"2023-10-09T04:31:05.199936+0000","last_fullsized":"2023-10-09T04:31:05.199936+0000","mapping_epoch":552,"log_start":"524'6057","ondisk_log_start":"524'6057","created":507,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6084,"log_dups_size":0,"ondisk_log_size":6084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:04.590139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62324736,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12141,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2429,"num_bytes_recovered":159186944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":1003,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238366+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:30:56.238366+0000","last_peered":"2023-10-09T04:30:56.238366+0000","last_clean":"2023-10-09T04:30:56.238366+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:30:56.238366+0000","last_undegraded":"2023-10-09T04:30:56.238366+0000","last_fullsized":"2023-10-09T04:30:56.238366+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":605,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238689+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:30:56.238689+0000","last_peered":"2023-10-09T04:30:56.238689+0000","last_clean":"2023-10-09T04:30:56.238689+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:30:56.238689+0000","last_undegraded":"2023-10-09T04:30:56.238689+0000","last_fullsized":"2023-10-09T04:30:56.238689+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":375,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234701+0000","last_change":"2023-10-09T04:29:44.198437+0000","last_active":"2023-10-09T04:30:56.234701+0000","last_peered":"2023-10-09T04:30:56.234701+0000","last_clean":"2023-10-09T04:30:56.234701+0000","last_became_active":"2023-10-09T04:29:44.198097+0000","last_became_peered":"2023-10-09T04:29:44.198097+0000","last_unstale":"2023-10-09T04:30:56.234701+0000","last_undegraded":"2023-10-09T04:30:56.234701+0000","last_fullsized":"2023-10-09T04:30:56.234701+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:36.976087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"560'12112","reported_seq":12175,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:31:02.155409+0000","last_change":"2023-10-09T04:27:20.775052+0000","last_active":"2023-10-09T04:31:02.155409+0000","last_peered":"2023-10-09T04:31:02.155409+0000","last_clean":"2023-10-09T04:31:02.155409+0000","last_became_active":"2023-10-09T04:27:20.774707+0000","last_became_peered":"2023-10-09T04:27:20.774707+0000","last_unstale":"2023-10-09T04:31:02.155409+0000","last_undegraded":"2023-10-09T04:31:02.155409+0000","last_fullsized":"2023-10-09T04:31:02.155409+0000","mapping_epoch":507,"log_start":"525'6223","ondisk_log_start":"525'6223","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5889,"log_dups_size":0,"ondisk_log_size":5889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:05.393595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73924608,"num_objects":1128,"num_object_clones":0,"num_object_copies":3384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12112,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":795,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.247557+0000","last_change":"2023-10-09T04:23:28.773885+0000","last_active":"2023-10-09T04:30:56.247557+0000","last_peered":"2023-10-09T04:30:56.247557+0000","last_clean":"2023-10-09T04:30:56.247557+0000","last_became_active":"2023-10-09T04:23:28.773412+0000","last_became_peered":"2023-10-09T04:23:28.773412+0000","last_unstale":"2023-10-09T04:30:56.247557+0000","last_undegraded":"2023-10-09T04:30:56.247557+0000","last_fullsized":"2023-10-09T04:30:56.247557+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:56.278803+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":595,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.239567+0000","last_change":"2023-10-09T04:30:52.027413+0000","last_active":"2023-10-09T04:30:56.239567+0000","last_peered":"2023-10-09T04:30:56.239567+0000","last_clean":"2023-10-09T04:30:56.239567+0000","last_became_active":"2023-10-09T04:30:51.982673+0000","last_became_peered":"2023-10-09T04:30:51.982673+0000","last_unstale":"2023-10-09T04:30:56.239567+0000","last_undegraded":"2023-10-09T04:30:56.239567+0000","last_fullsized":"2023-10-09T04:30:56.239567+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:22.580030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243497+0000","last_change":"2023-10-09T04:24:56.167836+0000","last_active":"2023-10-09T04:30:56.243497+0000","last_peered":"2023-10-09T04:30:56.243497+0000","last_clean":"2023-10-09T04:30:56.243497+0000","last_became_active":"2023-10-09T04:24:56.165965+0000","last_became_peered":"2023-10-09T04:24:56.165965+0000","last_unstale":"2023-10-09T04:30:56.243497+0000","last_undegraded":"2023-10-09T04:30:56.243497+0000","last_fullsized":"2023-10-09T04:30:56.243497+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:53.755761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"561'12193","reported_seq":19922,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.706890+0000","last_change":"2023-10-09T04:30:17.183742+0000","last_active":"2023-10-09T04:31:05.706890+0000","last_peered":"2023-10-09T04:31:05.706890+0000","last_clean":"2023-10-09T04:31:05.706890+0000","last_became_active":"2023-10-09T04:30:17.165879+0000","last_became_peered":"2023-10-09T04:30:17.165879+0000","last_unstale":"2023-10-09T04:31:05.706890+0000","last_undegraded":"2023-10-09T04:31:05.706890+0000","last_fullsized":"2023-10-09T04:31:05.706890+0000","mapping_epoch":544,"log_start":"526'6448","ondisk_log_start":"526'6448","created":507,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:46.438208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59310080,"num_objects":905,"num_object_clones":0,"num_object_copies":2715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12193,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3821,"num_bytes_recovered":250413056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":682,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.246296+0000","last_change":"2023-10-09T04:29:47.252712+0000","last_active":"2023-10-09T04:30:56.246296+0000","last_peered":"2023-10-09T04:30:56.246296+0000","last_clean":"2023-10-09T04:30:56.246296+0000","last_became_active":"2023-10-09T04:29:47.252283+0000","last_became_peered":"2023-10-09T04:29:47.252283+0000","last_unstale":"2023-10-09T04:30:56.246296+0000","last_undegraded":"2023-10-09T04:30:56.246296+0000","last_fullsized":"2023-10-09T04:30:56.246296+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:09.123986+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":546,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238498+0000","last_change":"2023-10-09T04:29:44.201736+0000","last_active":"2023-10-09T04:30:56.238498+0000","last_peered":"2023-10-09T04:30:56.238498+0000","last_clean":"2023-10-09T04:30:56.238498+0000","last_became_active":"2023-10-09T04:29:44.196504+0000","last_became_peered":"2023-10-09T04:29:44.196504+0000","last_unstale":"2023-10-09T04:30:56.238498+0000","last_undegraded":"2023-10-09T04:30:56.238498+0000","last_fullsized":"2023-10-09T04:30:56.238498+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:53.990636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.245709+0000","last_change":"2023-10-09T04:22:13.629906+0000","last_active":"2023-10-09T04:30:56.245709+0000","last_peered":"2023-10-09T04:30:56.245709+0000","last_clean":"2023-10-09T04:30:56.245709+0000","last_became_active":"2023-10-09T04:22:13.629584+0000","last_became_peered":"2023-10-09T04:22:13.629584+0000","last_unstale":"2023-10-09T04:30:56.245709+0000","last_undegraded":"2023-10-09T04:30:56.245709+0000","last_fullsized":"2023-10-09T04:30:56.245709+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:39.430430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"561'12016","reported_seq":12078,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.683131+0000","last_change":"2023-10-09T04:27:20.775264+0000","last_active":"2023-10-09T04:31:05.683131+0000","last_peered":"2023-10-09T04:31:05.683131+0000","last_clean":"2023-10-09T04:31:05.683131+0000","last_became_active":"2023-10-09T04:27:20.774957+0000","last_became_peered":"2023-10-09T04:27:20.774957+0000","last_unstale":"2023-10-09T04:31:05.683131+0000","last_undegraded":"2023-10-09T04:31:05.683131+0000","last_fullsized":"2023-10-09T04:31:05.683131+0000","mapping_epoch":507,"log_start":"526'6299","ondisk_log_start":"526'6299","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.572617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12016,"num_write_kb":411264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":817,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.246038+0000","last_change":"2023-10-09T04:27:04.271824+0000","last_active":"2023-10-09T04:30:56.246038+0000","last_peered":"2023-10-09T04:30:56.246038+0000","last_clean":"2023-10-09T04:30:56.246038+0000","last_became_active":"2023-10-09T04:24:58.648904+0000","last_became_peered":"2023-10-09T04:24:58.648904+0000","last_unstale":"2023-10-09T04:30:56.246038+0000","last_undegraded":"2023-10-09T04:30:56.246038+0000","last_fullsized":"2023-10-09T04:30:56.246038+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:04.848327+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":412,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.246825+0000","last_change":"2023-10-09T04:29:44.187314+0000","last_active":"2023-10-09T04:30:56.246825+0000","last_peered":"2023-10-09T04:30:56.246825+0000","last_clean":"2023-10-09T04:30:56.246825+0000","last_became_active":"2023-10-09T04:29:44.186972+0000","last_became_peered":"2023-10-09T04:29:44.186972+0000","last_unstale":"2023-10-09T04:30:56.246825+0000","last_undegraded":"2023-10-09T04:30:56.246825+0000","last_fullsized":"2023-10-09T04:30:56.246825+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:36.570707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":184,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827760+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:30:53.827760+0000","last_peered":"2023-10-09T04:30:53.827760+0000","last_clean":"2023-10-09T04:30:53.827760+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:30:53.827760+0000","last_undegraded":"2023-10-09T04:30:53.827760+0000","last_fullsized":"2023-10-09T04:30:53.827760+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"561'12337","reported_seq":12399,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.719446+0000","last_change":"2023-10-09T04:27:20.718937+0000","last_active":"2023-10-09T04:31:05.719446+0000","last_peered":"2023-10-09T04:31:05.719446+0000","last_clean":"2023-10-09T04:31:05.719446+0000","last_became_active":"2023-10-09T04:27:20.718721+0000","last_became_peered":"2023-10-09T04:27:20.718721+0000","last_unstale":"2023-10-09T04:31:05.719446+0000","last_undegraded":"2023-10-09T04:31:05.719446+0000","last_fullsized":"2023-10-09T04:31:05.719446+0000","mapping_epoch":507,"log_start":"527'6607","ondisk_log_start":"527'6607","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:47.168496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58392576,"num_objects":891,"num_object_clones":0,"num_object_copies":2673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12337,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1024,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234412+0000","last_change":"2023-10-09T04:26:23.674408+0000","last_active":"2023-10-09T04:30:56.234412+0000","last_peered":"2023-10-09T04:30:56.234412+0000","last_clean":"2023-10-09T04:30:56.234412+0000","last_became_active":"2023-10-09T04:26:23.674069+0000","last_became_peered":"2023-10-09T04:26:23.674069+0000","last_unstale":"2023-10-09T04:30:56.234412+0000","last_undegraded":"2023-10-09T04:30:56.234412+0000","last_fullsized":"2023-10-09T04:30:56.234412+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:25.460400+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":196,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827719+0000","last_change":"2023-10-09T04:24:56.192133+0000","last_active":"2023-10-09T04:30:53.827719+0000","last_peered":"2023-10-09T04:30:53.827719+0000","last_clean":"2023-10-09T04:30:53.827719+0000","last_became_active":"2023-10-09T04:24:56.169068+0000","last_became_peered":"2023-10-09T04:24:56.169068+0000","last_unstale":"2023-10-09T04:30:53.827719+0000","last_undegraded":"2023-10-09T04:30:53.827719+0000","last_fullsized":"2023-10-09T04:30:53.827719+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:57.548142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.244657+0000","last_change":"2023-10-09T04:24:56.179933+0000","last_active":"2023-10-09T04:30:56.244657+0000","last_peered":"2023-10-09T04:30:56.244657+0000","last_clean":"2023-10-09T04:30:56.244657+0000","last_became_active":"2023-10-09T04:24:56.178701+0000","last_became_peered":"2023-10-09T04:24:56.178701+0000","last_unstale":"2023-10-09T04:30:56.244657+0000","last_undegraded":"2023-10-09T04:30:56.244657+0000","last_fullsized":"2023-10-09T04:30:56.244657+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:54.382542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"560'12018","reported_seq":12081,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:31:01.922103+0000","last_change":"2023-10-09T04:27:20.775434+0000","last_active":"2023-10-09T04:31:01.922103+0000","last_peered":"2023-10-09T04:31:01.922103+0000","last_clean":"2023-10-09T04:31:01.922103+0000","last_became_active":"2023-10-09T04:27:20.774886+0000","last_became_peered":"2023-10-09T04:27:20.774886+0000","last_unstale":"2023-10-09T04:31:01.922103+0000","last_undegraded":"2023-10-09T04:31:01.922103+0000","last_fullsized":"2023-10-09T04:31:01.922103+0000","mapping_epoch":507,"log_start":"524'6066","ondisk_log_start":"524'6066","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:36.101412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69074944,"num_objects":1054,"num_object_clones":0,"num_object_copies":3162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12018,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":711,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242686+0000","last_change":"2023-10-09T04:29:47.267181+0000","last_active":"2023-10-09T04:30:56.242686+0000","last_peered":"2023-10-09T04:30:56.242686+0000","last_clean":"2023-10-09T04:30:56.242686+0000","last_became_active":"2023-10-09T04:29:47.266850+0000","last_became_peered":"2023-10-09T04:29:47.266850+0000","last_unstale":"2023-10-09T04:30:56.242686+0000","last_undegraded":"2023-10-09T04:30:56.242686+0000","last_fullsized":"2023-10-09T04:30:56.242686+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:20.803798+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242583+0000","last_change":"2023-10-09T04:24:56.169050+0000","last_active":"2023-10-09T04:30:56.242583+0000","last_peered":"2023-10-09T04:30:56.242583+0000","last_clean":"2023-10-09T04:30:56.242583+0000","last_became_active":"2023-10-09T04:24:56.167337+0000","last_became_peered":"2023-10-09T04:24:56.167337+0000","last_unstale":"2023-10-09T04:30:56.242583+0000","last_undegraded":"2023-10-09T04:30:56.242583+0000","last_fullsized":"2023-10-09T04:30:56.242583+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:24.306413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":143,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243936+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:30:56.243936+0000","last_peered":"2023-10-09T04:30:56.243936+0000","last_clean":"2023-10-09T04:30:56.243936+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:30:56.243936+0000","last_undegraded":"2023-10-09T04:30:56.243936+0000","last_fullsized":"2023-10-09T04:30:56.243936+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"561'12271","reported_seq":12335,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:06.023359+0000","last_change":"2023-10-09T04:27:20.714869+0000","last_active":"2023-10-09T04:31:06.023359+0000","last_peered":"2023-10-09T04:31:06.023359+0000","last_clean":"2023-10-09T04:31:06.023359+0000","last_became_active":"2023-10-09T04:27:20.714599+0000","last_became_peered":"2023-10-09T04:27:20.714599+0000","last_unstale":"2023-10-09T04:31:06.023359+0000","last_undegraded":"2023-10-09T04:31:06.023359+0000","last_fullsized":"2023-10-09T04:31:06.023359+0000","mapping_epoch":507,"log_start":"528'6991","ondisk_log_start":"528'6991","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5280,"log_dups_size":0,"ondisk_log_size":5280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:43.977629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54722560,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12271,"num_write_kb":419392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":684,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827805+0000","last_change":"2023-10-09T04:29:47.259563+0000","last_active":"2023-10-09T04:30:53.827805+0000","last_peered":"2023-10-09T04:30:53.827805+0000","last_clean":"2023-10-09T04:30:53.827805+0000","last_became_active":"2023-10-09T04:29:47.259210+0000","last_became_peered":"2023-10-09T04:29:47.259210+0000","last_unstale":"2023-10-09T04:30:53.827805+0000","last_undegraded":"2023-10-09T04:30:53.827805+0000","last_fullsized":"2023-10-09T04:30:53.827805+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:16.297729+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":146,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243573+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:30:56.243573+0000","last_peered":"2023-10-09T04:30:56.243573+0000","last_clean":"2023-10-09T04:30:56.243573+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:30:56.243573+0000","last_undegraded":"2023-10-09T04:30:56.243573+0000","last_fullsized":"2023-10-09T04:30:56.243573+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.246135+0000","last_change":"2023-10-09T04:24:56.167368+0000","last_active":"2023-10-09T04:30:56.246135+0000","last_peered":"2023-10-09T04:30:56.246135+0000","last_clean":"2023-10-09T04:30:56.246135+0000","last_became_active":"2023-10-09T04:24:56.165909+0000","last_became_peered":"2023-10-09T04:24:56.165909+0000","last_unstale":"2023-10-09T04:30:56.246135+0000","last_undegraded":"2023-10-09T04:30:56.246135+0000","last_fullsized":"2023-10-09T04:30:56.246135+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:54.177243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"561'12310","reported_seq":12374,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:04.293565+0000","last_change":"2023-10-09T04:27:20.717099+0000","last_active":"2023-10-09T04:31:04.293565+0000","last_peered":"2023-10-09T04:31:04.293565+0000","last_clean":"2023-10-09T04:31:04.293565+0000","last_became_active":"2023-10-09T04:27:20.716810+0000","last_became_peered":"2023-10-09T04:27:20.716810+0000","last_unstale":"2023-10-09T04:31:04.293565+0000","last_undegraded":"2023-10-09T04:31:04.293565+0000","last_fullsized":"2023-10-09T04:31:04.293565+0000","mapping_epoch":507,"log_start":"528'6751","ondisk_log_start":"528'6751","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5559,"log_dups_size":0,"ondisk_log_size":5559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:08.975352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63569920,"num_objects":970,"num_object_clones":0,"num_object_copies":2910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12310,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":700,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242318+0000","last_change":"2023-10-09T04:29:47.234552+0000","last_active":"2023-10-09T04:30:56.242318+0000","last_peered":"2023-10-09T04:30:56.242318+0000","last_clean":"2023-10-09T04:30:56.242318+0000","last_became_active":"2023-10-09T04:29:47.234172+0000","last_became_peered":"2023-10-09T04:29:47.234172+0000","last_unstale":"2023-10-09T04:30:56.242318+0000","last_undegraded":"2023-10-09T04:30:56.242318+0000","last_fullsized":"2023-10-09T04:30:56.242318+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:35.812833+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":228,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.249431+0000","last_change":"2023-10-09T04:20:02.107173+0000","last_active":"2023-10-09T04:30:56.249431+0000","last_peered":"2023-10-09T04:30:56.249431+0000","last_clean":"2023-10-09T04:30:56.249431+0000","last_became_active":"2023-10-09T04:20:02.106840+0000","last_became_peered":"2023-10-09T04:20:02.106840+0000","last_unstale":"2023-10-09T04:30:56.249431+0000","last_undegraded":"2023-10-09T04:30:56.249431+0000","last_fullsized":"2023-10-09T04:30:56.249431+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:40.237805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":146,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.246274+0000","last_change":"2023-10-09T04:30:51.977312+0000","last_active":"2023-10-09T04:30:56.246274+0000","last_peered":"2023-10-09T04:30:56.246274+0000","last_clean":"2023-10-09T04:30:56.246274+0000","last_became_active":"2023-10-09T04:30:51.976877+0000","last_became_peered":"2023-10-09T04:30:51.976877+0000","last_unstale":"2023-10-09T04:30:56.246274+0000","last_undegraded":"2023-10-09T04:30:56.246274+0000","last_fullsized":"2023-10-09T04:30:56.246274+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:10.085422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.d","version":"561'12210","reported_seq":12273,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.214852+0000","last_change":"2023-10-09T04:27:20.775191+0000","last_active":"2023-10-09T04:31:05.214852+0000","last_peered":"2023-10-09T04:31:05.214852+0000","last_clean":"2023-10-09T04:31:05.214852+0000","last_became_active":"2023-10-09T04:27:20.774880+0000","last_became_peered":"2023-10-09T04:27:20.774880+0000","last_unstale":"2023-10-09T04:31:05.214852+0000","last_undegraded":"2023-10-09T04:31:05.214852+0000","last_fullsized":"2023-10-09T04:31:05.214852+0000","mapping_epoch":507,"log_start":"524'6143","ondisk_log_start":"524'6143","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6067,"log_dups_size":0,"ondisk_log_size":6067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:16.329882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999384,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12210,"num_write_kb":419009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":687,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243628+0000","last_change":"2023-10-09T04:22:16.632617+0000","last_active":"2023-10-09T04:30:56.243628+0000","last_peered":"2023-10-09T04:30:56.243628+0000","last_clean":"2023-10-09T04:30:56.243628+0000","last_became_active":"2023-10-09T04:22:16.632221+0000","last_became_peered":"2023-10-09T04:22:16.632221+0000","last_unstale":"2023-10-09T04:30:56.243628+0000","last_undegraded":"2023-10-09T04:30:56.243628+0000","last_fullsized":"2023-10-09T04:30:56.243628+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:22.507465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:22.507465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:57.123379+0000","scrub_duration":0.0039754159999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":164,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828102+0000","last_change":"2023-10-09T04:23:06.340055+0000","last_active":"2023-10-09T04:30:53.828102+0000","last_peered":"2023-10-09T04:30:53.828102+0000","last_clean":"2023-10-09T04:30:53.828102+0000","last_became_active":"2023-10-09T04:23:06.339698+0000","last_became_peered":"2023-10-09T04:23:06.339698+0000","last_unstale":"2023-10-09T04:30:53.828102+0000","last_undegraded":"2023-10-09T04:30:53.828102+0000","last_fullsized":"2023-10-09T04:30:53.828102+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:50.698558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":517,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.249125+0000","last_change":"2023-10-09T04:29:44.187226+0000","last_active":"2023-10-09T04:30:56.249125+0000","last_peered":"2023-10-09T04:30:56.249125+0000","last_clean":"2023-10-09T04:30:56.249125+0000","last_became_active":"2023-10-09T04:29:44.186899+0000","last_became_peered":"2023-10-09T04:29:44.186899+0000","last_unstale":"2023-10-09T04:30:56.249125+0000","last_undegraded":"2023-10-09T04:30:56.249125+0000","last_fullsized":"2023-10-09T04:30:56.249125+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:21.406947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":375,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242502+0000","last_change":"2023-10-09T04:29:44.180975+0000","last_active":"2023-10-09T04:30:56.242502+0000","last_peered":"2023-10-09T04:30:56.242502+0000","last_clean":"2023-10-09T04:30:56.242502+0000","last_became_active":"2023-10-09T04:29:44.180615+0000","last_became_peered":"2023-10-09T04:29:44.180615+0000","last_unstale":"2023-10-09T04:30:56.242502+0000","last_undegraded":"2023-10-09T04:30:56.242502+0000","last_fullsized":"2023-10-09T04:30:56.242502+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:00.181777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.b","version":"560'12330","reported_seq":12391,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:31:02.152163+0000","last_change":"2023-10-09T04:27:20.712863+0000","last_active":"2023-10-09T04:31:02.152163+0000","last_peered":"2023-10-09T04:31:02.152163+0000","last_clean":"2023-10-09T04:31:02.152163+0000","last_became_active":"2023-10-09T04:27:20.712576+0000","last_became_peered":"2023-10-09T04:27:20.712576+0000","last_unstale":"2023-10-09T04:31:02.152163+0000","last_undegraded":"2023-10-09T04:31:02.152163+0000","last_fullsized":"2023-10-09T04:31:02.152163+0000","mapping_epoch":507,"log_start":"525'6351","ondisk_log_start":"525'6351","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:46.571993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71565312,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12330,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":604,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.829147+0000","last_change":"2023-10-09T04:29:47.257461+0000","last_active":"2023-10-09T04:30:53.829147+0000","last_peered":"2023-10-09T04:30:53.829147+0000","last_clean":"2023-10-09T04:30:53.829147+0000","last_became_active":"2023-10-09T04:29:47.257014+0000","last_became_peered":"2023-10-09T04:29:47.257014+0000","last_unstale":"2023-10-09T04:30:53.829147+0000","last_undegraded":"2023-10-09T04:30:53.829147+0000","last_fullsized":"2023-10-09T04:30:53.829147+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:52.834528+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":36,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828740+0000","last_change":"2023-10-09T04:29:44.191398+0000","last_active":"2023-10-09T04:30:53.828740+0000","last_peered":"2023-10-09T04:30:53.828740+0000","last_clean":"2023-10-09T04:30:53.828740+0000","last_became_active":"2023-10-09T04:29:44.191063+0000","last_became_peered":"2023-10-09T04:29:44.191063+0000","last_unstale":"2023-10-09T04:30:53.828740+0000","last_undegraded":"2023-10-09T04:30:53.828740+0000","last_fullsized":"2023-10-09T04:30:53.828740+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:56.910914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242848+0000","last_change":"2023-10-09T04:24:56.164175+0000","last_active":"2023-10-09T04:30:56.242848+0000","last_peered":"2023-10-09T04:30:56.242848+0000","last_clean":"2023-10-09T04:30:56.242848+0000","last_became_active":"2023-10-09T04:24:56.163143+0000","last_became_peered":"2023-10-09T04:24:56.163143+0000","last_unstale":"2023-10-09T04:30:56.242848+0000","last_undegraded":"2023-10-09T04:30:56.242848+0000","last_fullsized":"2023-10-09T04:30:56.242848+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:38:48.261909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827952+0000","last_change":"2023-10-09T04:30:51.977416+0000","last_active":"2023-10-09T04:30:53.827952+0000","last_peered":"2023-10-09T04:30:53.827952+0000","last_clean":"2023-10-09T04:30:53.827952+0000","last_became_active":"2023-10-09T04:30:51.977079+0000","last_became_peered":"2023-10-09T04:30:51.977079+0000","last_unstale":"2023-10-09T04:30:53.827952+0000","last_undegraded":"2023-10-09T04:30:53.827952+0000","last_fullsized":"2023-10-09T04:30:53.827952+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:06.715973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"560'11767","reported_seq":18749,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:31:01.893720+0000","last_change":"2023-10-09T04:30:26.112157+0000","last_active":"2023-10-09T04:31:01.893720+0000","last_peered":"2023-10-09T04:31:01.893720+0000","last_clean":"2023-10-09T04:31:01.893720+0000","last_became_active":"2023-10-09T04:30:26.074223+0000","last_became_peered":"2023-10-09T04:30:26.074223+0000","last_unstale":"2023-10-09T04:31:01.893720+0000","last_undegraded":"2023-10-09T04:31:01.893720+0000","last_fullsized":"2023-10-09T04:31:01.893720+0000","mapping_epoch":547,"log_start":"524'5884","ondisk_log_start":"524'5884","created":507,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5883,"log_dups_size":0,"ondisk_log_size":5883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:14.191269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71237632,"num_objects":1087,"num_object_clones":0,"num_object_copies":3261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11767,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3451,"num_bytes_recovered":226164736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":735,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828657+0000","last_change":"2023-10-09T04:26:12.737003+0000","last_active":"2023-10-09T04:30:53.828657+0000","last_peered":"2023-10-09T04:30:53.828657+0000","last_clean":"2023-10-09T04:30:53.828657+0000","last_became_active":"2023-10-09T04:26:12.736608+0000","last_became_peered":"2023-10-09T04:26:12.736608+0000","last_unstale":"2023-10-09T04:30:53.828657+0000","last_undegraded":"2023-10-09T04:30:53.828657+0000","last_fullsized":"2023-10-09T04:30:53.828657+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:30.055860+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":412,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832682+0000","last_change":"2023-10-09T04:26:21.346649+0000","last_active":"2023-10-09T04:30:53.832682+0000","last_peered":"2023-10-09T04:30:53.832682+0000","last_clean":"2023-10-09T04:30:53.832682+0000","last_became_active":"2023-10-09T04:26:21.346327+0000","last_became_peered":"2023-10-09T04:26:21.346327+0000","last_unstale":"2023-10-09T04:30:53.832682+0000","last_undegraded":"2023-10-09T04:30:53.832682+0000","last_fullsized":"2023-10-09T04:30:53.832682+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:09.977654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":502,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243775+0000","last_change":"2023-10-09T04:26:21.349691+0000","last_active":"2023-10-09T04:30:56.243775+0000","last_peered":"2023-10-09T04:30:56.243775+0000","last_clean":"2023-10-09T04:30:56.243775+0000","last_became_active":"2023-10-09T04:26:21.349201+0000","last_became_peered":"2023-10-09T04:26:21.349201+0000","last_unstale":"2023-10-09T04:30:56.243775+0000","last_undegraded":"2023-10-09T04:30:56.243775+0000","last_fullsized":"2023-10-09T04:30:56.243775+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:36:51.528800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":154,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828784+0000","last_change":"2023-10-09T04:29:44.189319+0000","last_active":"2023-10-09T04:30:53.828784+0000","last_peered":"2023-10-09T04:30:53.828784+0000","last_clean":"2023-10-09T04:30:53.828784+0000","last_became_active":"2023-10-09T04:29:44.188987+0000","last_became_peered":"2023-10-09T04:29:44.188987+0000","last_unstale":"2023-10-09T04:30:53.828784+0000","last_undegraded":"2023-10-09T04:30:53.828784+0000","last_fullsized":"2023-10-09T04:30:53.828784+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:02.006532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"561'12076","reported_seq":16619,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:06.074173+0000","last_change":"2023-10-09T04:30:44.959704+0000","last_active":"2023-10-09T04:31:06.074173+0000","last_peered":"2023-10-09T04:31:06.074173+0000","last_clean":"2023-10-09T04:31:06.074173+0000","last_became_active":"2023-10-09T04:30:44.926653+0000","last_became_peered":"2023-10-09T04:30:44.926653+0000","last_unstale":"2023-10-09T04:31:06.074173+0000","last_undegraded":"2023-10-09T04:31:06.074173+0000","last_fullsized":"2023-10-09T04:31:06.074173+0000","mapping_epoch":554,"log_start":"528'6742","ondisk_log_start":"528'6742","created":507,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:03.880322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58195968,"num_objects":888,"num_object_clones":0,"num_object_copies":2664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12076,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2230,"num_bytes_recovered":146145280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":505,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.235164+0000","last_change":"2023-10-09T04:29:44.188178+0000","last_active":"2023-10-09T04:30:56.235164+0000","last_peered":"2023-10-09T04:30:56.235164+0000","last_clean":"2023-10-09T04:30:56.235164+0000","last_became_active":"2023-10-09T04:29:44.187626+0000","last_became_peered":"2023-10-09T04:29:44.187626+0000","last_unstale":"2023-10-09T04:30:56.235164+0000","last_undegraded":"2023-10-09T04:30:56.235164+0000","last_fullsized":"2023-10-09T04:30:56.235164+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:14:01.695892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":1015,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234628+0000","last_change":"2023-10-09T04:23:08.634655+0000","last_active":"2023-10-09T04:30:56.234628+0000","last_peered":"2023-10-09T04:30:56.234628+0000","last_clean":"2023-10-09T04:30:56.234628+0000","last_became_active":"2023-10-09T04:23:08.634324+0000","last_became_peered":"2023-10-09T04:23:08.634324+0000","last_unstale":"2023-10-09T04:30:56.234628+0000","last_undegraded":"2023-10-09T04:30:56.234628+0000","last_fullsized":"2023-10-09T04:30:56.234628+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:03.975751+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242234+0000","last_change":"2023-10-09T04:29:44.195422+0000","last_active":"2023-10-09T04:30:56.242234+0000","last_peered":"2023-10-09T04:30:56.242234+0000","last_clean":"2023-10-09T04:30:56.242234+0000","last_became_active":"2023-10-09T04:29:44.194489+0000","last_became_peered":"2023-10-09T04:29:44.194489+0000","last_unstale":"2023-10-09T04:30:56.242234+0000","last_undegraded":"2023-10-09T04:30:56.242234+0000","last_fullsized":"2023-10-09T04:30:56.242234+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:33.263534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":123,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243787+0000","last_change":"2023-10-09T04:24:56.176396+0000","last_active":"2023-10-09T04:30:56.243787+0000","last_peered":"2023-10-09T04:30:56.243787+0000","last_clean":"2023-10-09T04:30:56.243787+0000","last_became_active":"2023-10-09T04:24:56.175109+0000","last_became_peered":"2023-10-09T04:24:56.175109+0000","last_unstale":"2023-10-09T04:30:56.243787+0000","last_undegraded":"2023-10-09T04:30:56.243787+0000","last_fullsized":"2023-10-09T04:30:56.243787+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:11.872866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"560'11986","reported_seq":12049,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:31:02.148280+0000","last_change":"2023-10-09T04:27:20.714619+0000","last_active":"2023-10-09T04:31:02.148280+0000","last_peered":"2023-10-09T04:31:02.148280+0000","last_clean":"2023-10-09T04:31:02.148280+0000","last_became_active":"2023-10-09T04:27:20.714302+0000","last_became_peered":"2023-10-09T04:27:20.714302+0000","last_unstale":"2023-10-09T04:31:02.148280+0000","last_undegraded":"2023-10-09T04:31:02.148280+0000","last_fullsized":"2023-10-09T04:31:02.148280+0000","mapping_epoch":507,"log_start":"524'6034","ondisk_log_start":"524'6034","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:02.553931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11986,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":656,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242902+0000","last_change":"2023-10-09T04:24:58.731960+0000","last_active":"2023-10-09T04:30:56.242902+0000","last_peered":"2023-10-09T04:30:56.242902+0000","last_clean":"2023-10-09T04:30:56.242902+0000","last_became_active":"2023-10-09T04:24:58.731691+0000","last_became_peered":"2023-10-09T04:24:58.731691+0000","last_unstale":"2023-10-09T04:30:56.242902+0000","last_undegraded":"2023-10-09T04:30:56.242902+0000","last_fullsized":"2023-10-09T04:30:56.242902+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:42.803978+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":561,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234549+0000","last_change":"2023-10-09T04:23:06.349099+0000","last_active":"2023-10-09T04:30:56.234549+0000","last_peered":"2023-10-09T04:30:56.234549+0000","last_clean":"2023-10-09T04:30:56.234549+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:30:56.234549+0000","last_undegraded":"2023-10-09T04:30:56.234549+0000","last_fullsized":"2023-10-09T04:30:56.234549+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":202,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832991+0000","last_change":"2023-10-09T04:27:20.722165+0000","last_active":"2023-10-09T04:30:53.832991+0000","last_peered":"2023-10-09T04:30:53.832991+0000","last_clean":"2023-10-09T04:30:53.832991+0000","last_became_active":"2023-10-09T04:27:20.721839+0000","last_became_peered":"2023-10-09T04:27:20.721839+0000","last_unstale":"2023-10-09T04:30:53.832991+0000","last_undegraded":"2023-10-09T04:30:53.832991+0000","last_fullsized":"2023-10-09T04:30:53.832991+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:26.595991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":184,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828292+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:30:53.828292+0000","last_peered":"2023-10-09T04:30:53.828292+0000","last_clean":"2023-10-09T04:30:53.828292+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:30:53.828292+0000","last_undegraded":"2023-10-09T04:30:53.828292+0000","last_fullsized":"2023-10-09T04:30:53.828292+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"561'12274","reported_seq":12336,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.149021+0000","last_change":"2023-10-09T04:27:20.775292+0000","last_active":"2023-10-09T04:31:05.149021+0000","last_peered":"2023-10-09T04:31:05.149021+0000","last_clean":"2023-10-09T04:31:05.149021+0000","last_became_active":"2023-10-09T04:27:20.774989+0000","last_became_peered":"2023-10-09T04:27:20.774989+0000","last_unstale":"2023-10-09T04:31:05.149021+0000","last_undegraded":"2023-10-09T04:31:05.149021+0000","last_fullsized":"2023-10-09T04:31:05.149021+0000","mapping_epoch":507,"log_start":"525'6191","ondisk_log_start":"525'6191","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6083,"log_dups_size":0,"ondisk_log_size":6083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:14.994784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58720256,"num_objects":896,"num_object_clones":0,"num_object_copies":2688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12274,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":679,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243348+0000","last_change":"2023-10-09T04:23:08.633549+0000","last_active":"2023-10-09T04:30:56.243348+0000","last_peered":"2023-10-09T04:30:56.243348+0000","last_clean":"2023-10-09T04:30:56.243348+0000","last_became_active":"2023-10-09T04:23:08.633188+0000","last_became_peered":"2023-10-09T04:23:08.633188+0000","last_unstale":"2023-10-09T04:30:56.243348+0000","last_undegraded":"2023-10-09T04:30:56.243348+0000","last_fullsized":"2023-10-09T04:30:56.243348+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:17.019225+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":584,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238572+0000","last_change":"2023-10-09T04:24:56.184869+0000","last_active":"2023-10-09T04:30:56.238572+0000","last_peered":"2023-10-09T04:30:56.238572+0000","last_clean":"2023-10-09T04:30:56.238572+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:30:56.238572+0000","last_undegraded":"2023-10-09T04:30:56.238572+0000","last_fullsized":"2023-10-09T04:30:56.238572+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":36,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828005+0000","last_change":"2023-10-09T04:29:44.178854+0000","last_active":"2023-10-09T04:30:53.828005+0000","last_peered":"2023-10-09T04:30:53.828005+0000","last_clean":"2023-10-09T04:30:53.828005+0000","last_became_active":"2023-10-09T04:29:44.178554+0000","last_became_peered":"2023-10-09T04:29:44.178554+0000","last_unstale":"2023-10-09T04:30:53.828005+0000","last_undegraded":"2023-10-09T04:30:53.828005+0000","last_fullsized":"2023-10-09T04:30:53.828005+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:56.730403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827836+0000","last_change":"2023-10-09T04:29:44.189508+0000","last_active":"2023-10-09T04:30:53.827836+0000","last_peered":"2023-10-09T04:30:53.827836+0000","last_clean":"2023-10-09T04:30:53.827836+0000","last_became_active":"2023-10-09T04:29:44.189002+0000","last_became_peered":"2023-10-09T04:29:44.189002+0000","last_unstale":"2023-10-09T04:30:53.827836+0000","last_undegraded":"2023-10-09T04:30:53.827836+0000","last_fullsized":"2023-10-09T04:30:53.827836+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:44.869908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"561'12235","reported_seq":12297,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.689587+0000","last_change":"2023-10-09T04:27:20.775271+0000","last_active":"2023-10-09T04:31:05.689587+0000","last_peered":"2023-10-09T04:31:05.689587+0000","last_clean":"2023-10-09T04:31:05.689587+0000","last_became_active":"2023-10-09T04:27:20.774991+0000","last_became_peered":"2023-10-09T04:27:20.774991+0000","last_unstale":"2023-10-09T04:31:05.689587+0000","last_undegraded":"2023-10-09T04:31:05.689587+0000","last_fullsized":"2023-10-09T04:31:05.689587+0000","mapping_epoch":507,"log_start":"527'6545","ondisk_log_start":"527'6545","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:07.264927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58392576,"num_objects":891,"num_object_clones":0,"num_object_copies":2673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12235,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":618,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243864+0000","last_change":"2023-10-09T04:27:10.344536+0000","last_active":"2023-10-09T04:30:56.243864+0000","last_peered":"2023-10-09T04:30:56.243864+0000","last_clean":"2023-10-09T04:30:56.243864+0000","last_became_active":"2023-10-09T04:26:23.684619+0000","last_became_peered":"2023-10-09T04:26:23.684619+0000","last_unstale":"2023-10-09T04:30:56.243864+0000","last_undegraded":"2023-10-09T04:30:56.243864+0000","last_fullsized":"2023-10-09T04:30:56.243864+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:25:07.464299+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":521,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.238907+0000","last_change":"2023-10-09T04:23:06.366626+0000","last_active":"2023-10-09T04:30:56.238907+0000","last_peered":"2023-10-09T04:30:56.238907+0000","last_clean":"2023-10-09T04:30:56.238907+0000","last_became_active":"2023-10-09T04:23:06.344713+0000","last_became_peered":"2023-10-09T04:23:06.344713+0000","last_unstale":"2023-10-09T04:30:56.238907+0000","last_undegraded":"2023-10-09T04:30:56.238907+0000","last_fullsized":"2023-10-09T04:30:56.238907+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:26.325686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.246181+0000","last_change":"2023-10-09T04:24:56.167263+0000","last_active":"2023-10-09T04:30:56.246181+0000","last_peered":"2023-10-09T04:30:56.246181+0000","last_clean":"2023-10-09T04:30:56.246181+0000","last_became_active":"2023-10-09T04:24:56.165898+0000","last_became_peered":"2023-10-09T04:24:56.165898+0000","last_unstale":"2023-10-09T04:30:56.246181+0000","last_undegraded":"2023-10-09T04:30:56.246181+0000","last_fullsized":"2023-10-09T04:30:56.246181+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:59.378691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":469,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.235008+0000","last_change":"2023-10-09T04:26:09.693757+0000","last_active":"2023-10-09T04:30:56.235008+0000","last_peered":"2023-10-09T04:30:56.235008+0000","last_clean":"2023-10-09T04:30:56.235008+0000","last_became_active":"2023-10-09T04:26:09.693496+0000","last_became_peered":"2023-10-09T04:26:09.693496+0000","last_unstale":"2023-10-09T04:30:56.235008+0000","last_undegraded":"2023-10-09T04:30:56.235008+0000","last_fullsized":"2023-10-09T04:30:56.235008+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:59.410607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.e","version":"561'11978","reported_seq":12040,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-09T04:31:05.219362+0000","last_change":"2023-10-09T04:27:20.721920+0000","last_active":"2023-10-09T04:31:05.219362+0000","last_peered":"2023-10-09T04:31:05.219362+0000","last_clean":"2023-10-09T04:31:05.219362+0000","last_became_active":"2023-10-09T04:27:20.721674+0000","last_became_peered":"2023-10-09T04:27:20.721674+0000","last_unstale":"2023-10-09T04:31:05.219362+0000","last_undegraded":"2023-10-09T04:31:05.219362+0000","last_fullsized":"2023-10-09T04:31:05.219362+0000","mapping_epoch":507,"log_start":"524'5960","ondisk_log_start":"524'5960","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:27:19.694573+0000","last_clean_scrub_stamp":"2023-10-09T04:27:19.694573+0000","objects_scrubbed":0,"log_size":6018,"log_dups_size":0,"ondisk_log_size":6018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:33.533237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61734912,"num_objects":942,"num_object_clones":0,"num_object_copies":2826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11978,"num_write_kb":413440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":755,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.247522+0000","last_change":"2023-10-09T04:29:47.253692+0000","last_active":"2023-10-09T04:30:56.247522+0000","last_peered":"2023-10-09T04:30:56.247522+0000","last_clean":"2023-10-09T04:30:56.247522+0000","last_became_active":"2023-10-09T04:29:47.253340+0000","last_became_peered":"2023-10-09T04:29:47.253340+0000","last_unstale":"2023-10-09T04:30:56.247522+0000","last_undegraded":"2023-10-09T04:30:56.247522+0000","last_fullsized":"2023-10-09T04:30:56.247522+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:20.432109+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:20.432109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:17.491055+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":360,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243756+0000","last_change":"2023-10-09T04:26:21.408541+0000","last_active":"2023-10-09T04:30:56.243756+0000","last_peered":"2023-10-09T04:30:56.243756+0000","last_clean":"2023-10-09T04:30:56.243756+0000","last_became_active":"2023-10-09T04:26:21.408132+0000","last_became_peered":"2023-10-09T04:26:21.408132+0000","last_unstale":"2023-10-09T04:30:56.243756+0000","last_undegraded":"2023-10-09T04:30:56.243756+0000","last_fullsized":"2023-10-09T04:30:56.243756+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:43.595459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":196,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828477+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:30:53.828477+0000","last_peered":"2023-10-09T04:30:53.828477+0000","last_clean":"2023-10-09T04:30:53.828477+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:30:53.828477+0000","last_undegraded":"2023-10-09T04:30:53.828477+0000","last_fullsized":"2023-10-09T04:30:53.828477+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243416+0000","last_change":"2023-10-09T04:29:44.183037+0000","last_active":"2023-10-09T04:30:56.243416+0000","last_peered":"2023-10-09T04:30:56.243416+0000","last_clean":"2023-10-09T04:30:56.243416+0000","last_became_active":"2023-10-09T04:29:44.182667+0000","last_became_peered":"2023-10-09T04:29:44.182667+0000","last_unstale":"2023-10-09T04:30:56.243416+0000","last_undegraded":"2023-10-09T04:30:56.243416+0000","last_fullsized":"2023-10-09T04:30:56.243416+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:41:35.337400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":604,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832756+0000","last_change":"2023-10-09T04:27:03.845612+0000","last_active":"2023-10-09T04:30:53.832756+0000","last_peered":"2023-10-09T04:30:53.832756+0000","last_clean":"2023-10-09T04:30:53.832756+0000","last_became_active":"2023-10-09T04:23:08.628550+0000","last_became_peered":"2023-10-09T04:23:08.628550+0000","last_unstale":"2023-10-09T04:30:53.832756+0000","last_undegraded":"2023-10-09T04:30:53.832756+0000","last_fullsized":"2023-10-09T04:30:53.832756+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:00.276189+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":193,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.827920+0000","last_change":"2023-10-09T04:30:51.979500+0000","last_active":"2023-10-09T04:30:53.827920+0000","last_peered":"2023-10-09T04:30:53.827920+0000","last_clean":"2023-10-09T04:30:53.827920+0000","last_became_active":"2023-10-09T04:30:51.979149+0000","last_became_peered":"2023-10-09T04:30:51.979149+0000","last_unstale":"2023-10-09T04:30:53.827920+0000","last_undegraded":"2023-10-09T04:30:53.827920+0000","last_fullsized":"2023-10-09T04:30:53.827920+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:52.702260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":110,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.244709+0000","last_change":"2023-10-09T04:30:51.973822+0000","last_active":"2023-10-09T04:30:56.244709+0000","last_peered":"2023-10-09T04:30:56.244709+0000","last_clean":"2023-10-09T04:30:56.244709+0000","last_became_active":"2023-10-09T04:30:51.973546+0000","last_became_peered":"2023-10-09T04:30:51.973546+0000","last_unstale":"2023-10-09T04:30:56.244709+0000","last_undegraded":"2023-10-09T04:30:56.244709+0000","last_fullsized":"2023-10-09T04:30:56.244709+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:42.932321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":635,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243266+0000","last_change":"2023-10-09T04:22:18.236512+0000","last_active":"2023-10-09T04:30:56.243266+0000","last_peered":"2023-10-09T04:30:56.243266+0000","last_clean":"2023-10-09T04:30:56.243266+0000","last_became_active":"2023-10-09T04:22:16.632403+0000","last_became_peered":"2023-10-09T04:22:16.632403+0000","last_unstale":"2023-10-09T04:30:56.243266+0000","last_undegraded":"2023-10-09T04:30:56.243266+0000","last_fullsized":"2023-10-09T04:30:56.243266+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:21.868307+0000","scrub_duration":0.003682917,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":492,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.244792+0000","last_change":"2023-10-09T04:29:44.199915+0000","last_active":"2023-10-09T04:30:56.244792+0000","last_peered":"2023-10-09T04:30:56.244792+0000","last_clean":"2023-10-09T04:30:56.244792+0000","last_became_active":"2023-10-09T04:29:44.188601+0000","last_became_peered":"2023-10-09T04:29:44.188601+0000","last_unstale":"2023-10-09T04:30:56.244792+0000","last_undegraded":"2023-10-09T04:30:56.244792+0000","last_fullsized":"2023-10-09T04:30:56.244792+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:10.093968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":126,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242776+0000","last_change":"2023-10-09T04:24:56.169453+0000","last_active":"2023-10-09T04:30:56.242776+0000","last_peered":"2023-10-09T04:30:56.242776+0000","last_clean":"2023-10-09T04:30:56.242776+0000","last_became_active":"2023-10-09T04:24:56.167398+0000","last_became_peered":"2023-10-09T04:24:56.167398+0000","last_unstale":"2023-10-09T04:30:56.242776+0000","last_undegraded":"2023-10-09T04:30:56.242776+0000","last_fullsized":"2023-10-09T04:30:56.242776+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:34.017574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":503,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243630+0000","last_change":"2023-10-09T04:29:44.203641+0000","last_active":"2023-10-09T04:30:56.243630+0000","last_peered":"2023-10-09T04:30:56.243630+0000","last_clean":"2023-10-09T04:30:56.243630+0000","last_became_active":"2023-10-09T04:29:44.194615+0000","last_became_peered":"2023-10-09T04:29:44.194615+0000","last_unstale":"2023-10-09T04:30:56.243630+0000","last_undegraded":"2023-10-09T04:30:56.243630+0000","last_fullsized":"2023-10-09T04:30:56.243630+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:30.344525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":63,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242392+0000","last_change":"2023-10-09T04:27:20.721322+0000","last_active":"2023-10-09T04:30:56.242392+0000","last_peered":"2023-10-09T04:30:56.242392+0000","last_clean":"2023-10-09T04:30:56.242392+0000","last_became_active":"2023-10-09T04:27:20.721072+0000","last_became_peered":"2023-10-09T04:27:20.721072+0000","last_unstale":"2023-10-09T04:30:56.242392+0000","last_undegraded":"2023-10-09T04:30:56.242392+0000","last_fullsized":"2023-10-09T04:30:56.242392+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:35.448295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":814,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.234755+0000","last_change":"2023-10-09T04:27:00.788289+0000","last_active":"2023-10-09T04:30:56.234755+0000","last_peered":"2023-10-09T04:30:56.234755+0000","last_clean":"2023-10-09T04:30:56.234755+0000","last_became_active":"2023-10-09T04:26:12.736494+0000","last_became_peered":"2023-10-09T04:26:12.736494+0000","last_unstale":"2023-10-09T04:30:56.234755+0000","last_undegraded":"2023-10-09T04:30:56.234755+0000","last_fullsized":"2023-10-09T04:30:56.234755+0000","mapping_epoch":468,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:38.056935+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":296,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.249718+0000","last_change":"2023-10-09T04:26:21.349932+0000","last_active":"2023-10-09T04:30:56.249718+0000","last_peered":"2023-10-09T04:30:56.249718+0000","last_clean":"2023-10-09T04:30:56.249718+0000","last_became_active":"2023-10-09T04:26:21.349579+0000","last_became_peered":"2023-10-09T04:26:21.349579+0000","last_unstale":"2023-10-09T04:30:56.249718+0000","last_undegraded":"2023-10-09T04:30:56.249718+0000","last_fullsized":"2023-10-09T04:30:56.249718+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:57.119779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":411,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.832609+0000","last_change":"2023-10-09T04:26:21.346890+0000","last_active":"2023-10-09T04:30:53.832609+0000","last_peered":"2023-10-09T04:30:53.832609+0000","last_clean":"2023-10-09T04:30:53.832609+0000","last_became_active":"2023-10-09T04:26:21.346567+0000","last_became_peered":"2023-10-09T04:26:21.346567+0000","last_unstale":"2023-10-09T04:30:53.832609+0000","last_undegraded":"2023-10-09T04:30:53.832609+0000","last_fullsized":"2023-10-09T04:30:53.832609+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:09.204466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":672,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.243714+0000","last_change":"2023-10-09T04:25:03.677979+0000","last_active":"2023-10-09T04:30:56.243714+0000","last_peered":"2023-10-09T04:30:56.243714+0000","last_clean":"2023-10-09T04:30:56.243714+0000","last_became_active":"2023-10-09T04:25:03.677410+0000","last_became_peered":"2023-10-09T04:25:03.677410+0000","last_unstale":"2023-10-09T04:30:56.243714+0000","last_undegraded":"2023-10-09T04:30:56.243714+0000","last_fullsized":"2023-10-09T04:30:56.243714+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:49.875014+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":203,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-09T04:30:53.828963+0000","last_change":"2023-10-09T04:29:44.196276+0000","last_active":"2023-10-09T04:30:53.828963+0000","last_peered":"2023-10-09T04:30:53.828963+0000","last_clean":"2023-10-09T04:30:53.828963+0000","last_became_active":"2023-10-09T04:29:44.187018+0000","last_became_peered":"2023-10-09T04:29:44.187018+0000","last_unstale":"2023-10-09T04:30:53.828963+0000","last_undegraded":"2023-10-09T04:30:53.828963+0000","last_fullsized":"2023-10-09T04:30:53.828963+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:41.215750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":136,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-09T04:30:56.242116+0000","last_change":"2023-10-09T04:30:51.978882+0000","last_active":"2023-10-09T04:30:56.242116+0000","last_peered":"2023-10-09T04:30:56.242116+0000","last_clean":"2023-10-09T04:30:56.242116+0000","last_became_active":"2023-10-09T04:30:51.977708+0000","last_became_peered":"2023-10-09T04:30:51.977708+0000","last_unstale":"2023-10-09T04:30:56.242116+0000","last_undegraded":"2023-10-09T04:30:56.242116+0000","last_fullsized":"2023-10-09T04:30:56.242116+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:47.210981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1002635288,"num_objects":15300,"num_object_clones":0,"num_object_copies":45900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15300,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194254,"num_write_kb":6705665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11931,"num_bytes_recovered":781910016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3335991296,"data_stored":3335979080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92962,"ondisk_log_size":92962,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":12238,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1391225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201743,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1265896,"kb_used_data":365984,"kb_used_omap":0,"kb_used_meta":899904,"kb_avail":92479256,"statfs":{"total":95995035648,"available":94698758144,"internally_reserved":0,"allocated":374767616,"data_stored":373207556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921501696},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:58 2023","interfaces":[{"interface":"back","average":{"1min":18.584,"5min":6.8810000000000002,"15min":4.2770000000000001},"min":{"1min":0.28299999999999997,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":218.697,"5min":218.697,"15min":218.697},"last":2.3460000000000001},{"interface":"front","average":{"1min":18.992000000000001,"5min":6.6779999999999999,"15min":4.2759999999999998},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.221},"max":{"1min":218.83099999999999,"5min":218.83099999999999,"15min":218.83099999999999},"last":3.2229999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":5.6619999999999999,"15min":6.5970000000000004},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.22600000000000001},"max":{"1min":16.129000000000001,"5min":213.41800000000001,"15min":261.935},"last":1.798},{"interface":"front","average":{"1min":1.873,"5min":5.2249999999999996,"15min":6.3550000000000004},"min":{"1min":0.29699999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.2969999999999997,"5min":213.352,"15min":262.07600000000002},"last":0.84499999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:30:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":2.0470000000000002,"15min":2.742},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.25},"max":{"1min":7.5309999999999997,"5min":37.131999999999998,"15min":253.41200000000001},"last":1.091},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.9350000000000001,"15min":3.04},"min":{"1min":0.38500000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5439999999999996,"5min":37.145000000000003,"15min":253.39400000000001},"last":3.1890000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":5.7809999999999997,"5min":3.6659999999999999,"15min":3.464},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":77.245000000000005,"5min":77.245000000000005,"15min":255.97300000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":5.3540000000000001,"5min":3.5489999999999999,"15min":3.6339999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":77.263999999999996,"5min":77.263999999999996,"15min":256.06099999999998},"last":0.55300000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:30:58 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.147,"15min":0.93999999999999995},"min":{"1min":0.27700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.7490000000000006,"5min":9.7490000000000006,"15min":9.7490000000000006},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.135,"5min":1.23,"15min":1.2669999999999999},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.251},"max":{"1min":4.4160000000000004,"5min":6.1299999999999999,"15min":12.566000000000001},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":1.3089999999999999,"15min":1.4379999999999999},"min":{"1min":0.29699999999999999,"5min":0.28899999999999998,"15min":0.254},"max":{"1min":5.0620000000000003,"5min":7.4820000000000002,"15min":17.882999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.3859999999999999,"5min":1.278,"15min":1.2729999999999999},"min":{"1min":0.311,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":7.1230000000000002,"5min":7.8360000000000003,"15min":17.568999999999999},"last":0.85999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":1.431,"15min":1.367},"min":{"1min":0.35299999999999998,"5min":0.31,"15min":0.31},"max":{"1min":6.5380000000000003,"5min":9.5069999999999997,"15min":9.5069999999999997},"last":0.375},{"interface":"front","average":{"1min":1.879,"5min":1.488,"15min":1.1619999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":9.8089999999999993,"5min":9.8089999999999993,"15min":9.8089999999999993},"last":0.32100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463379,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1391132,"kb_used_data":491604,"kb_used_omap":0,"kb_used_meta":899520,"kb_avail":92354020,"statfs":{"total":95995035648,"available":94570516480,"internally_reserved":0,"allocated":503402496,"data_stored":501854289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921108480},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:55 2023","interfaces":[{"interface":"back","average":{"1min":10.416,"5min":6.851,"15min":4.7960000000000003},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":134.34399999999999,"5min":162.524,"15min":162.524},"last":0.32400000000000001},{"interface":"front","average":{"1min":10.141,"5min":6.806,"15min":4.8630000000000004},"min":{"1min":0.26400000000000001,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":130.87100000000001,"5min":161.73599999999999,"15min":161.73599999999999},"last":0.33900000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":1.972,"15min":4.4710000000000001},"min":{"1min":0.24299999999999999,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":10.055,"5min":18.971,"15min":315.73599999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":3.1269999999999998,"5min":2.004,"15min":4.5510000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.603999999999999,"5min":18.943999999999999,"15min":316.91500000000002},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:30:55 2023","interfaces":[{"interface":"back","average":{"1min":3.4199999999999999,"5min":3.331,"15min":2.7730000000000001},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.8330000000000002,"5min":116.084,"15min":116.084},"last":0.35799999999999998},{"interface":"front","average":{"1min":3.46,"5min":3.3780000000000001,"15min":2.8159999999999998},"min":{"1min":0.34300000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.8710000000000004,"5min":116.11199999999999,"15min":116.11199999999999},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":8.1199999999999992,"5min":2.8100000000000001,"15min":2.331},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.25600000000000001},"max":{"1min":127.44199999999999,"5min":127.44199999999999,"15min":127.44199999999999},"last":0.31},{"interface":"front","average":{"1min":9.3219999999999992,"5min":3.0379999999999998,"15min":2.3969999999999998},"min":{"1min":0.34200000000000003,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":127.51900000000001,"5min":127.51900000000001,"15min":127.51900000000001},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.0029999999999999,"15min":1.532},"min":{"1min":0.27600000000000002,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":5.5659999999999998,"5min":17.545000000000002,"15min":161.27500000000001},"last":90.221999999999994},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.1439999999999999,"15min":1.637},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.22600000000000001},"max":{"1min":3.0699999999999998,"5min":15.212999999999999,"15min":160.32599999999999},"last":90.116}]},{"osd":5,"last update":"Mon Oct 9 04:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.6839999999999999,"15min":1.306},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26100000000000001},"max":{"1min":2.7959999999999998,"5min":74.043000000000006,"15min":74.043000000000006},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.875,"5min":1.7989999999999999,"15min":1.3280000000000001},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.27900000000000003},"max":{"1min":2.8180000000000001,"5min":74.073999999999998,"15min":74.073999999999998},"last":0.39700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:30:50 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.92600000000000005,"15min":1.6319999999999999},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7480000000000002,"5min":4.6189999999999998,"15min":161.39400000000001},"last":93.212000000000003},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.234,"15min":1.722},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.27800000000000002},"max":{"1min":2.6520000000000001,"5min":10.77,"15min":161.41200000000001},"last":93.257000000000005}]}]},{"osd":1,"up_from":89,"seq":382252089688,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1477808,"kb_used_data":590888,"kb_used_omap":9,"kb_used_meta":886838,"kb_avail":92267344,"statfs":{"total":95995035648,"available":94481760256,"internally_reserved":0,"allocated":605069312,"data_stored":602688537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":908123113},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":3.5609999999999999,"15min":4.3239999999999998},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":33.351999999999997,"5min":86.963999999999999,"15min":259.26799999999997},"last":0.188},{"interface":"front","average":{"1min":5.4699999999999998,"5min":3.988,"15min":5.3090000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.19400000000000001},"max":{"1min":35.271000000000001,"5min":83.213999999999999,"15min":258.46899999999999},"last":2.4740000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:30:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":2.1659999999999999,"15min":3.1219999999999999},"min":{"1min":0.20899999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.919,"5min":74.474000000000004,"15min":259.315},"last":0.34999999999999998},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.3719999999999999,"15min":3.484},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.718,"5min":75.653000000000006,"15min":259.733},"last":2.5139999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":14.018000000000001,"5min":4.8899999999999997,"15min":3.242},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.22600000000000001},"max":{"1min":156.261,"5min":156.261,"15min":156.261},"last":0.376},{"interface":"front","average":{"1min":12.929,"5min":5.1210000000000004,"15min":3.6000000000000001},"min":{"1min":0.36299999999999999,"5min":0.27700000000000002,"15min":0.24299999999999999},"max":{"1min":155.684,"5min":155.684,"15min":155.684},"last":2.7599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":2.028,"15min":2.9860000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26900000000000002},"max":{"1min":4.8170000000000002,"5min":39.052,"15min":257.83300000000003},"last":2.5779999999999998},{"interface":"front","average":{"1min":2.778,"5min":2.0880000000000001,"15min":2.7839999999999998},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":14.32,"5min":39.040999999999997,"15min":257.863},"last":2.4950000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3420000000000001,"5min":1.6619999999999999,"15min":2.1419999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":4.8330000000000002,"5min":17.751999999999999,"15min":38.848999999999997},"last":2.6179999999999999},{"interface":"front","average":{"1min":2.694,"5min":1.99,"15min":2.2799999999999998},"min":{"1min":0.44400000000000001,"5min":0.35699999999999998,"15min":0.29799999999999999},"max":{"1min":14.356999999999999,"5min":14.356999999999999,"15min":39.593000000000004},"last":2.5680000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":2.9740000000000002,"5min":2.8180000000000001,"15min":2.1659999999999999},"min":{"1min":0.46800000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.406000000000001,"5min":50.814999999999998,"15min":50.814999999999998},"last":2.5939999999999999},{"interface":"front","average":{"1min":3.161,"5min":2.46,"15min":1.915},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29999999999999999},"max":{"1min":13.026,"5min":50.749000000000002,"15min":50.749000000000002},"last":0.42699999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:30:40 2023","interfaces":[{"interface":"back","average":{"1min":2.988,"5min":2.4180000000000001,"15min":3.887},"min":{"1min":0.499,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":14.382999999999999,"5min":36.765999999999998,"15min":259.08499999999998},"last":2.5329999999999999},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.1000000000000001,"15min":3.5990000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33100000000000002},"max":{"1min":12.994,"5min":36.75,"15min":259.13299999999998},"last":0.40500000000000003}]}]},{"osd":0,"up_from":84,"seq":360777253213,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375948,"kb_used_data":508356,"kb_used_omap":28,"kb_used_meta":867491,"kb_avail":92369204,"statfs":{"total":95995035648,"available":94586064896,"internally_reserved":0,"allocated":520556544,"data_stored":518028550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":888311632},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:30:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.6930000000000001,"15min":4.4560000000000004},"min":{"1min":0.193,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":12.641999999999999,"5min":12.641999999999999,"15min":209.215},"last":0.98899999999999999},{"interface":"front","average":{"1min":2.141,"5min":1.615,"15min":4.157},"min":{"1min":0.20599999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.7220000000000004,"5min":16.131,"15min":120.934},"last":0.57099999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":1.9390000000000001,"15min":2.516},"min":{"1min":0.217,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":9.7699999999999996,"5min":17.562999999999999,"15min":88.088999999999999},"last":1.343},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.0529999999999999,"15min":2.887},"min":{"1min":0.22900000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.9619999999999997,"5min":9.9659999999999993,"15min":88.004999999999995},"last":0.52900000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:30:29 2023","interfaces":[{"interface":"back","average":{"1min":4.327,"5min":2.0019999999999998,"15min":2.7149999999999999},"min":{"1min":0.23400000000000001,"5min":0.21099999999999999,"15min":0.192},"max":{"1min":19.507999999999999,"5min":19.507999999999999,"15min":99.709000000000003},"last":6.8970000000000002},{"interface":"front","average":{"1min":4.1769999999999996,"5min":2.286,"15min":2.8109999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":19.741,"5min":19.741,"15min":99.560000000000002},"last":5.2089999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.1320000000000001,"15min":2.5089999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28899999999999998},"max":{"1min":7.9770000000000003,"5min":86.906999999999996,"15min":88.058000000000007},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.8700000000000001,"5min":2.2519999999999998,"15min":2.4830000000000001},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.24199999999999999},"max":{"1min":5.625,"5min":86.891000000000005,"15min":86.891000000000005},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":3.7869999999999999,"15min":3.855},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":13.198,"5min":134.93000000000001,"15min":156.96100000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.9009999999999998,"5min":3.5960000000000001,"15min":3.3879999999999999},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.29099999999999998},"max":{"1min":8.5399999999999991,"5min":135.04599999999999,"15min":156.97300000000001},"last":1.117}]},{"osd":6,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":2.698,"5min":1.5169999999999999,"15min":1.8759999999999999},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.34300000000000003},"max":{"1min":8.4920000000000009,"5min":8.4920000000000009,"15min":63.468000000000004},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.4910000000000001,"15min":2.1749999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34100000000000003},"max":{"1min":11.052,"5min":12.26,"15min":88.180000000000007},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":2.5670000000000002,"15min":2.3170000000000002},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":8.4870000000000001,"5min":78.307000000000002,"15min":78.307000000000002},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.748,"5min":2.4369999999999998,"15min":2.645},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.548999999999999,"5min":78.216999999999999,"15min":88.120000000000005},"last":0.59599999999999997}]}]},{"osd":2,"up_from":94,"seq":403726926162,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196556,"kb_used_data":371652,"kb_used_omap":81,"kb_used_meta":824814,"kb_avail":92548596,"statfs":{"total":95995035648,"available":94769762304,"internally_reserved":0,"allocated":380571648,"data_stored":378169056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":844609754},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":3.8530000000000002,"15min":3.5830000000000002},"min":{"1min":0.19900000000000001,"5min":0.18099999999999999,"15min":0.17699999999999999},"max":{"1min":12.644,"5min":132.87299999999999,"15min":132.87299999999999},"last":8.0950000000000006},{"interface":"front","average":{"1min":2.819,"5min":4.2480000000000002,"15min":3.6619999999999999},"min":{"1min":0.23400000000000001,"5min":0.188,"15min":0.188},"max":{"1min":12.710000000000001,"5min":130.398,"15min":130.398},"last":0.317}]},{"osd":1,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":4.109,"5min":3.3090000000000002,"15min":4.7800000000000002},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.191},"max":{"1min":14.109999999999999,"5min":122.17100000000001,"15min":158.13399999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":4.2169999999999996,"5min":3.1680000000000001,"15min":4.8209999999999997},"min":{"1min":0.21299999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":17.061,"5min":121.91800000000001,"15min":161.12},"last":4.2830000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:30:10 2023","interfaces":[{"interface":"back","average":{"1min":3.9510000000000001,"5min":2.3490000000000002,"15min":2.5179999999999998},"min":{"1min":0.27300000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":43.707999999999998,"5min":45.023000000000003,"15min":58.997},"last":0.42099999999999999},{"interface":"front","average":{"1min":5.641,"5min":2.7200000000000002,"15min":2.8679999999999999},"min":{"1min":0.26600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":43.427999999999997,"5min":44.923999999999999,"15min":66.135000000000005},"last":2.306}]},{"osd":4,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":4.3099999999999996,"15min":2.9609999999999999},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.4149999999999991,"5min":196.57599999999999,"15min":196.57599999999999},"last":0.755},{"interface":"front","average":{"1min":1.956,"5min":3.1619999999999999,"15min":2.3450000000000002},"min":{"1min":0.312,"5min":0.27900000000000003,"15min":0.26800000000000002},"max":{"1min":9.0289999999999999,"5min":201.81100000000001,"15min":201.81100000000001},"last":0.496}]},{"osd":5,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":3.3999999999999999,"15min":3.0870000000000002},"min":{"1min":0.379,"5min":0.28299999999999997,"15min":0.26400000000000001},"max":{"1min":8.6699999999999999,"5min":190.06899999999999,"15min":190.06899999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.7370000000000001,"5min":4.4509999999999996,"15min":3.4119999999999999},"min":{"1min":0.34200000000000003,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":9.8949999999999996,"5min":190.08500000000001,"15min":190.08500000000001},"last":0.72099999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":1.7989999999999999,"15min":1.853},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":9.0779999999999994,"5min":58.469999999999999,"15min":58.469999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.2469999999999999,"5min":2.4489999999999998,"15min":2.3159999999999998},"min":{"1min":0.40699999999999997,"5min":0.36199999999999999,"15min":0.26800000000000002},"max":{"1min":8.6989999999999998,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.66700000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:30:43 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":3.2250000000000001,"15min":2.7589999999999999},"min":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.32800000000000001},"max":{"1min":9.1120000000000001,"5min":191.792,"15min":191.792},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.585,"5min":4.391,"15min":3.4529999999999998},"min":{"1min":0.42099999999999999,"5min":0.372,"15min":0.34399999999999997},"max":{"1min":8.843,"5min":191.828,"15min":191.828},"last":0.60599999999999998}]}]},{"osd":3,"up_from":99,"seq":425201762636,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1547544,"kb_used_data":458576,"kb_used_omap":7,"kb_used_meta":1088952,"kb_avail":92197608,"statfs":{"total":95995035648,"available":94410350592,"internally_reserved":0,"allocated":469581824,"data_stored":467170989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1115087464},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":3.7530000000000001,"5min":5.319,"15min":4.0540000000000003},"min":{"1min":0.189,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":29.254999999999999,"5min":119.97799999999999,"15min":119.97799999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":7.0410000000000004,"5min":6.3849999999999998,"15min":4.7759999999999998},"min":{"1min":0.26600000000000001,"5min":0.223,"15min":0.185},"max":{"1min":29.506,"5min":120.307,"15min":120.307},"last":0.63700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":4.2320000000000002,"5min":3.3580000000000001,"15min":4.4299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":16.427,"5min":121.842,"15min":170.196},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.6779999999999999,"15min":4.1299999999999999},"min":{"1min":0.22900000000000001,"5min":0.224,"15min":0.221},"max":{"1min":9.1519999999999992,"5min":119.673,"15min":166.953},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.103,"15min":2.2959999999999998},"min":{"1min":0.25800000000000001,"5min":0.217,"15min":0.187},"max":{"1min":4.1929999999999996,"5min":5.8300000000000001,"15min":127.599},"last":0.46200000000000002},{"interface":"front","average":{"1min":5.1680000000000001,"5min":2.2930000000000001,"15min":3.452},"min":{"1min":0.34399999999999997,"5min":0.23799999999999999,"15min":0.221},"max":{"1min":17.463000000000001,"5min":17.463000000000001,"15min":129.98599999999999},"last":4.5490000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":4.2830000000000004,"5min":3.8210000000000002,"15min":3.1800000000000002},"min":{"1min":0.38800000000000001,"5min":0.26900000000000002,"15min":0.25700000000000001},"max":{"1min":16.588999999999999,"5min":83.010000000000005,"15min":211.84700000000001},"last":4.5149999999999997},{"interface":"front","average":{"1min":3.7810000000000001,"5min":3.3290000000000002,"15min":3.1579999999999999},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":16.492000000000001,"5min":82.915000000000006,"15min":211.17500000000001},"last":1.5720000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":3.6000000000000001,"5min":2.802,"15min":2.4710000000000001},"min":{"1min":0.33600000000000002,"5min":0.30099999999999999,"15min":0.28100000000000003},"max":{"1min":16.52,"5min":88.295000000000002,"15min":88.295000000000002},"last":1.127},{"interface":"front","average":{"1min":0.81000000000000005,"5min":2.222,"15min":1.825},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.248},"max":{"1min":4.1580000000000004,"5min":86.968999999999994,"15min":86.968999999999994},"last":0.48899999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.75,"15min":1.5860000000000001},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.26500000000000001},"max":{"1min":4.1180000000000003,"5min":64.378,"15min":64.378},"last":0.628},{"interface":"front","average":{"1min":4.0830000000000002,"5min":2.641,"15min":2.1600000000000001},"min":{"1min":0.45000000000000001,"5min":0.36599999999999999,"15min":0.29999999999999999},"max":{"1min":16.632000000000001,"5min":64.353999999999999,"15min":64.353999999999999},"last":4.6479999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:30:26 2023","interfaces":[{"interface":"back","average":{"1min":4.4089999999999998,"5min":4.2149999999999999,"15min":3.4620000000000002},"min":{"1min":0.442,"5min":0.38,"15min":0.34799999999999998},"max":{"1min":16.661000000000001,"5min":86.897000000000006,"15min":212.62700000000001},"last":4.46},{"interface":"front","average":{"1min":4.0430000000000001,"5min":3.6419999999999999,"15min":3.488},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":16.553000000000001,"5min":87.105999999999995,"15min":212.04599999999999},"last":0.57099999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986652,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1157600,"kb_used_data":251672,"kb_used_omap":0,"kb_used_meta":905920,"kb_avail":92587552,"statfs":{"total":95995035648,"available":94809653248,"internally_reserved":0,"allocated":257712128,"data_stored":256159405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927662080},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:30:41 2023","interfaces":[{"interface":"back","average":{"1min":8.0099999999999998,"5min":6.5099999999999998,"15min":6.665},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20799999999999999},"max":{"1min":110.95,"5min":207.45699999999999,"15min":207.45699999999999},"last":2.4409999999999998},{"interface":"front","average":{"1min":7.2160000000000002,"5min":6.4530000000000003,"15min":6.7549999999999999},"min":{"1min":0.23200000000000001,"5min":0.223,"15min":0.223},"max":{"1min":111.057,"5min":207.114,"15min":207.114},"last":5.8200000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:30:28 2023","interfaces":[{"interface":"back","average":{"1min":2.456,"5min":1.8340000000000001,"15min":4.6180000000000003},"min":{"1min":0.35099999999999998,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":8.1839999999999993,"5min":33.433999999999997,"15min":135.25399999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.661,"15min":4.4950000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.251},"max":{"1min":13.685,"5min":32.683,"15min":134.44300000000001},"last":2.5139999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:30:59 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":5.8499999999999996,"15min":4.1379999999999999},"min":{"1min":0.35999999999999999,"5min":0.26500000000000001,"15min":0.23599999999999999},"max":{"1min":7.1020000000000003,"5min":203.61500000000001,"15min":203.61500000000001},"last":4.4219999999999997},{"interface":"front","average":{"1min":2.8700000000000001,"5min":5.8140000000000001,"15min":4.1369999999999996},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9400000000000004,"5min":203.601,"15min":203.601},"last":4.4050000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:30:28 2023","interfaces":[{"interface":"back","average":{"1min":16.184000000000001,"5min":5.3200000000000003,"15min":3.5539999999999998},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":135.97800000000001,"5min":135.97800000000001,"15min":154.13200000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":15.048,"5min":5.218,"15min":3.5},"min":{"1min":0.309,"5min":0.309,"15min":0.29399999999999998},"max":{"1min":135.899,"5min":135.899,"15min":154.077},"last":3.2069999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.2090000000000001,"15min":1.8220000000000001},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.7469999999999999,"5min":16.753,"15min":111.809},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1120000000000001,"15min":1.9890000000000001},"min":{"1min":0.27600000000000002,"5min":0.253,"15min":0.252},"max":{"1min":6.6449999999999996,"5min":17.792000000000002,"15min":112.002},"last":4.7060000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:30:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.0720000000000001,"15min":1.7270000000000001},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.4120000000000008,"5min":10.039999999999999,"15min":76.307000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.296,"5min":1.0449999999999999,"15min":1.5700000000000001},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.1760000000000002,"5min":9.1760000000000002,"15min":78.421999999999997},"last":0.42999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:30:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.3089999999999999,"15min":1.169},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":5.1550000000000002,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.085,"5min":1.486,"15min":1.359},"min":{"1min":0.32300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.6710000000000003,"5min":12.863,"15min":12.863},"last":4.6710000000000003}]}]},{"osd":5,"up_from":0,"seq":167503725016,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":494600192,"data_stored":494600192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":573440000,"data_stored":573440000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":363925504,"data_stored":363921432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":460062720,"data_stored":460062720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":240652288,"data_stored":240648216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":378470400,"data_stored":378470400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":475205632,"data_stored":475201560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":349634560,"data_stored":349634560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:31:08.431 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:31:08.431 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:08.431 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 7, 3, 6, 0, 4, 2] 2023-10-09T04:31:08.431 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T04:31:08.507 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:08 smithi181 ceph-mon[196979]: from='client.36912 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:31:08.510 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:08.522 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:08 smithi163 ceph-mon[213841]: from='client.36912 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:31:08.522 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:08 smithi163 ceph-mon[221067]: from='client.36912 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:31:08.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:31:08.544 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:08.636 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:08.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:08.695 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:08.743 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:08.774 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:08.846 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:08.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:08.953 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:09.025 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:09.034 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:09.146 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:09.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:09.254 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:09.309 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:09.312 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:09 smithi163 ceph-mon[221067]: pgmap v1483: 119 pgs: 119 active+clean; 883 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 785 op/s 2023-10-09T04:31:09.315 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:09 smithi163 ceph-mon[213841]: pgmap v1483: 119 pgs: 119 active+clean; 883 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 785 op/s 2023-10-09T04:31:09.371 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:09.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:09.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:09.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:09 smithi181 ceph-mon[196979]: pgmap v1483: 119 pgs: 119 active+clean; 883 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 785 op/s 2023-10-09T04:31:09.539 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:09.597 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:09.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:09.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:09.752 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:09.827 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:09.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:09.932 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:09.958 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.033 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:10.143 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:10.150 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:31:10.210 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.419 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:10.429 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:10.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:10.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:10.581 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.694 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:10.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:10.802 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:10.888 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.907 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:31:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:31:10] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.43.0" 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69232 10.e 10:75e66242:::benchmark_data_smithi163_523108_object69231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.591183+0000", 2023-10-09T04:31:10.968 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.37308962600002, 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0726746409999999, 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69232 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:10.969 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591183+0000", 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591183+0000", 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:10.970 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591181+0000", 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591240+0000", 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.872e-05 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591257+0000", 2023-10-09T04:31:10.971 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6988999999999999e-05 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.591264+0000", 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.5089999999999999e-06 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.645952+0000", 2023-10-09T04:31:10.972 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0546883129999998 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.646113+0000", 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016113 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.646349+0000", 2023-10-09T04:31:10.973 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000236264 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.650803+0000", 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044541820000000001 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.650932+0000", 2023-10-09T04:31:10.974 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012804900000000001 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.662657+0000", 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011725549 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.662702+0000", 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4826000000000001e-05 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.975 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663858+0000", 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.001155643 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69248 10.e 10:75e713a4:::benchmark_data_smithi163_523108_object69247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.836531+0000", 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.12774175599998, 2023-10-09T04:31:10.976 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8380690340000001, 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69248 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836531+0000", 2023-10-09T04:31:10.977 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836531+0000", 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836529+0000", 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:10.978 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836574+0000", 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4994999999999999e-05 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836593+0000", 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.9465000000000001e-05 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.979 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.836602+0000", 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.3480000000000004e-06 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.648625+0000", 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.812023709 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.980 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.648776+0000", 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015048399999999999 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.648960+0000", 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000184227 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656320+0000", 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007359516 2023-10-09T04:31:10.981 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656337+0000", 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.6969999999999998e-05 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.672446+0000", 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016109893 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.982 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.672524+0000", 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7594000000000003e-05 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674600+0000", 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020758339999999999 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.983 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69252 10.e 10:7c6b28f5:::benchmark_data_smithi163_523108_object69251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.841254+0000", 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.12301902600001, 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8353127420000002, 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69252 2023-10-09T04:31:10.984 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841254+0000", 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841254+0000", 2023-10-09T04:31:10.985 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841253+0000", 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841321+0000", 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7558000000000006e-05 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841330+0000", 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.7389999999999994e-06 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.841342+0000", 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1891e-05 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.987 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.650979+0000", 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.8096368279999999 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.651091+0000", 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000111653 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:10.988 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.651205+0000", 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000114021 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656348+0000", 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0051431910000000001 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656408+0000", 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0182000000000001e-05 2023-10-09T04:31:10.989 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674675+0000", 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018266871 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674726+0000", 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0769999999999997e-05 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.990 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.676566+0000", 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00184059 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69257 10.e 10:79c28b6b:::benchmark_data_smithi163_523108_object69256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.872079+0000", 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.09219391699997, 2023-10-09T04:31:10.991 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.809892643, 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69257 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872079+0000", 2023-10-09T04:31:10.992 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872079+0000", 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872077+0000", 2023-10-09T04:31:10.993 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872133+0000", 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5439000000000002e-05 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872148+0000", 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5488000000000001e-05 2023-10-09T04:31:10.994 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.872155+0000", 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.8549999999999996e-06 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.652867+0000", 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7807122660000001 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.995 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.652971+0000", 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010373400000000001 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.653086+0000", 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000115046 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656440+0000", 2023-10-09T04:31:10.996 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0033538729999999998 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656448+0000", 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.8509999999999999e-06 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679138+0000", 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.022690393999999999 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.997 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679211+0000", 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2861999999999995e-05 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.681971+0000", 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027603689999999999 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.998 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69264 10.e 10:74615863:::benchmark_data_smithi163_523108_object69263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:28.906946+0000", 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 462.05732698399999, 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7769406810000001, 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69264 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:10.999 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.906946+0000", 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.906946+0000", 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.000 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.906945+0000", 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.907005+0000", 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9942000000000003e-05 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.001 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.907014+0000", 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6739999999999992e-06 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:28.907018+0000", 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9060000000000004e-06 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.002 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.654647+0000", 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7476289889999999 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.654799+0000", 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015171999999999999 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.654916+0000", 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000117608 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.003 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660310+0000", 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0053939180000000001 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660361+0000", 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0278999999999998e-05 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.682017+0000", 2023-10-09T04:31:11.004 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.021656412999999999 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.682060+0000", 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3090000000000002e-05 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.683886+0000", 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018261359999999999 2023-10-09T04:31:11.005 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69282 10.e 10:7bf871c6:::benchmark_data_smithi163_523108_object69281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.015311+0000", 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.94896137900002, 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6712104050000001, 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.006 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69282 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015311+0000", 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.007 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015311+0000", 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015310+0000", 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015365+0000", 2023-10-09T04:31:11.008 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5050999999999999e-05 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015379+0000", 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4032000000000001e-05 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.015385+0000", 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0580000000000003e-06 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.009 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656478+0000", 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6410930989999999 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656596+0000", 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118272 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.656740+0000", 2023-10-09T04:31:11.010 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000144048 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660410+0000", 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0036698579999999998 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.660419+0000", 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.1630000000000002e-06 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.011 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.684011+0000", 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023592014000000001 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.684054+0000", 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3112000000000003e-05 2023-10-09T04:31:11.012 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686522+0000", 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00246714 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69285 10.e 10:7ce41a17:::benchmark_data_smithi163_523108_object69284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.071025+0000", 2023-10-09T04:31:11.013 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.89324712799998, 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6243140670000003, 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69285 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071025+0000", 2023-10-09T04:31:11.014 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071025+0000", 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071024+0000", 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.015 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071048+0000", 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3989e-05 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071062+0000", 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.4272e-05 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.016 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.071068+0000", 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7080000000000002e-06 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.658261+0000", 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.587193251 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.017 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.658420+0000", 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000158627 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.658622+0000", 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000202456 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663871+0000", 2023-10-09T04:31:11.018 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0052484669999999997 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663911+0000", 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0809000000000002e-05 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.693365+0000", 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029453842000000001 2023-10-09T04:31:11.019 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.693406+0000", 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.0679999999999997e-05 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.695339+0000", 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0019334759999999999 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.020 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69287 10.e 10:7fcc9332:::benchmark_data_smithi163_523108_object69286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.083976+0000", 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.88029657499999, 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.6132521139999998, 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.021 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69287 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083976+0000", 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083976+0000", 2023-10-09T04:31:11.022 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083975+0000", 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.083998+0000", 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.3048e-05 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.023 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.084012+0000", 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3766e-05 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.084017+0000", 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3340000000000001e-06 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.024 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.663942+0000", 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.579925121 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.664062+0000", 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011936500000000001 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.664171+0000", 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000109647 2023-10-09T04:31:11.025 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674622+0000", 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010451026 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679015+0000", 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0043928140000000001 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.026 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.695384+0000", 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.016368627 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.695421+0000", 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.7499999999999997e-05 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697228+0000", 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0018069659999999999 2023-10-09T04:31:11.027 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69303 10.e 10:75ecb351:::benchmark_data_smithi163_523108_object69302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.130126+0000", 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.83414645800002, 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5815739410000003, 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.028 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69303 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130126+0000", 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.029 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130126+0000", 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130125+0000", 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130177+0000", 2023-10-09T04:31:11.030 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1683e-05 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130189+0000", 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2656e-05 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.130195+0000", 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.6319999999999996e-06 2023-10-09T04:31:11.031 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.666273+0000", 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.536077922 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.666380+0000", 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106826 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.666512+0000", 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000131939 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.674633+0000", 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0081214129999999992 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.679074+0000", 2023-10-09T04:31:11.033 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0044414240000000002 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.709593+0000", 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.030518841000000001 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.709640+0000", 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7169000000000002e-05 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.034 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.711700+0000", 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0020595779999999998 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69310 10.e 10:7b904be6:::benchmark_data_smithi163_523108_object69309:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.185883+0000", 2023-10-09T04:31:11.035 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.77838986299997, 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.5277444510000002, 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69310 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.037 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185883+0000", 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185883+0000", 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.038 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185881+0000", 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185908+0000", 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7138999999999999e-05 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.039 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185924+0000", 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5197999999999999e-05 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.185932+0000", 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9859999999999994e-06 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.668476+0000", 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.482544866 2023-10-09T04:31:11.040 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.668602+0000", 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000125854 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.668778+0000", 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000175584 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.041 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.678946+0000", 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010168513000000001 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.683948+0000", 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0050016749999999997 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.711780+0000", 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.027831794 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.711844+0000", 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4277999999999997e-05 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.713627+0000", 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0017830280000000001 2023-10-09T04:31:11.043 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69329 10.e 10:703d185a:::benchmark_data_smithi163_523108_object69328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.291333+0000", 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.67293910400002, 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.4245665820000002, 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.044 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69329 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291333+0000", 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.045 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291333+0000", 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291332+0000", 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.046 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291376+0000", 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.392e-05 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291387+0000", 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0954999999999999e-05 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.047 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.291392+0000", 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0150000000000003e-06 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.670409+0000", 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3790162119999998 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.048 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.670574+0000", 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016513000000000001 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.670688+0000", 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011466700000000001 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.678958+0000", 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0082692180000000001 2023-10-09T04:31:11.049 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.683974+0000", 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.005016408 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.713704+0000", 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.029730143000000001 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.050 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.713777+0000", 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.2943999999999998e-05 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.715900+0000", 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021229460000000001 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.051 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69426 10.e 10:7098e19e:::benchmark_data_smithi163_523108_object69425:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:29.928499+0000", 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 461.03577399599999, 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7898835379999998, 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.052 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69426 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928499+0000", 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.053 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928499+0000", 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928497+0000", 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928551+0000", 2023-10-09T04:31:11.054 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.4336000000000003e-05 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928567+0000", 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5727999999999999e-05 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:29.928574+0000", 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9240000000000001e-06 2023-10-09T04:31:11.055 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686587+0000", 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7580135079999999 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686760+0000", 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000172788 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.056 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.686949+0000", 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000188333 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697250+0000", 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010301273 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.057 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697305+0000", 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5287000000000001e-05 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.715978+0000", 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.018672993999999998 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.716045+0000", 2023-10-09T04:31:11.058 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.7008999999999995e-05 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.718382+0000", 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023370040000000002 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:69489 10.e 10:7a3246dd:::benchmark_data_smithi163_523108_object69488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T04:31:11.059 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:23:30.498807+0000", 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 460.4654658, 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.224276691, 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 69489 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.060 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498807+0000", 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498807+0000", 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498805+0000", 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.061 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498863+0000", 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7686000000000003e-05 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498876+0000", 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3013999999999999e-05 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.062 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:30.498882+0000", 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8100000000000003e-06 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.689318+0000", 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.190435871 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.063 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.689466+0000", 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000148209 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.689604+0000", 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000138465 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697261+0000", 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.007656688 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.064 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.697367+0000", 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010572400000000001 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.720882+0000", 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.023515794999999999 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.065 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.720954+0000", 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1761999999999995e-05 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:23:32.723083+0000", 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021292540000000001 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.066 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116398 10.f 10:f6de70b0:::benchmark_data_smithi163_523108_object42382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:08.726639+0000", 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 302.23763358899998, 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.1461028689999999, 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116398 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.067 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726639+0000", 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726639+0000", 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.068 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726638+0000", 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726643+0000", 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8149999999999996e-06 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.069 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726656+0000", 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3525e-05 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726661+0000", 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0749999999999997e-06 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726708+0000", 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.6900000000000002e-05 2023-10-09T04:31:11.070 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:08.726715+0000", 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.4400000000000002e-06 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.739515+0000", 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0128005419999999 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.071 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.739527+0000", 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1764e-05 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.840092+0000", 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.100565035 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.840229+0000", 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000136718 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.072 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.840762+0000", 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00053350200000000004 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865462+0000", 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.024699523000000001 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865510+0000", 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7876e-05 2023-10-09T04:31:11.073 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865616+0000", 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000106165 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872563+0000", 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0069471330000000003 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.074 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872623+0000", 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0189000000000003e-05 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872742+0000", 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000118701 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.075 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116433 10.f 10:f66d42fa:::benchmark_data_smithi163_523108_object42417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.800654+0000", 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 301.16361826500003, 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.082506435, 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116433 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.076 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800654+0000", 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800654+0000", 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.077 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800653+0000", 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800659+0000", 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0159999999999997e-06 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800672+0000", 2023-10-09T04:31:11.078 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3023000000000001e-05 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800678+0000", 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9689999999999999e-06 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800729+0000", 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.1001999999999997e-05 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.079 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.800738+0000", 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.816e-06 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.847848+0000", 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0471098089999999 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.848001+0000", 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000152948 2023-10-09T04:31:11.080 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.848139+0000", 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013824200000000001 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865546+0000", 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.017406883000000001 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.081 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865579+0000", 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.3408999999999998e-05 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872754+0000", 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0071743529999999996 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.882996+0000", 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010242816 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.082 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883047+0000", 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0534000000000001e-05 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883161+0000", 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000113752 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.083 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116446 10.f 10:fd542c17:::benchmark_data_smithi163_523108_object42430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.830509+0000", 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 301.13376352799997, 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.053200197, 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.084 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116446 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830509+0000", 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830509+0000", 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.085 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830507+0000", 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830515+0000", 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.9189999999999999e-06 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830529+0000", 2023-10-09T04:31:11.086 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3729e-05 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.830536+0000", 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.3479999999999998e-06 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.832864+0000", 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0023279910000000002 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.087 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.832876+0000", 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.1646e-05 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.851302+0000", 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.018425677 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.088 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.851473+0000", 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00017120500000000001 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.851653+0000", 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000179727 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865654+0000", 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.014001632 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.089 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.865698+0000", 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.3856999999999997e-05 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872760+0000", 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0070616699999999999 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.090 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883560+0000", 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010800362000000001 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883605+0000", 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.4913999999999997e-05 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883709+0000", 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010422 2023-10-09T04:31:11.091 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116453 10.f 10:fc5b6f4e:::benchmark_data_smithi163_523108_object42437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:09.840683+0000", 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 301.12358960699999, 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0432569810000001, 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.092 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116453 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840683+0000", 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840683+0000", 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.093 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840682+0000", 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840688+0000", 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.7629999999999997e-06 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.094 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840700+0000", 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2116999999999999e-05 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840705+0000", 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9799999999999998e-06 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840750+0000", 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5630000000000002e-05 2023-10-09T04:31:11.095 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:09.840759+0000", 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.5629999999999992e-06 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.854741+0000", 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0139816880000001 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.854875+0000", 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00013423499999999999 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.855026+0000", 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015111899999999999 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867814+0000", 2023-10-09T04:31:11.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.012787587 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867914+0000", 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.9958999999999996e-05 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.098 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872766+0000", 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048521149999999997 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883765+0000", 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.010999581 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883820+0000", 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5129e-05 2023-10-09T04:31:11.099 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883940+0000", 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119577 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116493 10.f 10:fed8895e:::benchmark_data_smithi163_523108_object42477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:10.132543+0000", 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 300.83172981899997, 2023-10-09T04:31:11.100 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.75157952400000005, 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116493 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132543+0000", 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.101 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132543+0000", 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132541+0000", 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.102 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132547+0000", 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.9499999999999998e-06 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132559+0000", 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2272e-05 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132565+0000", 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3859999999999998e-06 2023-10-09T04:31:11.103 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132615+0000", 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0396999999999999e-05 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.132622+0000", 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0040000000000002e-06 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.858203+0000", 2023-10-09T04:31:11.104 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.72558060099999999 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.858358+0000", 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00015483900000000001 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.858531+0000", 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000173752 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.105 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867848+0000", 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0093166810000000003 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867945+0000", 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6560999999999999e-05 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872772+0000", 2023-10-09T04:31:11.106 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0048271959999999997 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.883980+0000", 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011207943999999999 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884025+0000", 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.5442000000000003e-05 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.107 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884122+0000", 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.6967999999999997e-05 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:116495 10.f 10:f09f63e6:::benchmark_data_smithi163_523108_object42479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:10.138865+0000", 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 300.825407918, 2023-10-09T04:31:11.108 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.74547842600000003, 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 116495 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138865+0000", 2023-10-09T04:31:11.109 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138865+0000", 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138864+0000", 2023-10-09T04:31:11.110 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138869+0000", 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.8509999999999998e-06 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138878+0000", 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.4409999999999999e-06 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.111 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138883+0000", 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.135e-06 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138931+0000", 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.7457000000000003e-05 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.138938+0000", 2023-10-09T04:31:11.112 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.622e-06 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.861740+0000", 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.72280164000000002 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.861906+0000", 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00016563600000000001 2023-10-09T04:31:11.113 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.862090+0000", 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00018444199999999999 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.867881+0000", 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0057914000000000004 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.114 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872553+0000", 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0046718059999999997 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.872778+0000", 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000224937 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884175+0000", 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.011396509000000001 2023-10-09T04:31:11.115 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884224+0000", 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.9230000000000001e-05 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:10.884343+0000", 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000119158 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.116 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46073.0:120724 10.f 10:fc37408d:::benchmark_data_smithi163_523108_object46708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:26:20.337568+0000", 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 290.62670450899998, 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.067171087, 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "reached pg", 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46073", 2023-10-09T04:31:11.117 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/884821460", 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 120724 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337568+0000", 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337568+0000", 2023-10-09T04:31:11.118 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337567+0000", 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337574+0000", 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1060000000000002e-06 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.119 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337589+0000", 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.5855999999999999e-05 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.337597+0000", 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.1369999999999996e-06 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.120 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.338389+0000", 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00079237399999999999 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.338572+0000", 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000182972 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 2,3,4", 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.338730+0000", 2023-10-09T04:31:11.121 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000158066 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.341931+0000", 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0032007619999999998 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:11.122 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.342216+0000", 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000285586 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.355426+0000", 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.013209449999999999 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:20.355436+0000", 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.0579e-05 2023-10-09T04:31:11.123 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.404678+0000", 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.049242062 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:26:21.404739+0000", 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.0647999999999999e-05 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.124 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:11.125 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:11.125 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:31:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:11.126 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:11.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:11.227 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:11.263 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:11.274 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:11 smithi163 ceph-mon[213841]: pgmap v1484: 119 pgs: 119 active+clean; 869 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T04:31:11.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:11 smithi163 ceph-mon[221067]: pgmap v1484: 119 pgs: 119 active+clean; 869 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T04:31:11.316 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:11.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:11.419 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:11.477 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:11.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:11 smithi181 ceph-mon[196979]: pgmap v1484: 119 pgs: 119 active+clean; 869 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T04:31:11.579 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:11.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:11.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:11.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:11.708 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:11.751 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:11.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:11.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:11.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:31:11.854 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:11.931 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.163 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:12.186 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:12.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:12.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:12.310 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:31:12.316 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:12.332 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.408 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:12.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:12.552 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:12 smithi181 ceph-mon[196979]: 1.12 scrub starts 2023-10-09T04:31:12.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:12 smithi181 ceph-mon[196979]: 1.12 scrub ok 2023-10-09T04:31:12.553 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:12 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1875491964' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:12.555 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:12 smithi163 ceph-mon[213841]: 1.12 scrub starts 2023-10-09T04:31:12.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:12 smithi163 ceph-mon[213841]: 1.12 scrub ok 2023-10-09T04:31:12.580 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:12 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1875491964' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:12.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:12 smithi163 ceph-mon[221067]: 1.12 scrub starts 2023-10-09T04:31:12.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:12 smithi163 ceph-mon[221067]: 1.12 scrub ok 2023-10-09T04:31:12.585 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:12 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1875491964' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:12.615 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:12.720 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:12.764 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:12.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:13.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:13.171 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:13.479 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: pgmap v1485: 119 pgs: 119 active+clean; 732 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: failed to encode map e562 with expected crc 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1875491964' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[213841]: failed to encode map e562 with expected crc 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: pgmap v1485: 119 pgs: 119 active+clean; 732 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:13.480 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:31:13.481 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: failed to encode map e562 with expected crc 2023-10-09T04:31:13.481 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1875491964' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:13.481 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T04:31:13.481 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:13 smithi163 ceph-mon[221067]: failed to encode map e562 with expected crc 2023-10-09T04:31:13.504 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:13.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:13.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:13.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: pgmap v1485: 119 pgs: 119 active+clean; 732 MiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 752 op/s 2023-10-09T04:31:13.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:13.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:31:13.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: failed to encode map e562 with expected crc 2023-10-09T04:31:13.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1875491964' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:13.578 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T04:31:13.579 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:13 smithi181 ceph-mon[196979]: failed to encode map e562 with expected crc 2023-10-09T04:31:13.614 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:13.688 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:13.916 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:13.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:14.025 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:14.051 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:14.094 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:14.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:14.139 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2023-10-09T04:31:14.139 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ 2023-10-09T04:31:14.139 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.139 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.46577.0:201150 11.f 11:fcf33e54:::benchmark_data_smithi163_601242_object96371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:31:13.973724+0000", 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.16501088, 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.16503984699999999, 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.46577", 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 201150 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:14.140 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973724+0000", 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973724+0000", 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:14.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973723+0000", 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973728+0000", 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.0350000000000001e-06 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973742+0000", 2023-10-09T04:31:14.142 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.3154e-05 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973748+0000", 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.1e-06 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973803+0000", 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.5605e-05 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.143 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.973929+0000", 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012565100000000001 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.974044+0000", 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00011543400000000001 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op_commit", 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.976766+0000", 2023-10-09T04:31:14.144 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0027216330000000002 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:13.978875+0000", 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0021088140000000001 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138705+0000", 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.15982977700000001 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.145 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "commit_sent", 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138757+0000", 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.2275999999999999e-05 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_repop_reply(client.46577.0:201150 11.f e562/507 ondisk, result = 0)", 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-10-09T04:31:14.138622+0000", 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 0.000113021, 2023-10-09T04:31:14.146 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000225765, 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "started", 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138622+0000", 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138622+0000", 2023-10-09T04:31:14.147 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138621+0000", 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138625+0000", 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.9790000000000004e-06 2023-10-09T04:31:14.148 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138634+0000", 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.619e-06 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138638+0000", 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4.481e-06 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.149 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138692+0000", 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.3539999999999999e-05 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138699+0000", 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 6.9310000000000002e-06 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: { 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-10-09T04:31:14.138848+0000", 2023-10-09T04:31:14.150 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00014878600000000001 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout: } 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout: ], 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout: "num_ops": 2 2023-10-09T04:31:14.151 INFO:teuthology.orchestra.run.smithi181.stdout:} 2023-10-09T04:31:14.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:14.241 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:14.331 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.6. 2023-10-09T04:31:14.363 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:14.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:14.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:14.466 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:14.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2531972463' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:31:14.492 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:14 smithi163 ceph-mon[221067]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:31:14.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:14 smithi163 conmon[213817]: 2023-10-09T04:31:14.219+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e562 definitely_dead 0 2023-10-09T04:31:14.492 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2531972463' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:31:14.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:14 smithi163 ceph-mon[213841]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:31:14.494 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2531972463' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:31:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:14 smithi181 ceph-mon[196979]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:31:14.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:14.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:14.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:14.693 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:14.721 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:14.765 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:14.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:14.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:14.868 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:14.895 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:15.049 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:31:15.104 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:15.208 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:15.235 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:15.284 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:15.322 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:15 smithi181 ceph-mon[196979]: pgmap v1487: 119 pgs: 119 active+clean; 663 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:31:15.323 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:15 smithi181 ceph-mon[196979]: failed to encode map e563 with expected crc 2023-10-09T04:31:15.323 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:15 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2531972463' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:31:15.323 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:15 smithi181 ceph-mon[196979]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T04:31:15.323 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:15 smithi181 ceph-mon[196979]: failed to encode map e563 with expected crc 2023-10-09T04:31:15.388 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:15.419 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:15.461 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:15.466 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:15.530 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:31:15.545 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:31:15.549 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:31:15.567 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:31:15.571 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:15.572 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[221067]: pgmap v1487: 119 pgs: 119 active+clean; 663 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:31:15.572 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[221067]: failed to encode map e563 with expected crc 2023-10-09T04:31:15.572 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2531972463' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:31:15.572 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[221067]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T04:31:15.573 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[221067]: failed to encode map e563 with expected crc 2023-10-09T04:31:15.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[213841]: pgmap v1487: 119 pgs: 119 active+clean; 663 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T04:31:15.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[213841]: failed to encode map e563 with expected crc 2023-10-09T04:31:15.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2531972463' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:31:15.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[213841]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T04:31:15.573 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:15 smithi163 ceph-mon[213841]: failed to encode map e563 with expected crc 2023-10-09T04:31:15.587 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:31:15.599 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:15.606 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:31:15.629 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:31:15.648 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:15.650 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:31:15.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:15.668 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.670 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:15.753 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:15.780 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:15.924 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:16.028 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:16.054 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.147 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:16.251 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:16.276 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.340 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:16.447 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:16.529 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T04:31:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2540301460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[221067]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2540301460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:16.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:16 smithi163 ceph-mon[213841]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T04:31:16.591 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:16.619 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:16.686 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.686 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T04:31:16.686 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.686 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.686 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:16.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2540301460' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:16.687 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:16 smithi181 ceph-mon[196979]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T04:31:16.724 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:16.793 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.929 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:17.033 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:17.120 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:17.265 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:17.265 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:17.352 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:17.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:17.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:17.457 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:17.488 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:17.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.585 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: pgmap v1490: 119 pgs: 13 remapped+peering, 21 peering, 85 active+clean; 545 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 861 op/s 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12620 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.8 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12336 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.3 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12723 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.4 starting backfill to osd.1 from (0'0,0'0] MAX to 562'12385 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.5 starting backfill to osd.0 from (0'0,0'0] MAX to 562'12571 2023-10-09T04:31:17.586 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.a starting backfill to osd.2 from (0'0,0'0] MAX to 562'12468 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 562'12619 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 562'12515 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[213841]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: pgmap v1490: 119 pgs: 13 remapped+peering, 21 peering, 85 active+clean; 545 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 861 op/s 2023-10-09T04:31:17.587 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12620 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.8 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12336 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.3 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12723 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.4 starting backfill to osd.1 from (0'0,0'0] MAX to 562'12385 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.5 starting backfill to osd.0 from (0'0,0'0] MAX to 562'12571 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.a starting backfill to osd.2 from (0'0,0'0] MAX to 562'12468 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 562'12619 2023-10-09T04:31:17.588 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 562'12515 2023-10-09T04:31:17.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:31:17.589 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:17 smithi163 ceph-mon[221067]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e564 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: pgmap v1490: 119 pgs: 13 remapped+peering, 21 peering, 85 active+clean; 545 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 861 op/s 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.757 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: failed to encode map e565 with expected crc 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12620 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.8 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12336 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.3 starting backfill to osd.2 from (0'0,0'0] MAX to 562'12723 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.4 starting backfill to osd.1 from (0'0,0'0] MAX to 562'12385 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.5 starting backfill to osd.0 from (0'0,0'0] MAX to 562'12571 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.a starting backfill to osd.2 from (0'0,0'0] MAX to 562'12468 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 562'12619 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 562'12515 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:31:17.758 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:17 smithi181 ceph-mon[196979]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T04:31:17.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:17.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:17.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:17.970 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:18.022 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:18.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:18.126 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:18.154 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:18.271 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:18.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:18.340 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:18.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:18.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[221067]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[221067]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[221067]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[221067]: pgmap v1493: 119 pgs: 2 active+remapped+backfill_wait, 3 active+undersized+remapped, 13 remapped+peering, 21 peering, 80 active+clean; 505 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 984/24276 objects misplaced (4.053%); 0 B/s, 1 objects/s recovering 2023-10-09T04:31:18.405 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[221067]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T04:31:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[213841]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[213841]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[213841]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[213841]: pgmap v1493: 119 pgs: 2 active+remapped+backfill_wait, 3 active+undersized+remapped, 13 remapped+peering, 21 peering, 80 active+clean; 505 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 984/24276 objects misplaced (4.053%); 0 B/s, 1 objects/s recovering 2023-10-09T04:31:18.406 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:18 smithi163 ceph-mon[213841]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T04:31:18.425 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:18.569 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:18.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:18.593 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:31:18.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:18 smithi181 ceph-mon[196979]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:18 smithi181 ceph-mon[196979]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:18 smithi181 ceph-mon[196979]: failed to encode map e566 with expected crc 2023-10-09T04:31:18.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:18 smithi181 ceph-mon[196979]: pgmap v1493: 119 pgs: 2 active+remapped+backfill_wait, 3 active+undersized+remapped, 13 remapped+peering, 21 peering, 80 active+clean; 505 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 984/24276 objects misplaced (4.053%); 0 B/s, 1 objects/s recovering 2023-10-09T04:31:18.671 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:18 smithi181 ceph-mon[196979]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T04:31:18.675 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:18.747 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:18.851 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:18.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:18.956 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:19.064 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:19.096 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:19.187 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:19.204 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:19.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:19.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:19.352 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:19.420 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:19.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:19.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:19.558 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:19.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:19.575 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:19.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1275460564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:19.576 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:19 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:19.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:19.605 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:19.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1275460564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:19.606 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: failed to encode map e567 with expected crc 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1275460564' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:19.609 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:19 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:19.783 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:19.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:19.889 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:19.925 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:19.988 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:19.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:20.092 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:20.118 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:20.160 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:20.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:20.166 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 3, 0, 4, 2] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:31:20.166 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:20.166 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:31:20.166 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-09T04:31:20.167 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-09T04:31:20.263 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:20.294 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:20.342 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:31:20.542 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:20.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:20 smithi181 ceph-mon[196979]: pgmap v1495: 119 pgs: 2 active+remapped+backfill_wait, 3 active+undersized+remapped, 13 remapped+peering, 20 peering, 81 active+clean; 498 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 344 op/s; 984/23958 objects misplaced (4.107%); 0 B/s, 1 objects/s recovering 2023-10-09T04:31:20.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:20 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:20.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:20 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:20.544 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:20 smithi181 ceph-mon[196979]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T04:31:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:20.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:20.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[213841]: pgmap v1495: 119 pgs: 2 active+remapped+backfill_wait, 3 active+undersized+remapped, 13 remapped+peering, 20 peering, 81 active+clean; 498 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 344 op/s; 984/23958 objects misplaced (4.107%); 0 B/s, 1 objects/s recovering 2023-10-09T04:31:20.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:20.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:20.638 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[213841]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T04:31:20.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[221067]: pgmap v1495: 119 pgs: 2 active+remapped+backfill_wait, 3 active+undersized+remapped, 13 remapped+peering, 20 peering, 81 active+clean; 498 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 344 op/s; 984/23958 objects misplaced (4.107%); 0 B/s, 1 objects/s recovering 2023-10-09T04:31:20.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:20.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:20.638 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:20 smithi163 ceph-mon[221067]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T04:31:20.646 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:20.674 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:20.732 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:20.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:20.841 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:20.869 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:20.965 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:20.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:21.069 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.081 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:31:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:31:20] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.43.0" 2023-10-09T04:31:21.100 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.205 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:21.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:21.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:21.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:21.323 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:21.323 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:21.343 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.502 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:21.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:21.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:21.632 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:21 smithi181 ceph-mon[196979]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.632 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:21 smithi181 ceph-mon[196979]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.632 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:21 smithi181 ceph-mon[196979]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.632 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:21 smithi181 ceph-mon[196979]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.633 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:21 smithi181 ceph-mon[196979]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T04:31:21.635 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.678 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:21.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:21.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:21.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:21.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[213841]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.807 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[213841]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[213841]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[213841]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[213841]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[221067]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[221067]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[221067]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[221067]: failed to encode map e568 with expected crc 2023-10-09T04:31:21.808 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:21 smithi163 ceph-mon[221067]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T04:31:21.811 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:22.099 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:22.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:22.205 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:22.218 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:22.239 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:22.284 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:31:22.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:22.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:22.406 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:22.434 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:22.485 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:22.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:22.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: pgmap v1498: 119 pgs: 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 4 active+remapped+backfill_wait, 2 active+undersized+remapped, 109 active+clean; 407 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 1813/19587 objects misplaced (9.256%); 18 MiB/s, 295 objects/s recovering 2023-10-09T04:31:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:22 smithi181 ceph-mon[196979]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T04:31:22.591 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:22.617 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:22.626 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.626 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.626 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.626 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: pgmap v1498: 119 pgs: 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 4 active+remapped+backfill_wait, 2 active+undersized+remapped, 109 active+clean; 407 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 1813/19587 objects misplaced (9.256%); 18 MiB/s, 295 objects/s recovering 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[221067]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.627 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: failed to encode map e569 with expected crc 2023-10-09T04:31:22.628 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: pgmap v1498: 119 pgs: 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 4 active+remapped+backfill_wait, 2 active+undersized+remapped, 109 active+clean; 407 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 1813/19587 objects misplaced (9.256%); 18 MiB/s, 295 objects/s recovering 2023-10-09T04:31:22.628 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:22 smithi163 ceph-mon[213841]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T04:31:22.637 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:31:22.659 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:22.661 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:31:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:22.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:22.694 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:31:22.717 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:31:22.737 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:31:22.755 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:31:22.765 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:22.777 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:31:22.799 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:22.808 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.813 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:31:22.923 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:22.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:23.028 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:23.114 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:31:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4159402083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: failed to encode map e571 with expected crc 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4159402083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: failed to encode map e571 with expected crc 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:23.584 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:23 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:23.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: failed to encode map e570 with expected crc 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4159402083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: failed to encode map e571 with expected crc 2023-10-09T04:31:23.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:23.770 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:23 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:23.960 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:23.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:24.068 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:24.098 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:24.109 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:31:24.142 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:24.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:24.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:24.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:24.279 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:24.377 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.377 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.378 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.378 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:24.378 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:24.378 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T04:31:24.378 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: failed to encode map e572 with expected crc 2023-10-09T04:31:24.378 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:24 smithi181 ceph-mon[196979]: pgmap v1502: 119 pgs: 2 active+remapped+backfilling, 2 peering, 2 active+clean+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 367 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 401 op/s; 829/17664 objects misplaced (4.693%); 29 MiB/s, 457 objects/s recovering 2023-10-09T04:31:24.435 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:24.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:24.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:24.629 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: failed to encode map e572 with expected crc 2023-10-09T04:31:24.641 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[221067]: pgmap v1502: 119 pgs: 2 active+remapped+backfilling, 2 peering, 2 active+clean+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 367 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 401 op/s; 829/17664 objects misplaced (4.693%); 29 MiB/s, 457 objects/s recovering 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: failed to encode map e571 with expected crc 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T04:31:24.644 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: failed to encode map e572 with expected crc 2023-10-09T04:31:24.645 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:24 smithi163 ceph-mon[213841]: pgmap v1502: 119 pgs: 2 active+remapped+backfilling, 2 peering, 2 active+clean+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 367 MiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 401 op/s; 829/17664 objects misplaced (4.693%); 29 MiB/s, 457 objects/s recovering 2023-10-09T04:31:24.755 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:24.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:24.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:24.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:24.925 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:25.081 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:25.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:25.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:25.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:25.246 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:25.260 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:25.353 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:25.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:25.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:25.460 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:25.491 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:25.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:25.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:25.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:25.700 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:25.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:25 smithi163 ceph-mon[213841]: failed to encode map e572 with expected crc 2023-10-09T04:31:25.710 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:25 smithi163 ceph-mon[213841]: failed to encode map e572 with expected crc 2023-10-09T04:31:25.711 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:25 smithi163 ceph-mon[221067]: failed to encode map e572 with expected crc 2023-10-09T04:31:25.711 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:25 smithi163 ceph-mon[221067]: failed to encode map e572 with expected crc 2023-10-09T04:31:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:25 smithi181 ceph-mon[196979]: failed to encode map e572 with expected crc 2023-10-09T04:31:25.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:25 smithi181 ceph-mon[196979]: failed to encode map e572 with expected crc 2023-10-09T04:31:25.822 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:25.949 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:25.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:25.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:26.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:26.096 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:26.146 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:26.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:26.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:26.277 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:26.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:26.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:26.453 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-09T04:31:26.490 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:26.567 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:26.691 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:31:26.749 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:26 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4064972073' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:31:26.749 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:26 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:31:26.749 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:26 smithi163 ceph-mon[213841]: pgmap v1503: 119 pgs: 1 peering, 1 active+clean+remapped, 117 active+clean; 295 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 20 MiB/s, 322 objects/s recovering 2023-10-09T04:31:26.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:26 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4064972073' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:31:26.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:26 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:31:26.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:26 smithi163 ceph-mon[221067]: pgmap v1503: 119 pgs: 1 peering, 1 active+clean+remapped, 117 active+clean; 295 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 20 MiB/s, 322 objects/s recovering 2023-10-09T04:31:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:26 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4064972073' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:31:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:26 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-09T04:31:26.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:26 smithi181 ceph-mon[196979]: pgmap v1503: 119 pgs: 1 peering, 1 active+clean+remapped, 117 active+clean; 295 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 20 MiB/s, 322 objects/s recovering 2023-10-09T04:31:27.207 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:27.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:27.317 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:27.349 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:27.393 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:27.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:27.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:27.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: failed to encode map e573 with expected crc 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: failed to encode map e573 with expected crc 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.523 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: failed to encode map e573 with expected crc 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: failed to encode map e573 with expected crc 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.524 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:27 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.526 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:27.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: failed to encode map e573 with expected crc 2023-10-09T04:31:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: failed to encode map e573 with expected crc 2023-10-09T04:31:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-09T04:31:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T04:31:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:27 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:27.973 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:27.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:27.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:28.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:28.113 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:28.270 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:28.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:28.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:28.377 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:28.489 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:28.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:28 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:28.738 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:28 smithi163 ceph-mon[221067]: pgmap v1505: 119 pgs: 119 active+clean; 236 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 360 op/s; 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:31:28.738 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:28 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:28.739 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:28 smithi163 ceph-mon[213841]: pgmap v1505: 119 pgs: 119 active+clean; 236 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 360 op/s; 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:31:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:28 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:28.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:28 smithi181 ceph-mon[196979]: pgmap v1505: 119 pgs: 119 active+clean; 236 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 360 op/s; 8.8 MiB/s, 141 objects/s recovering 2023-10-09T04:31:28.811 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:28.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:28.926 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:28.943 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:28.953 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:28.996 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:28.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.074 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:31:29.074 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [ 2023-10-09T04:31:29.074 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.074 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:207957 11.b 11:d021b417:::benchmark_data_smithi163_601242_object103178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-10-09T04:31:29.074 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:31:29.068421+0000", 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0047420209999999999, 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0047715509999999997, 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207957 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.075 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.068421+0000", 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.068421+0000", 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.068419+0000", 2023-10-09T04:31:29.076 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999976 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.068430+0000", 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.1759e-05 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.068455+0000", 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.4372e-05 2023-10-09T04:31:29.077 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.068467+0000", 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.2068e-05 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.069601+0000", 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0011341229999999999 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.078 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.069732+0000", 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00013134900000000001 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.069858+0000", 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012598899999999999 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "op_commit", 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072298+0000", 2023-10-09T04:31:29.079 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.0024397889999999999 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:207960 11.5 e573/569 11:a5880eb3:::benchmark_data_smithi163_601242_object103181:head v 573'13004, mlcod=573'13002)", 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:31:29.071873+0000", 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.0012898219999999999, 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001405248, 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:31:29.080 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071873+0000", 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071873+0000", 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:31:29.081 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071872+0000", 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.999999 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071879+0000", 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.2989999999999999e-06 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071890+0000", 2023-10-09T04:31:29.082 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0907000000000001e-05 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071898+0000", 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.1899999999999998e-06 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071955+0000", 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.7692000000000003e-05 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.083 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071978+0000", 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 2.2946999999999999e-05 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_repop(client.46577.0:207964 11.7 e573/552 11:e9e0b98d:::benchmark_data_smithi163_601242_object103185:head v 573'12996, mlcod=573'12993)", 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:31:29.071945+0000", 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.001217678, 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.001413215, 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "started", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071945+0000", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071945+0000", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071945+0000", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071949+0000", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.2069999999999997e-06 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071956+0000", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.0910000000000004e-06 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071961+0000", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4.7690000000000004e-06 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.071991+0000", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.0501000000000002e-05 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072009+0000", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.7462e-05 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:207961 11.b 11:d69ee881:::benchmark_data_smithi163_601242_object103182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:31:29.072469+0000", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.00069414499999999996, 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00095155100000000002, 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207961 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072469+0000", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072469+0000", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072467+0000", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072473+0000", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.5899999999999998e-06 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072483+0000", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 1.0429e-05 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072491+0000", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 7.8259999999999993e-06 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "reached_pg", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072597+0000", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00010621800000000001 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "started", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072713+0000", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.000115612 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072839+0000", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00012580999999999999 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "description": "osd_op(client.46577.0:207962 11.b 11.7367ab3b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "initiated_at": "2023-10-09T04:31:29.072504+0000", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "age": 0.00065895899999999998, 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0.00098799999999999995, 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "type_data": { 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "flag_point": "queued for pg", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client_info": { 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client": "client.46577", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "client_addr": "172.21.15.163:0/2247406169", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: "tid": 207962 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "events": [ 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "initiated", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072504+0000", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "throttled", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072504+0000", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 0 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "header_read", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072503+0000", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "all_read", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072507+0000", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 3.3670000000000001e-06 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "dispatched", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072514+0000", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 6.9949999999999999e-06 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: }, 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: { 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: "time": "2023-10-09T04:31:29.072519+0000", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: "duration": 5.3290000000000001e-06 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: ] 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: } 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: ], 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 5 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:31:29.100 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:29.129 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:29.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:29.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:29.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:29.503 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:29.556 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:30.038 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:31:30.101 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:30.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:30.216 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:30.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4136385280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.289 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.289 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4136385280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.290 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.299 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4136385280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.873 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:31:30.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:31:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:31:30] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.43.0" 2023-10-09T04:31:31.202 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:31.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:31.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: failed to encode map e574 with expected crc 2023-10-09T04:31:31.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: failed to encode map e574 with expected crc 2023-10-09T04:31:31.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:31.208 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T04:31:31.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: pgmap v1507: 119 pgs: 119 active+clean; 222 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 368 op/s; 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:31:31.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: failed to encode map e574 with expected crc 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: failed to encode map e574 with expected crc 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: pgmap v1507: 119 pgs: 119 active+clean; 222 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 368 op/s; 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:31:31.212 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:31 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:31.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:31.313 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: failed to encode map e574 with expected crc 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: failed to encode map e574 with expected crc 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: pgmap v1507: 119 pgs: 119 active+clean; 222 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 368 op/s; 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:31:31.349 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:31 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:31.352 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:31.361 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Removed 104776 objects 2023-10-09T04:31:31.361 INFO:tasks.radosbench.radosbench.0.smithi163.stdout:Clean up completed and total clean up time :151.847 2023-10-09T04:31:31.400 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:31.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:31.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:31.531 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:31.573 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:31.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:31.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:31.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:31.703 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:31.745 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:31.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:31.849 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:31.877 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:32.037 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:32.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:32.141 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.168 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:32.210 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:32.262 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:32.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.341 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:32.383 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:32.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:32.524 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:32.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:32.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:32.773 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:31:32.792 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:31:32.819 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:31:32.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:32.855 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:32.856 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:31:32.876 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:31:32.895 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:31:32.896 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:32.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:32.913 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:31:32.932 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:31:32.937 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.937 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.937 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.938 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.938 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.938 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.938 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:32.938 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:33.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:33.031 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:33.074 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:33.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:33.182 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:33.209 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:31:33.210 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:33.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[221067]: pgmap v1508: 119 pgs: 119 active+clean; 88 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 558 op/s; 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:31:33.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:33.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:33.413 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2782790669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:33.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[213841]: pgmap v1508: 119 pgs: 119 active+clean; 88 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 558 op/s; 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:31:33.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:33.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:33.414 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2782790669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:33.452 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 3, 0, 4, 2] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:31:33.452 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:33.452 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T04:31:33.453 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:31:33.453 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:31:33.453 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T04:31:33.475 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T04:31:33.475 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T04:31:33.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:33 smithi181 ceph-mon[196979]: pgmap v1508: 119 pgs: 119 active+clean; 88 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 558 op/s; 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:31:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:33 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:33 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2782790669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:33.758 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:33.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:33.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:33.863 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:33.895 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.203 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:34.259 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:34.307 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:34.337 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:34.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:34.489 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:34.530 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.628 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:34.699 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:34.699 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:34.701 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:34.788 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:34.892 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:34.924 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.970 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:34.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:35.076 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:35.107 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:35.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[221067]: pgmap v1509: 119 pgs: 119 active+clean; 30 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 529 op/s 2023-10-09T04:31:35.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:35.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.441 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[213841]: pgmap v1509: 119 pgs: 119 active+clean; 30 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 529 op/s 2023-10-09T04:31:35.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:35.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.442 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:35 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:35 smithi181 ceph-mon[196979]: pgmap v1509: 119 pgs: 119 active+clean; 30 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 529 op/s 2023-10-09T04:31:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:35.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:35 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:36.157 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:36.229 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:31:36.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:36.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:36.425 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:36.457 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:36.507 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:36.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:36.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:36.613 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:36.640 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:36.683 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:36.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:36.790 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:36.817 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:37.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[213841]: pgmap v1510: 119 pgs: 119 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 484 op/s 2023-10-09T04:31:37.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:37.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:31:37.328 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:37.532 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:37.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:37.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:37.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[221067]: pgmap v1510: 119 pgs: 119 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 484 op/s 2023-10-09T04:31:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:31:37.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:37 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:37.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:37.666 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:37.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:37 smithi181 ceph-mon[196979]: pgmap v1510: 119 pgs: 119 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 484 op/s 2023-10-09T04:31:37.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:37 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:37.667 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:37 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:31:37.668 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:37 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:31:37.670 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:37.703 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:37.719 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:37.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:37.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:37.826 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:37.855 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:37.906 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:37.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:37.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:38.010 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:38.042 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:38.088 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:38.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:38.195 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:38.225 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:38.242 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:31:38.242 INFO:teuthology.orchestra.run.smithi163.stdout: "success": "" 2023-10-09T04:31:38.242 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:31:38.853 INFO:teuthology.orchestra.run.smithi163.stderr:pool 'unique_pool_5' removed 2023-10-09T04:31:39.080 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:31:39.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3509311973' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[221067]: pgmap v1511: 119 pgs: 119 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 377 op/s 2023-10-09T04:31:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1879552571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:39.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:39.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3509311973' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:39.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:39.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[213841]: pgmap v1511: 119 pgs: 119 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 377 op/s 2023-10-09T04:31:39.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1879552571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:39.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:38 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3509311973' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:38 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:31:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:38 smithi181 ceph-mon[196979]: pgmap v1511: 119 pgs: 119 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 377 op/s 2023-10-09T04:31:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1879552571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:38 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:39.325 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:39.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:39.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:39.430 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:39.466 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:39.980 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:39.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[213841]: failed to encode map e575 with expected crc 2023-10-09T04:31:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:31:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[213841]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T04:31:40.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:40.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[221067]: failed to encode map e575 with expected crc 2023-10-09T04:31:40.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:31:40.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:40.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:39 smithi163 ceph-mon[221067]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T04:31:40.084 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:40.110 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:40.250 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:31:40.250 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T04:31:40.250 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:31:40.250 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T04:31:40.251 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:31:40.251 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:40.253 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:40.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:40.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:39 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:39 smithi181 ceph-mon[196979]: failed to encode map e575 with expected crc 2023-10-09T04:31:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:31:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:39 smithi181 ceph-mon[196979]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T04:31:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:40.296 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:31:40.296 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:40.356 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:40.403 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:40.504 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:40.504 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:31:40.561 INFO:tasks.workunit:timeout=3h 2023-10-09T04:31:40.561 INFO:tasks.workunit:cleanup=True 2023-10-09T04:31:40.561 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:31:40.654 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:31:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:31:40] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2023-10-09T04:31:40.807 INFO:tasks.workunit.client.0.smithi163.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T04:31:41.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:40 smithi163 ceph-mon[221067]: failed to encode map e575 with expected crc 2023-10-09T04:31:41.001 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:40 smithi163 ceph-mon[221067]: pgmap v1513: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:41.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:40 smithi163 ceph-mon[213841]: failed to encode map e575 with expected crc 2023-10-09T04:31:41.002 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:40 smithi163 ceph-mon[213841]: pgmap v1513: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:41.046 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:41.150 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:41.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:40 smithi181 ceph-mon[196979]: failed to encode map e575 with expected crc 2023-10-09T04:31:41.179 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:40 smithi181 ceph-mon[196979]: pgmap v1513: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:41.182 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:41.231 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:41.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:41.366 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:41.411 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:41.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:41.543 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:41.586 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:41.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:41.722 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:41.785 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:41.785 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:41.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:42.072 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:42.098 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.141 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:42.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:42.273 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.315 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:42.421 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:42.446 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.469 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:31:42.489 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:42.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:42.594 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:42.624 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.723 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:42.798 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:31:42.823 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:31:42.828 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:42.846 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:31:42.855 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.873 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:42.882 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:31:42.900 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:42.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:42.916 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:31:42.936 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:31:42.954 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:31:42.973 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:31:42.974 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:31:42.975 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:31:43.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:43.030 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.072 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:43.113 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:31:43.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:43.206 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:43.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:43 smithi163 ceph-mon[221067]: pgmap v1514: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:43.259 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4283684922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:43.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:43 smithi163 ceph-mon[213841]: pgmap v1514: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:43.259 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4283684922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:43.355 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:43.380 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.422 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:43 smithi181 ceph-mon[196979]: pgmap v1514: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:43.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4283684922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:43.530 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:43.556 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.597 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:43.706 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:43.731 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.774 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:43.881 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:43.909 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.056 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:44.160 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:44.191 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.242 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:44.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:44.346 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:44.370 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 3, 0, 4, 2] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:31:44.370 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:44.370 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [1, 7, 3, 0, 4, 2] 2023-10-09T04:31:44.371 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T04:31:44.371 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.417 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:44.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:44.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:44.547 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.590 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:44.697 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:44.723 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.838 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:44.890 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:44.930 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:44.944 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:44.975 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:45.160 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:45.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:45.264 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:45.292 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:45.298 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:45 smithi163 ceph-mon[221067]: pgmap v1515: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:45.298 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:45 smithi163 ceph-mon[213841]: pgmap v1515: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:45.334 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:45.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:45.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:45.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:45.473 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:45.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:45 smithi181 ceph-mon[196979]: pgmap v1515: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:45.554 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:31:45.554 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:31:45.554 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:31:45.554 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:31:45.629 INFO:tasks.workunit.client.0.smithi163.stderr:Updating files: 91% (10531/11517) Updating files: 92% (10596/11517) Updating files: 93% (10711/11517) Updating files: 94% (10826/11517) Updating files: 95% (10942/11517) Updating files: 96% (11057/11517) Updating files: 97% (11172/11517) Updating files: 98% (11287/11517) Updating files: 99% (11402/11517) Updating files: 100% (11517/11517) Updating files: 100% (11517/11517), done. 2023-10-09T04:31:45.646 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T04:31:45.716 INFO:tasks.workunit.client.0.smithi163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T04:31:45.719 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:31:45.719 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T04:31:45.720 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:45.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:45.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:45.814 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T04:31:45.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:45.850 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:45.862 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T04:31:45.893 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:45.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:45.906 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:31:45.907 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:31:45.907 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T04:31:45.947 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:31:45.991 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:31:46.000 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:46.022 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T04:31:46.022 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T04:31:46.026 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:46.068 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:46.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:46.086 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T04:31:46.087 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T04:31:46.088 DEBUG:teuthology.orchestra.run.smithi163:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T04:31:46.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:46.201 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:46.242 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:46.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:46.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:46.260 INFO:tasks.workunit.client.0.smithi163.stdout:Running main() from gmock_main.cc 2023-10-09T04:31:46.260 INFO:tasks.workunit.client.0.smithi163.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T04:31:46.260 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] Global test environment set-up. 2023-10-09T04:31:46.260 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T04:31:46.350 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:46.375 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:46.417 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:46.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:46.522 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:46.548 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:47.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:47 smithi181 ceph-mon[196979]: pgmap v1516: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:47.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:47 smithi163 ceph-mon[221067]: pgmap v1516: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:47.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:47 smithi163 ceph-mon[213841]: pgmap v1516: 103 pgs: 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:48.206 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T04:31:48.221 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_all_features (15 ms) 2023-10-09T04:31:48.222 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T04:31:48.411 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.copyup (190 ms) 2023-10-09T04:31:48.411 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T04:31:48.424 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.sparse_copyup (14 ms) 2023-10-09T04:31:48.424 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T04:31:48.432 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-10-09T04:31:48.432 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T04:31:48.442 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-10-09T04:31:48.442 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T04:31:48.468 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.directory_methods (25 ms) 2023-10-09T04:31:48.468 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.create 2023-10-09T04:31:48.480 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.create (13 ms) 2023-10-09T04:31:48.480 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T04:31:48.482 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-10-09T04:31:48.482 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T04:31:48.483 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T04:31:48.483 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T04:31:48.485 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-10-09T04:31:48.485 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T04:31:48.487 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_access_timestamp (3 ms) 2023-10-09T04:31:48.487 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T04:31:48.488 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-09T04:31:48.488 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T04:31:48.492 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-10-09T04:31:48.492 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T04:31:48.495 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-09T04:31:48.495 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T04:31:48.500 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2023-10-09T04:31:48.500 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T04:31:48.517 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-10-09T04:31:48.517 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T04:31:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:48 smithi181 ceph-mon[196979]: failed to encode map e576 with expected crc 2023-10-09T04:31:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:48 smithi181 ceph-mon[196979]: failed to encode map e576 with expected crc 2023-10-09T04:31:48.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:48 smithi181 ceph-mon[196979]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T04:31:48.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2604736680' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-663370-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:31:48.524 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T04:31:48.524 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T04:31:48.566 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.parents_v1 (42 ms) 2023-10-09T04:31:48.566 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[213841]: failed to encode map e576 with expected crc 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[213841]: failed to encode map e576 with expected crc 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[213841]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2604736680' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-663370-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[221067]: failed to encode map e576 with expected crc 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[221067]: failed to encode map e576 with expected crc 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[221067]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T04:31:48.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2604736680' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-663370-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:31:48.595 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2023-10-09T04:31:48.595 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T04:31:48.617 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-10-09T04:31:48.618 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T04:31:48.624 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2023-10-09T04:31:48.625 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T04:31:48.633 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T04:31:48.634 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T04:31:48.638 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2023-10-09T04:31:48.638 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T04:31:48.645 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-10-09T04:31:48.646 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T04:31:48.655 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_update (9 ms) 2023-10-09T04:31:48.655 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T04:31:48.655 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-10-09T04:31:48.656 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T04:31:48.662 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2023-10-09T04:31:48.662 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T04:31:48.671 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.object_map_snap_remove (10 ms) 2023-10-09T04:31:48.672 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T04:31:48.804 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.flags (132 ms) 2023-10-09T04:31:48.804 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T04:31:48.992 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.metadata (188 ms) 2023-10-09T04:31:48.992 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T04:31:48.999 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T04:31:48.999 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T04:31:49.032 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror (34 ms) 2023-10-09T04:31:49.032 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T04:31:49.048 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-10-09T04:31:49.048 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: pgmap v1518: 135 pgs: 11 creating+peering, 21 unknown, 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2604736680' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-663370-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[221067]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: pgmap v1518: 135 pgs: 11 creating+peering, 21 unknown, 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2604736680' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-663370-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:49 smithi163 ceph-mon[213841]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.505 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:49.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: pgmap v1518: 135 pgs: 11 creating+peering, 21 unknown, 103 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail 2023-10-09T04:31:49.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2604736680' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi163-663370-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T04:31:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T04:31:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:49 smithi181 ceph-mon[196979]: failed to encode map e577 with expected crc 2023-10-09T04:31:49.611 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:49.648 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:49.689 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:31:49.724 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:49.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:49.752 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:49.832 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:49.857 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:49.902 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:49.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:50.011 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:50.046 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[213841]: failed to encode map e578 with expected crc 2023-10-09T04:31:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[213841]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T04:31:50.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[213841]: failed to encode map e578 with expected crc 2023-10-09T04:31:50.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[221067]: failed to encode map e578 with expected crc 2023-10-09T04:31:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[221067]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T04:31:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[221067]: failed to encode map e578 with expected crc 2023-10-09T04:31:50.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:50.492 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:50.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:50.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:50 smithi181 ceph-mon[196979]: failed to encode map e578 with expected crc 2023-10-09T04:31:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:50 smithi181 ceph-mon[196979]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T04:31:50.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:50 smithi181 ceph-mon[196979]: failed to encode map e578 with expected crc 2023-10-09T04:31:50.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:50.597 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:50.630 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:50.786 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:50.791 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:31:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:31:50] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-10-09T04:31:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:50.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:50.892 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:50.927 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:50.988 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:50.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:50.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:51.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 conmon[213817]: 2023-10-09T04:31:51.029+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e578 definitely_dead 0 2023-10-09T04:31:51.093 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:51.120 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:51.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:51.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:51.239 INFO:teuthology.orchestra.run.smithi163.stderr:marked out osd.3. 2023-10-09T04:31:51.247 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:31:51.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:51.293 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:51.345 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:51.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:51.450 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:51.476 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:51.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:51 smithi181 ceph-mon[196979]: pgmap v1521: 135 pgs: 11 creating+peering, 19 unknown, 105 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2023-10-09T04:31:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/302553912' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:51 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4277195907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:31:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:31:51.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:51 smithi181 ceph-mon[196979]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:31:51.520 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:51.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[213841]: pgmap v1521: 135 pgs: 11 creating+peering, 19 unknown, 105 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/302553912' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4277195907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[213841]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[221067]: pgmap v1521: 135 pgs: 11 creating+peering, 19 unknown, 105 active+clean; 582 KiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/302553912' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:51.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4277195907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:31:51.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:31:51.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:51 smithi163 ceph-mon[221067]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:31:51.625 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:51.654 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:52.395 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:52.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:52.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:52.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:52.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: osdmap e579: 8 total, 8 up, 5 in 2023-10-09T04:31:52.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:52 smithi181 ceph-mon[196979]: osdmap e580: 8 total, 8 up, 5 in 2023-10-09T04:31:52.529 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:52.563 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image_status (3516 ms) 2023-10-09T04:31:52.563 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T04:31:52.571 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:52.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:31:52.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: osdmap e579: 8 total, 8 up, 5 in 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[221067]: osdmap e580: 8 total, 8 up, 5 in 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: failed to encode map e579 with expected crc 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: osdmap e579: 8 total, 8 up, 5 in 2023-10-09T04:31:52.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:52 smithi163 ceph-mon[213841]: osdmap e580: 8 total, 8 up, 5 in 2023-10-09T04:31:52.664 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_image_map (101 ms) 2023-10-09T04:31:52.664 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T04:31:52.672 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2023-10-09T04:31:52.672 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T04:31:52.678 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:52.684 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2023-10-09T04:31:52.684 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T04:31:52.686 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T04:31:52.686 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T04:31:52.688 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-10-09T04:31:52.688 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T04:31:52.691 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T04:31:52.691 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T04:31:52.697 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T04:31:52.697 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T04:31:52.700 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T04:31:52.700 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T04:31:52.704 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T04:31:52.704 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T04:31:52.706 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T04:31:52.706 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T04:31:52.707 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:52.709 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T04:31:52.709 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T04:31:52.712 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T04:31:52.712 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T04:31:52.717 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-10-09T04:31:52.717 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T04:31:52.719 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T04:31:52.719 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T04:31:52.722 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T04:31:52.723 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T04:31:52.725 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T04:31:52.725 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T04:31:52.726 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T04:31:52.726 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T04:31:52.728 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T04:31:52.729 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T04:31:52.731 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T04:31:52.731 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T04:31:52.734 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T04:31:52.734 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T04:31:52.736 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T04:31:52.737 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T04:31:52.740 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-10-09T04:31:52.740 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T04:31:52.752 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:52.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:52.758 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2023-10-09T04:31:52.758 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T04:31:52.857 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:52.891 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:52.973 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (215 ms) 2023-10-09T04:31:52.973 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T04:31:52.976 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T04:31:52.976 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T04:31:52.978 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T04:31:52.978 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T04:31:52.986 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-10-09T04:31:52.986 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T04:31:52.992 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-10-09T04:31:52.993 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T04:31:53.012 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-10-09T04:31:53.012 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T04:31:53.022 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T04:31:53.022 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T04:31:53.034 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-10-09T04:31:53.035 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T04:31:53.041 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-10-09T04:31:53.042 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T04:31:53.052 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-10-09T04:31:53.052 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T04:31:53.066 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-10-09T04:31:53.066 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T04:31:53.164 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:53.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:53.195 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:53.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:53.298 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:53.357 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:53.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:53.464 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:53.490 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e579 with expected crc 2023-10-09T04:31:53.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: pgmap v1523: 135 pgs: 11 creating+peering, 124 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 1.7 MiB/s rd, 854 KiB/s wr, 147 op/s 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:53 smithi181 ceph-mon[196979]: osdmap e581: 8 total, 8 up, 5 in 2023-10-09T04:31:53.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:53.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:53.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e579 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: pgmap v1523: 135 pgs: 11 creating+peering, 124 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 1.7 MiB/s rd, 854 KiB/s wr, 147 op/s 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[221067]: osdmap e581: 8 total, 8 up, 5 in 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e579 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: pgmap v1523: 135 pgs: 11 creating+peering, 124 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 1.7 MiB/s rd, 854 KiB/s wr, 147 op/s 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: failed to encode map e580 with expected crc 2023-10-09T04:31:53.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:53 smithi163 ceph-mon[213841]: osdmap e581: 8 total, 8 up, 5 in 2023-10-09T04:31:53.638 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:53.675 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:53.678 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:31:53.700 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:31:53.721 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:31:53.725 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:53.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:53.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:53.746 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:31:53.761 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:31:53.780 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:31:53.801 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:31:53.819 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.821 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:53.831 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:53.858 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:54.003 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:54.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:54.107 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:54.135 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:54.228 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1162 ms) 2023-10-09T04:31:54.228 INFO:tasks.workunit.client.0.smithi163.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T04:31:54.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[213841]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[213841]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[213841]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[213841]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2879079530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:54.263 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[213841]: osdmap e582: 8 total, 8 up, 5 in 2023-10-09T04:31:54.409 INFO:tasks.workunit.client.0.smithi163.stdout:[ OK ] TestClsRbd.sparsify (181 ms) 2023-10-09T04:31:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:54 smithi181 ceph-mon[196979]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:54 smithi181 ceph-mon[196979]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:54 smithi181 ceph-mon[196979]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:54 smithi181 ceph-mon[196979]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:54 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2879079530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:54.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:54 smithi181 ceph-mon[196979]: osdmap e582: 8 total, 8 up, 5 in 2023-10-09T04:31:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[221067]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[221067]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[221067]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[221067]: failed to encode map e581 with expected crc 2023-10-09T04:31:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2879079530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:54.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:54 smithi163 ceph-mon[221067]: osdmap e582: 8 total, 8 up, 5 in 2023-10-09T04:31:55.132 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:55.221 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:31:55.236 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:55.260 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] 70 tests from TestClsRbd (6203 ms total) 2023-10-09T04:31:55.260 INFO:tasks.workunit.client.0.smithi163.stdout: 2023-10-09T04:31:55.260 INFO:tasks.workunit.client.0.smithi163.stdout:[----------] Global test environment tear-down 2023-10-09T04:31:55.260 INFO:tasks.workunit.client.0.smithi163.stdout:[==========] 70 tests from 1 test suite ran. (9000 ms total) 2023-10-09T04:31:55.261 INFO:tasks.workunit.client.0.smithi163.stdout:[ PASSED ] 70 tests. 2023-10-09T04:31:55.263 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T04:31:55.264 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T04:31:55.291 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.324 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T04:31:55.324 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:31:55.340 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: pgmap v1526: 135 pgs: 26 peering, 1 unknown, 108 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 2.6 MiB/s rd, 1.4 MiB/s wr, 1.65k op/s 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:31:55.343 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[213841]: osdmap e583: 8 total, 8 up, 5 in 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: pgmap v1526: 135 pgs: 26 peering, 1 unknown, 108 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 2.6 MiB/s rd, 1.4 MiB/s wr, 1.65k op/s 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:31:55.344 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:55 smithi163 ceph-mon[221067]: osdmap e583: 8 total, 8 up, 5 in 2023-10-09T04:31:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:55.446 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:55.475 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: pgmap v1526: 135 pgs: 26 peering, 1 unknown, 108 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 2.6 MiB/s rd, 1.4 MiB/s wr, 1.65k op/s 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: failed to encode map e582 with expected crc 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:31:55.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:55 smithi181 ceph-mon[196979]: osdmap e583: 8 total, 8 up, 5 in 2023-10-09T04:31:55.575 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:55.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:55.678 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:55.693 DEBUG:teuthology.parallel:result is None 2023-10-09T04:31:55.693 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:31:55.709 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.748 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:31:55.749 DEBUG:teuthology.orchestra.run.smithi163:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:55.763 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:55.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:55.782 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:31:55.782 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:31:55.782 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T04:31:55.782 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:31:55.782 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T04:31:55.783 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:31:55.783 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:55.819 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:31:55.841 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:31:55.841 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:55.868 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:55.880 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:55.881 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:55.898 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:31:55.899 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:31:55.900 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.949 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:55.956 INFO:tasks.workunit:timeout=3h 2023-10-09T04:31:55.956 INFO:tasks.workunit:cleanup=True 2023-10-09T04:31:55.957 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:31:56.033 INFO:tasks.workunit.client.0.smithi163.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T04:31:56.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:56.084 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:56.128 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:56.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:56.235 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:56.262 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:56.307 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:56.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:56.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:56.440 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:56.481 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:56.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:56.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:56.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:56 smithi181 ceph-mon[196979]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:56 smithi181 ceph-mon[196979]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:56 smithi181 ceph-mon[196979]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:56 smithi181 ceph-mon[196979]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:56 smithi181 ceph-mon[196979]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:56 smithi181 ceph-mon[196979]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[221067]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[221067]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[221067]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[221067]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[221067]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[221067]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[213841]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[213841]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[213841]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[213841]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[213841]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:56 smithi163 ceph-mon[213841]: failed to encode map e583 with expected crc 2023-10-09T04:31:56.588 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:56.617 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:56.933 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:56.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:56.939 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:56.989 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:31:57.038 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:57.065 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.107 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:57.214 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:57.243 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:57 smithi181 ceph-mon[196979]: pgmap v1529: 103 pgs: 5 activating, 1 active, 20 peering, 1 unknown, 76 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:57.564 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:57.566 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:57 smithi163 ceph-mon[221067]: pgmap v1529: 103 pgs: 5 activating, 1 active, 20 peering, 1 unknown, 76 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:57.566 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:57 smithi163 ceph-mon[213841]: pgmap v1529: 103 pgs: 5 activating, 1 active, 20 peering, 1 unknown, 76 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:31:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:57.668 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:57.700 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.746 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:57.772 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:57.838 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:57.853 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:57.884 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.923 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 0, 4, 2] out_osds: [5, 6, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:31:57.923 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:31:57.923 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T04:31:57.924 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T04:31:57.930 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:58.035 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:58.064 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:58.217 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:58.247 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.348 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:58.452 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:58.483 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.531 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:58.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:58.662 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.706 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:58.812 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:58.834 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is set 2023-10-09T04:31:58.839 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:58 smithi163 ceph-mon[221067]: pgmap v1530: 103 pgs: 5 activating, 1 active, 97 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 63 KiB/s, 1 objects/s recovering 2023-10-09T04:31:58.861 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2959361655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:58.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:58 smithi163 ceph-mon[213841]: pgmap v1530: 103 pgs: 5 activating, 1 active, 97 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 63 KiB/s, 1 objects/s recovering 2023-10-09T04:31:58.861 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2959361655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:58.880 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:58.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:59.013 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:59.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:58 smithi181 ceph-mon[196979]: pgmap v1530: 103 pgs: 5 activating, 1 active, 97 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 63 KiB/s, 1 objects/s recovering 2023-10-09T04:31:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2959361655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:00.061 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: failed to encode map e584 with expected crc 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: failed to encode map e584 with expected crc 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2959361655' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: osdmap e584: 8 total, 8 up, 5 in 2023-10-09T04:32:00.063 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: 1.4 scrub starts 2023-10-09T04:32:00.064 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[221067]: 1.4 scrub ok 2023-10-09T04:32:00.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-10-09T04:32:00.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:32:00.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:00.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: failed to encode map e584 with expected crc 2023-10-09T04:32:00.064 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: failed to encode map e584 with expected crc 2023-10-09T04:32:00.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2959361655' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:00.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: osdmap e584: 8 total, 8 up, 5 in 2023-10-09T04:32:00.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: 1.4 scrub starts 2023-10-09T04:32:00.065 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:31:59 smithi163 ceph-mon[213841]: 1.4 scrub ok 2023-10-09T04:32:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:00.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:00.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-10-09T04:32:00.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:32:00.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:00.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: failed to encode map e584 with expected crc 2023-10-09T04:32:00.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: failed to encode map e584 with expected crc 2023-10-09T04:32:00.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2959361655' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:00.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: osdmap e584: 8 total, 8 up, 5 in 2023-10-09T04:32:00.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: 1.4 scrub starts 2023-10-09T04:32:00.197 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:31:59 smithi181 ceph-mon[196979]: 1.4 scrub ok 2023-10-09T04:32:00.199 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:00.248 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:00.354 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:00.382 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:00.589 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:00.655 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:00.694 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:00.724 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:00.966 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:00.969 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:32:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:32:00] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-10-09T04:32:00.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:00.997 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:00.998 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:01.071 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:01.103 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:01.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:01 smithi181 ceph-mon[196979]: pgmap v1532: 103 pgs: 5 activating, 1 active, 97 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 62 KiB/s, 1 objects/s recovering 2023-10-09T04:32:01.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:01 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:01 smithi163 ceph-mon[213841]: pgmap v1532: 103 pgs: 5 activating, 1 active, 97 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 62 KiB/s, 1 objects/s recovering 2023-10-09T04:32:01.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:01 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:01.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:01 smithi163 ceph-mon[221067]: pgmap v1532: 103 pgs: 5 activating, 1 active, 97 active+clean; 599 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 62 KiB/s, 1 objects/s recovering 2023-10-09T04:32:01.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:01 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:01.617 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T04:32:01.675 INFO:tasks.workunit.client.0.smithi163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T04:32:01.677 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:32:01.678 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T04:32:01.731 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T04:32:01.784 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T04:32:02.448 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T04:32:02.449 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:32:02.450 INFO:tasks.workunit.client.0.smithi163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T04:32:02.553 INFO:tasks.workunit.client.0.smithi163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T04:32:02.556 DEBUG:teuthology.orchestra.run.smithi163:> set -ex 2023-10-09T04:32:02.556 DEBUG:teuthology.orchestra.run.smithi163:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T04:32:02.614 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T04:32:02.615 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T04:32:02.615 DEBUG:teuthology.orchestra.run.smithi163:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T04:32:02.675 INFO:tasks.workunit.client.0.smithi163.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T04:32:02.675 INFO:tasks.workunit.client.0.smithi163.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T04:32:02.675 INFO:tasks.workunit.client.0.smithi163.stderr:+ TMPDIR=/tmp/rbd_import_export_667414 2023-10-09T04:32:02.676 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm -rf /tmp/rbd_import_export_667414 2023-10-09T04:32:02.676 INFO:tasks.workunit.client.0.smithi163.stderr:+ mkdir /tmp/rbd_import_export_667414 2023-10-09T04:32:02.677 INFO:tasks.workunit.client.0.smithi163.stderr:+ trap 'rm -rf /tmp/rbd_import_export_667414' INT TERM EXIT 2023-10-09T04:32:02.677 INFO:tasks.workunit.client.0.smithi163.stderr:+ mkdir foo.667414 2023-10-09T04:32:02.677 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import foo.667414 foo.dir 2023-10-09T04:32:02.683 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:32:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:03 smithi181 ceph-mon[196979]: pgmap v1533: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 41 KiB/s, 1 objects/s recovering 2023-10-09T04:32:03.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:03 smithi163 ceph-mon[221067]: pgmap v1533: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 41 KiB/s, 1 objects/s recovering 2023-10-09T04:32:03.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:03 smithi163 ceph-mon[213841]: pgmap v1533: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 41 KiB/s, 1 objects/s recovering 2023-10-09T04:32:03.776 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: cannot import a directory 2023-10-09T04:32:03.776 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 0% complete...failed. 2023-10-09T04:32:03.777 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: import failed: (21) Is a directory 2023-10-09T04:32:03.779 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T04:32:03.779 INFO:tasks.workunit.client.0.smithi163.stderr:+ rmdir foo.667414 2023-10-09T04:32:03.780 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_667414/img bs=1k count=1 seek=10 2023-10-09T04:32:03.781 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T04:32:03.781 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T04:32:03.781 INFO:tasks.workunit.client.0.smithi163.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.6403e-05 s, 22.1 MB/s 2023-10-09T04:32:03.781 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_667414/img bs=1k count=10 seek=100 2023-10-09T04:32:03.782 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records in 2023-10-09T04:32:03.782 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records out 2023-10-09T04:32:03.782 INFO:tasks.workunit.client.0.smithi163.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.1871e-05 s, 197 MB/s 2023-10-09T04:32:03.782 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_667414/img bs=1k count=100 seek=1000 2023-10-09T04:32:03.783 INFO:tasks.workunit.client.0.smithi163.stderr:70+1 records in 2023-10-09T04:32:03.783 INFO:tasks.workunit.client.0.smithi163.stderr:70+1 records out 2023-10-09T04:32:03.783 INFO:tasks.workunit.client.0.smithi163.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000174517 s, 413 MB/s 2023-10-09T04:32:03.783 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_667414/img bs=1k seek=10000 2023-10-09T04:32:03.783 INFO:tasks.workunit.client.0.smithi163.stderr:139+1 records in 2023-10-09T04:32:03.784 INFO:tasks.workunit.client.0.smithi163.stderr:139+1 records out 2023-10-09T04:32:03.784 INFO:tasks.workunit.client.0.smithi163.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000293253 s, 488 MB/s 2023-10-09T04:32:03.784 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_667414/img bs=1k seek=100000 2023-10-09T04:32:03.816 INFO:tasks.workunit.client.0.smithi163.stderr:70+1 records in 2023-10-09T04:32:03.816 INFO:tasks.workunit.client.0.smithi163.stderr:70+1 records out 2023-10-09T04:32:03.816 INFO:tasks.workunit.client.0.smithi163.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0314216 s, 2.3 MB/s 2023-10-09T04:32:03.816 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_667414/img bs=1k seek=1000000 2023-10-09T04:32:03.817 INFO:tasks.workunit.client.0.smithi163.stderr:189+1 records in 2023-10-09T04:32:03.817 INFO:tasks.workunit.client.0.smithi163.stderr:189+1 records out 2023-10-09T04:32:03.817 INFO:tasks.workunit.client.0.smithi163.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000434126 s, 446 MB/s 2023-10-09T04:32:03.817 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T04:32:03.825 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:03.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:03.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:03.857 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 0% complete...failed. 2023-10-09T04:32:03.857 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:32:03.860 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T04:32:03.861 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format /tmp/rbd_import_export_667414/img testimg 2023-10-09T04:32:03.930 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:03.960 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:03.986 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:32:04.031 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:32:04.051 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:32:04.073 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:32:04.095 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:32:04.113 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:04.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:04.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:04.124 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:32:04.145 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:32:04.165 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:32:04.196 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:32:04.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.197 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.198 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.198 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.198 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.198 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.198 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:32:04.220 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:04.248 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:04.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2956537969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2956537969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2956537969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:05.059 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:05.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:05.163 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:05.197 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:05.246 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:05.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:05.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:05.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[213841]: pgmap v1534: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 36 KiB/s, 1 objects/s recovering 2023-10-09T04:32:05.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[213841]: failed to encode map e585 with expected crc 2023-10-09T04:32:05.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[213841]: failed to encode map e585 with expected crc 2023-10-09T04:32:05.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[213841]: osdmap e585: 8 total, 8 up, 5 in 2023-10-09T04:32:05.331 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[221067]: pgmap v1534: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 36 KiB/s, 1 objects/s recovering 2023-10-09T04:32:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[221067]: failed to encode map e585 with expected crc 2023-10-09T04:32:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[221067]: failed to encode map e585 with expected crc 2023-10-09T04:32:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[221067]: osdmap e585: 8 total, 8 up, 5 in 2023-10-09T04:32:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:05.333 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:05.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:05.381 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:05.426 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:05.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:05.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:05 smithi181 ceph-mon[196979]: pgmap v1534: 103 pgs: 103 active+clean; 582 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 36 KiB/s, 1 objects/s recovering 2023-10-09T04:32:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:05 smithi181 ceph-mon[196979]: failed to encode map e585 with expected crc 2023-10-09T04:32:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:05 smithi181 ceph-mon[196979]: failed to encode map e585 with expected crc 2023-10-09T04:32:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:05 smithi181 ceph-mon[196979]: osdmap e585: 8 total, 8 up, 5 in 2023-10-09T04:32:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:05.531 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:05.564 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:05.695 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:05.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:05.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:05.711 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:05.801 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:05.837 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:05.928 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T04:32:05.929 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:05.933 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg /tmp/rbd_import_export_667414/img2 2023-10-09T04:32:06.248 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:06.251 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:06.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:06.255 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg - 2023-10-09T04:32:06.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:06.382 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:06.425 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:06.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:06.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:06.530 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:06.561 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[221067]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[221067]: osdmap e586: 8 total, 8 up, 5 in 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[221067]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[221067]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[213841]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[213841]: osdmap e586: 8 total, 8 up, 5 in 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[213841]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.562 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:06 smithi163 ceph-mon[213841]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.566 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:06 smithi181 ceph-mon[196979]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:06 smithi181 ceph-mon[196979]: osdmap e586: 8 total, 8 up, 5 in 2023-10-09T04:32:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:06 smithi181 ceph-mon[196979]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:06 smithi181 ceph-mon[196979]: failed to encode map e586 with expected crc 2023-10-09T04:32:06.823 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:06.825 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:06 smithi163 conmon[213817]: 2023-10-09T04:32:06.754+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e586 definitely_dead 0 2023-10-09T04:32:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:06.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:06.928 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:06.955 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:06.998 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:07.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:07.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:07.125 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:07.129 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T04:32:07.134 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:07.217 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T04:32:07.220 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img2 2023-10-09T04:32:07.321 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.5. 2023-10-09T04:32:07.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:32:07.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:07 smithi163 ceph-mon[221067]: pgmap v1537: 103 pgs: 103 active+clean; 2.3 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s rd, 1 op/s 2023-10-09T04:32:07.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1183666612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T04:32:07.395 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T04:32:07.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:07 smithi163 ceph-mon[213841]: pgmap v1537: 103 pgs: 103 active+clean; 2.3 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s rd, 1 op/s 2023-10-09T04:32:07.395 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1183666612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T04:32:07.396 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T04:32:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:07 smithi181 ceph-mon[196979]: pgmap v1537: 103 pgs: 103 active+clean; 2.3 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s rd, 1 op/s 2023-10-09T04:32:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1183666612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T04:32:07.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T04:32:08.017 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img3 2023-10-09T04:32:08.466 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img2 /tmp/rbd_import_export_667414/img3 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: osdmap e587: 8 total, 8 up, 6 in 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: osdmap e588: 8 total, 8 up, 6 in 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[213841]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T04:32:08.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: osdmap e587: 8 total, 8 up, 6 in 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: osdmap e588: 8 total, 8 up, 6 in 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:08 smithi163 ceph-mon[221067]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.638 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:08.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:08.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:08.742 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:08.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: osdmap e587: 8 total, 8 up, 6 in 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e587 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: osdmap e588: 8 total, 8 up, 6 in 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:08 smithi181 ceph-mon[196979]: failed to encode map e588 with expected crc 2023-10-09T04:32:08.771 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:08.815 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:08.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:08.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:08.922 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:08.948 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:08.989 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:08.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:09.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:09.120 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: failed to encode map e588 with expected crc 2023-10-09T04:32:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: failed to encode map e588 with expected crc 2023-10-09T04:32:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: pgmap v1540: 103 pgs: 103 active+clean; 4.9 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 130 KiB/s rd, 104 KiB/s wr, 23 op/s 2023-10-09T04:32:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: osdmap e589: 8 total, 8 up, 6 in 2023-10-09T04:32:09.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: failed to encode map e588 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: failed to encode map e588 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: pgmap v1540: 103 pgs: 103 active+clean; 4.9 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 130 KiB/s rd, 104 KiB/s wr, 23 op/s 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: osdmap e589: 8 total, 8 up, 6 in 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:09 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: failed to encode map e588 with expected crc 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: failed to encode map e588 with expected crc 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: pgmap v1540: 103 pgs: 103 active+clean; 4.9 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 130 KiB/s rd, 104 KiB/s wr, 23 op/s 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: osdmap e589: 8 total, 8 up, 6 in 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:09.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:09 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[221067]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:10 smithi163 ceph-mon[213841]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.725 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format - testimg 2023-10-09T04:32:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:10 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:10 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:10 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:10.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:10 smithi181 ceph-mon[196979]: failed to encode map e589 with expected crc 2023-10-09T04:32:11.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:32:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:32:10] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.43.0" 2023-10-09T04:32:11.152 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:11.153 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:11.156 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg /tmp/rbd_import_export_667414/img2 2023-10-09T04:32:11.463 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:11.466 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg - 2023-10-09T04:32:11.581 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: pgmap v1542: 103 pgs: 4 activating+remapped, 99 active+clean; 4.8 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 114 KiB/s rd, 101 KiB/s wr, 24 op/s; 1/96 objects misplaced (1.042%); 11 KiB/s, 0 objects/s recovering 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: osdmap e590: 8 total, 8 up, 6 in 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[221067]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: pgmap v1542: 103 pgs: 4 activating+remapped, 99 active+clean; 4.8 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 114 KiB/s rd, 101 KiB/s wr, 24 op/s; 1/96 objects misplaced (1.042%); 11 KiB/s, 0 objects/s recovering 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: osdmap e590: 8 total, 8 up, 6 in 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:11 smithi163 ceph-mon[213841]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: pgmap v1542: 103 pgs: 4 activating+remapped, 99 active+clean; 4.8 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 114 KiB/s rd, 101 KiB/s wr, 24 op/s; 1/96 objects misplaced (1.042%); 11 KiB/s, 0 objects/s recovering 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: osdmap e590: 8 total, 8 up, 6 in 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:11 smithi181 ceph-mon[196979]: failed to encode map e590 with expected crc 2023-10-09T04:32:11.798 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:11.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:11.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:11.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:11.936 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:11.986 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:11.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:12.090 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T04:32:12.092 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:12.127 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:12.194 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:12.270 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:12.273 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T04:32:12.285 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:12.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:12.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:12.373 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T04:32:12.376 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img2 2023-10-09T04:32:12.390 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:12.417 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:12.460 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:12.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:12.567 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: osdmap e591: 8 total, 8 up, 6 in 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[221067]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: osdmap e591: 8 total, 8 up, 6 in 2023-10-09T04:32:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:12 smithi163 ceph-mon[213841]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.602 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:12.678 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:32:12.678 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:32:12.678 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:32:12.678 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: osdmap e591: 8 total, 8 up, 6 in 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:12 smithi181 ceph-mon[196979]: failed to encode map e591 with expected crc 2023-10-09T04:32:12.956 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:12.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:13.046 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img3 2023-10-09T04:32:13.062 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:13.092 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:13.141 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:13.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:13.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:13.244 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:13.273 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:13.319 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:13.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:13.423 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:13.455 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:13.504 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:13.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:13.514 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img2 /tmp/rbd_import_export_667414/img3 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[221067]: pgmap v1545: 103 pgs: 3 peering, 2 active+undersized+remapped, 4 activating+remapped, 94 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 206 KiB/s rd, 84 KiB/s wr, 58 op/s; 1/87 objects misplaced (1.149%); 87 KiB/s, 1 objects/s recovering 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[221067]: osdmap e592: 8 total, 8 up, 6 in 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3858936398' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[213841]: pgmap v1545: 103 pgs: 3 peering, 2 active+undersized+remapped, 4 activating+remapped, 94 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 206 KiB/s rd, 84 KiB/s wr, 58 op/s; 1/87 objects misplaced (1.149%); 87 KiB/s, 1 objects/s recovering 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[213841]: osdmap e592: 8 total, 8 up, 6 in 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3858936398' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:13.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:13 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:13.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:13.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:13 smithi181 ceph-mon[196979]: pgmap v1545: 103 pgs: 3 peering, 2 active+undersized+remapped, 4 activating+remapped, 94 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 206 KiB/s rd, 84 KiB/s wr, 58 op/s; 1/87 objects misplaced (1.149%); 87 KiB/s, 1 objects/s recovering 2023-10-09T04:32:13.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:13 smithi181 ceph-mon[196979]: osdmap e592: 8 total, 8 up, 6 in 2023-10-09T04:32:13.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3858936398' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:13.635 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:13 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:13.637 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:13.680 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:13.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:13.786 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:13.812 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:13.841 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is set 2023-10-09T04:32:13.857 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:13.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:13.961 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:13.987 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:14.028 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:14.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:14.132 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:14.162 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:14.207 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:14.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:14.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:14.314 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:14.341 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:14.384 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:14.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:14.491 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:14 smithi181 ceph-mon[196979]: failed to encode map e592 with expected crc 2023-10-09T04:32:14.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:14 smithi181 ceph-mon[196979]: failed to encode map e592 with expected crc 2023-10-09T04:32:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:14 smithi181 ceph-mon[196979]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:14 smithi181 ceph-mon[196979]: failed to encode map e593 with expected crc 2023-10-09T04:32:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:14 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:14 smithi181 ceph-mon[196979]: osdmap e593: 8 total, 8 up, 6 in 2023-10-09T04:32:14.520 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[213841]: failed to encode map e592 with expected crc 2023-10-09T04:32:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[213841]: failed to encode map e592 with expected crc 2023-10-09T04:32:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[213841]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[213841]: failed to encode map e593 with expected crc 2023-10-09T04:32:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[213841]: osdmap e593: 8 total, 8 up, 6 in 2023-10-09T04:32:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[221067]: failed to encode map e592 with expected crc 2023-10-09T04:32:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[221067]: failed to encode map e592 with expected crc 2023-10-09T04:32:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[221067]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[221067]: failed to encode map e593 with expected crc 2023-10-09T04:32:14.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:14.584 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:14 smithi163 ceph-mon[221067]: osdmap e593: 8 total, 8 up, 6 in 2023-10-09T04:32:14.968 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:14.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:15.071 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:15.102 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:15.150 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:15.257 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.290 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:15.342 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:15.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:15.446 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:15.475 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:15 smithi181 ceph-mon[196979]: failed to encode map e593 with expected crc 2023-10-09T04:32:15.475 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:15 smithi181 ceph-mon[196979]: pgmap v1548: 103 pgs: 3 peering, 2 active+undersized+remapped, 4 activating+remapped, 94 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 215 KiB/s rd, 72 KiB/s wr, 56 op/s; 1/87 objects misplaced (1.149%); 80 KiB/s, 1 objects/s recovering 2023-10-09T04:32:15.477 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:15.524 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:15 smithi163 ceph-mon[221067]: failed to encode map e593 with expected crc 2023-10-09T04:32:15.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:15 smithi163 ceph-mon[221067]: pgmap v1548: 103 pgs: 3 peering, 2 active+undersized+remapped, 4 activating+remapped, 94 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 215 KiB/s rd, 72 KiB/s wr, 56 op/s; 1/87 objects misplaced (1.149%); 80 KiB/s, 1 objects/s recovering 2023-10-09T04:32:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:15 smithi163 ceph-mon[213841]: failed to encode map e593 with expected crc 2023-10-09T04:32:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:15 smithi163 ceph-mon[213841]: pgmap v1548: 103 pgs: 3 peering, 2 active+undersized+remapped, 4 activating+remapped, 94 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 215 KiB/s rd, 72 KiB/s wr, 56 op/s; 1/87 objects misplaced (1.149%); 80 KiB/s, 1 objects/s recovering 2023-10-09T04:32:15.632 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:15.660 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:15.704 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:15.811 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:15.843 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:15.895 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:15.999 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:16.027 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.072 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:16.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:16.175 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:16.204 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.253 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:16.358 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:16.390 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.438 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:16.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:16 smithi181 ceph-mon[196979]: pgmap v1549: 103 pgs: 103 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 275 KiB/s rd, 90 KiB/s wr, 110 op/s; 99 KiB/s, 1 objects/s recovering 2023-10-09T04:32:16.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:16 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T04:32:16.542 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:16.570 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.614 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:16.719 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:16.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:16 smithi163 ceph-mon[221067]: pgmap v1549: 103 pgs: 103 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 275 KiB/s rd, 90 KiB/s wr, 110 op/s; 99 KiB/s, 1 objects/s recovering 2023-10-09T04:32:16.748 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:16 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T04:32:16.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:16 smithi163 ceph-mon[213841]: pgmap v1549: 103 pgs: 103 active+clean; 1.2 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 275 KiB/s rd, 90 KiB/s wr, 110 op/s; 99 KiB/s, 1 objects/s recovering 2023-10-09T04:32:16.748 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:16 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T04:32:16.751 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.801 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:16.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:16.934 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.981 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:16.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:17.086 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:17.091 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 0, 4, 2, 5] out_osds: [6, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:32:17.091 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:32:17.091 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T04:32:17.091 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T04:32:17.118 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:17.171 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:17.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:17.275 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:17.301 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:17.347 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:17.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:17.452 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:17.479 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:18 smithi181 ceph-mon[196979]: pgmap v1550: 103 pgs: 103 active+clean; 471 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 225 KiB/s rd, 77 KiB/s wr, 74 op/s; 85 KiB/s, 1 objects/s recovering 2023-10-09T04:32:19.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:18 smithi163 ceph-mon[221067]: pgmap v1550: 103 pgs: 103 active+clean; 471 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 225 KiB/s rd, 77 KiB/s wr, 74 op/s; 85 KiB/s, 1 objects/s recovering 2023-10-09T04:32:19.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:18 smithi163 ceph-mon[213841]: pgmap v1550: 103 pgs: 103 active+clean; 471 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 225 KiB/s rd, 77 KiB/s wr, 74 op/s; 85 KiB/s, 1 objects/s recovering 2023-10-09T04:32:19.499 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd help export 2023-10-09T04:32:19.499 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -q export-format 2023-10-09T04:32:19.519 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_667414/img bs=1k count=10 seek=100 2023-10-09T04:32:19.568 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records in 2023-10-09T04:32:19.568 INFO:tasks.workunit.client.0.smithi163.stderr:10+0 records out 2023-10-09T04:32:19.568 INFO:tasks.workunit.client.0.smithi163.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.2619e-05 s, 141 MB/s 2023-10-09T04:32:19.568 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format /tmp/rbd_import_export_667414/img testimg 2023-10-09T04:32:19.624 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:19.625 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:19.627 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap create testimg@snap 2023-10-09T04:32:19.859 INFO:tasks.workunit.client.0.smithi163.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:32:19.865 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T04:32:19.916 INFO:tasks.workunit.client.0.smithi163.stderr:++ rbd image-meta list testimg 2023-10-09T04:32:19.954 INFO:tasks.workunit.client.0.smithi163.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T04:32:19.954 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T04:32:19.954 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T04:32:19.954 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1' 2023-10-09T04:32:19.954 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:19.991 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:19.994 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667414/img_v2 testimg_import 2023-10-09T04:32:20.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:20.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:20.119 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:20.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:20.252 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:20.301 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:20.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:20.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:32:20.405 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:20.436 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.486 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:20.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:20.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:20.620 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:20.621 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.666 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:20.773 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:20.801 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.874 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:32:20.877 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T04:32:20.914 INFO:tasks.workunit.client.0.smithi163.stdout:rbd image 'testimg_import': 2023-10-09T04:32:20.914 INFO:tasks.workunit.client.0.smithi163.stdout: size 110 KiB in 1 objects 2023-10-09T04:32:20.914 INFO:tasks.workunit.client.0.smithi163.stdout: order 22 (4 MiB objects) 2023-10-09T04:32:20.914 INFO:tasks.workunit.client.0.smithi163.stdout: snapshot_count: 1 2023-10-09T04:32:20.914 INFO:tasks.workunit.client.0.smithi163.stdout: id: b7c2330cd5a4 2023-10-09T04:32:20.914 INFO:tasks.workunit.client.0.smithi163.stdout: block_name_prefix: rbd_data.b7c2330cd5a4 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: format: 2 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: op_features: 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: flags: 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: create_timestamp: Mon Oct 9 04:32:20 2023 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: access_timestamp: Mon Oct 9 04:32:20 2023 2023-10-09T04:32:20.915 INFO:tasks.workunit.client.0.smithi163.stdout: modify_timestamp: Mon Oct 9 04:32:20 2023 2023-10-09T04:32:20.917 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import@snap 2023-10-09T04:32:20.950 INFO:tasks.workunit.client.0.smithi163.stdout:rbd image 'testimg_import': 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: size 110 KiB in 1 objects 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: order 22 (4 MiB objects) 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: snapshot_count: 1 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: id: b7c2330cd5a4 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: block_name_prefix: rbd_data.b7c2330cd5a4 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: format: 2 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: op_features: 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: flags: 2023-10-09T04:32:20.951 INFO:tasks.workunit.client.0.smithi163.stdout: create_timestamp: Mon Oct 9 04:32:20 2023 2023-10-09T04:32:20.952 INFO:tasks.workunit.client.0.smithi163.stdout: access_timestamp: Mon Oct 9 04:32:20 2023 2023-10-09T04:32:20.952 INFO:tasks.workunit.client.0.smithi163.stdout: modify_timestamp: Mon Oct 9 04:32:20 2023 2023-10-09T04:32:20.952 INFO:tasks.workunit.client.0.smithi163.stdout: protected: False 2023-10-09T04:32:20.954 INFO:tasks.workunit.client.0.smithi163.stderr:++ rbd image-meta list testimg_import 2023-10-09T04:32:20.992 INFO:tasks.workunit.client.0.smithi163.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1' 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T04:32:20.993 INFO:tasks.workunit.client.0.smithi163.stderr: 2023-10-09T04:32:20.994 INFO:tasks.workunit.client.0.smithi163.stderr:Key Value 2023-10-09T04:32:20.994 INFO:tasks.workunit.client.0.smithi163.stderr:key1 value1' ']' 2023-10-09T04:32:20.994 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg_import /tmp/rbd_import_export_667414/img_import 2023-10-09T04:32:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:32:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:32:20] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2023-10-09T04:32:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: osdmap e594: 8 total, 8 up, 6 in 2023-10-09T04:32:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[213841]: pgmap v1552: 103 pgs: 103 active+clean; 471 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 156 KiB/s rd, 46 KiB/s wr, 65 op/s; 35 KiB/s, 0 objects/s recovering 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: osdmap e594: 8 total, 8 up, 6 in 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.084 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:20 smithi163 ceph-mon[221067]: pgmap v1552: 103 pgs: 103 active+clean; 471 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 156 KiB/s rd, 46 KiB/s wr, 65 op/s; 35 KiB/s, 0 objects/s recovering 2023-10-09T04:32:21.090 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T04:32:21.093 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img_import 2023-10-09T04:32:21.094 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_667414/img /tmp/rbd_import_export_667414/img_import 2023-10-09T04:32:21.094 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/img --format %b 2023-10-09T04:32:21.095 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=24 2023-10-09T04:32:21.095 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/img_import --format %b 2023-10-09T04:32:21.096 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=224 2023-10-09T04:32:21.096 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=200 2023-10-09T04:32:21.097 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=200 2023-10-09T04:32:21.097 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T04:32:21.097 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_667414/img_snap 2023-10-09T04:32:21.129 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T04:32:21.132 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_667414/img_snap_import 2023-10-09T04:32:21.167 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T04:32:21.170 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667414/img_snap /tmp/rbd_import_export_667414/img_snap_import 2023-10-09T04:32:21.170 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_667414/img_snap /tmp/rbd_import_export_667414/img_snap_import 2023-10-09T04:32:21.171 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/img_snap --format %b 2023-10-09T04:32:21.171 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=224 2023-10-09T04:32:21.172 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/img_snap_import --format %b 2023-10-09T04:32:21.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=224 2023-10-09T04:32:21.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:21.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:21.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:32:21.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:21.174 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_import 2023-10-09T04:32:21.174 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_snap 2023-10-09T04:32:21.174 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_snap_import 2023-10-09T04:32:21.175 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T04:32:21.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: osdmap e594: 8 total, 8 up, 6 in 2023-10-09T04:32:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: failed to encode map e594 with expected crc 2023-10-09T04:32:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:20 smithi181 ceph-mon[196979]: pgmap v1552: 103 pgs: 103 active+clean; 471 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 156 KiB/s rd, 46 KiB/s wr, 65 op/s; 35 KiB/s, 0 objects/s recovering 2023-10-09T04:32:21.560 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:21.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:21.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:21.664 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:21.666 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:21.696 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:21.864 INFO:tasks.workunit.client.0.smithi163.stderr: Removing snap: 100% complete...done. 2023-10-09T04:32:21.872 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T04:32:21.967 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:21.970 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap rm testimg@snap 2023-10-09T04:32:22.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[221067]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[221067]: osdmap e595: 8 total, 8 up, 6 in 2023-10-09T04:32:22.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[221067]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[221067]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[221067]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[213841]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[213841]: osdmap e595: 8 total, 8 up, 6 in 2023-10-09T04:32:22.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[213841]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[213841]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:21 smithi163 ceph-mon[213841]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.225 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:22.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:22.238 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:32:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:21 smithi181 ceph-mon[196979]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:21 smithi181 ceph-mon[196979]: osdmap e595: 8 total, 8 up, 6 in 2023-10-09T04:32:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:21 smithi181 ceph-mon[196979]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:21 smithi181 ceph-mon[196979]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:21 smithi181 ceph-mon[196979]: failed to encode map e595 with expected crc 2023-10-09T04:32:22.329 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:22.358 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:22.410 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:22.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:22.515 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:22.551 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:22.600 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:22.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:22.704 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:22.734 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:22.870 INFO:tasks.workunit.client.0.smithi163.stderr: Removing snap: 100% complete...done. 2023-10-09T04:32:22.877 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm testimg 2023-10-09T04:32:22.958 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:32:22.982 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:32:22.989 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:22.993 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --order 20 /tmp/rbd_import_export_667414/img testimg 2023-10-09T04:32:23.007 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:32:23.037 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:32:23.059 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:32:23.064 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:23.065 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:32:23.067 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:23.078 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:32:23.104 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:32:23.111 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:23.116 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667414/img_v2 testimg_import 2023-10-09T04:32:23.123 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.125 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.126 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:32:23.135 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:23.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[221067]: osdmap e596: 8 total, 8 up, 6 in 2023-10-09T04:32:23.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[221067]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[221067]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[221067]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[221067]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.139 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[221067]: pgmap v1555: 103 pgs: 103 active+clean; 691 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 82 KiB/s rd, 6.7 KiB/s wr, 63 op/s 2023-10-09T04:32:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:23.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:23.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[213841]: osdmap e596: 8 total, 8 up, 6 in 2023-10-09T04:32:23.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[213841]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[213841]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[213841]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[213841]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.140 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:22 smithi163 ceph-mon[213841]: pgmap v1555: 103 pgs: 103 active+clean; 691 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 82 KiB/s rd, 6.7 KiB/s wr, 63 op/s 2023-10-09T04:32:23.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:23.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:22 smithi181 ceph-mon[196979]: osdmap e596: 8 total, 8 up, 6 in 2023-10-09T04:32:23.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:22 smithi181 ceph-mon[196979]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:22 smithi181 ceph-mon[196979]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:22 smithi181 ceph-mon[196979]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:22 smithi181 ceph-mon[196979]: failed to encode map e596 with expected crc 2023-10-09T04:32:23.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:22 smithi181 ceph-mon[196979]: pgmap v1555: 103 pgs: 103 active+clean; 691 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 82 KiB/s rd, 6.7 KiB/s wr, 63 op/s 2023-10-09T04:32:23.269 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:23.295 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:32:23.300 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T04:32:23.300 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep order 2023-10-09T04:32:23.300 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk '{print $2}' 2023-10-09T04:32:23.300 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 20 2023-10-09T04:32:23.337 INFO:tasks.workunit.client.0.smithi163.stdout:20 2023-10-09T04:32:23.338 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:23.338 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T04:32:23.422 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:23.425 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T04:32:23.430 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 conmon[213817]: 2023-10-09T04:32:23.394+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e597 definitely_dead 0 2023-10-09T04:32:23.519 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:23.522 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_667414/img testimg 2023-10-09T04:32:23.568 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:23.571 INFO:tasks.workunit.client.0.smithi163.stderr:++ rbd info testimg 2023-10-09T04:32:23.571 INFO:tasks.workunit.client.0.smithi163.stderr:++ grep features 2023-10-09T04:32:23.607 INFO:tasks.workunit.client.0.smithi163.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T04:32:23.607 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: ' 2023-10-09T04:32:23.607 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:23.710 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:23.712 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:23.713 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667414/img_v2 testimg_import 2023-10-09T04:32:23.767 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:32:23.771 INFO:tasks.workunit.client.0.smithi163.stderr:++ rbd info testimg_import 2023-10-09T04:32:23.771 INFO:tasks.workunit.client.0.smithi163.stderr:++ grep features 2023-10-09T04:32:23.806 INFO:tasks.workunit.client.0.smithi163.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T04:32:23.806 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: ' 2023-10-09T04:32:23.806 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' ' features: layering 2023-10-09T04:32:23.806 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: ' '!=' ' features: layering 2023-10-09T04:32:23.806 INFO:tasks.workunit.client.0.smithi163.stderr: op_features: ' ']' 2023-10-09T04:32:23.807 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:23.807 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T04:32:23.822 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:23.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:23.898 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.3. 2023-10-09T04:32:23.927 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:23.953 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: osdmap e597: 8 total, 8 up, 6 in 2023-10-09T04:32:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1293221457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/495873466' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: osdmap e597: 8 total, 8 up, 6 in 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1293221457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/495873466' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:32:24.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: osdmap e597: 8 total, 8 up, 6 in 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: failed to encode map e597 with expected crc 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1293221457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:24.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/495873466' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:32:24.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:32:24.968 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:24.971 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T04:32:25.084 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T04:32:25.100 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:25.103 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_667414/img testimg 2023-10-09T04:32:25.136 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: osdmap e598: 8 total, 8 up, 7 in 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[213841]: pgmap v1558: 103 pgs: 103 active+clean; 691 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 74 KiB/s rd, 10 KiB/s wr, 93 op/s 2023-10-09T04:32:25.137 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: osdmap e598: 8 total, 8 up, 7 in 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.138 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:24 smithi163 ceph-mon[221067]: pgmap v1558: 103 pgs: 103 active+clean; 691 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 74 KiB/s rd, 10 KiB/s wr, 93 op/s 2023-10-09T04:32:25.140 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:25.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:25.170 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:25.173 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:25.211 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:25.214 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667414/img_v2 testimg_import 2023-10-09T04:32:25.245 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:25.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: osdmap e598: 8 total, 8 up, 7 in 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: failed to encode map e598 with expected crc 2023-10-09T04:32:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:24 smithi181 ceph-mon[196979]: pgmap v1558: 103 pgs: 103 active+clean; 691 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 74 KiB/s rd, 10 KiB/s wr, 93 op/s 2023-10-09T04:32:25.277 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:25.281 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:32:25.286 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T04:32:25.286 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 'stripe unit' 2023-10-09T04:32:25.287 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T04:32:25.328 INFO:tasks.workunit.client.0.smithi163.stdout: stripe unit: 4 KiB 2023-10-09T04:32:25.329 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import 2023-10-09T04:32:25.329 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 'stripe count' 2023-10-09T04:32:25.329 INFO:tasks.workunit.client.0.smithi163.stderr:+ awk '{print $3}' 2023-10-09T04:32:25.330 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 1000 2023-10-09T04:32:25.367 INFO:tasks.workunit.client.0.smithi163.stdout:1000 2023-10-09T04:32:25.367 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/img_v2 2023-10-09T04:32:25.368 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T04:32:25.455 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T04:32:25.458 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T04:32:25.546 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T04:32:25.550 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_667414/img testimg 2023-10-09T04:32:25.602 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:25.602 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:25.605 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:25.608 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap create testimg@snap1 2023-10-09T04:32:26.009 INFO:tasks.workunit.client.0.smithi163.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:32:26.016 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap create testimg@snap2 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: osdmap e599: 8 total, 8 up, 7 in 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.030 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: osdmap e599: 8 total, 8 up, 7 in 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.031 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.032 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:25 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: osdmap e599: 8 total, 8 up, 7 in 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: failed to encode map e599 with expected crc 2023-10-09T04:32:26.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:25 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:26.281 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:26.385 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:26.413 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.457 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:26.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:26.591 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.633 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:26.738 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:26.768 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.893 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:26.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:26.912 INFO:tasks.workunit.client.0.smithi163.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T04:32:26.919 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap protect testimg@snap2 2023-10-09T04:32:26.955 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:32:26.962 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667414/snap_protect 2023-10-09T04:32:26.998 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:27.003 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:27.007 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667414/snap_protect testimg_import 2023-10-09T04:32:27.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[221067]: osdmap e600: 8 total, 8 up, 7 in 2023-10-09T04:32:27.026 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[221067]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[221067]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[221067]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[221067]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[221067]: pgmap v1561: 103 pgs: 9 remapped+peering, 94 active+clean; 574 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 314 KiB/s rd, 16 KiB/s wr, 410 op/s; 0 B/s, 2 objects/s recovering 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[213841]: osdmap e600: 8 total, 8 up, 7 in 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[213841]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[213841]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[213841]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[213841]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.027 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:26 smithi163 ceph-mon[213841]: pgmap v1561: 103 pgs: 9 remapped+peering, 94 active+clean; 574 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 314 KiB/s rd, 16 KiB/s wr, 410 op/s; 0 B/s, 2 objects/s recovering 2023-10-09T04:32:27.029 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:27.059 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:32:27.136 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:27.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:27.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:27.240 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:27.269 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:27.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:26 smithi181 ceph-mon[196979]: osdmap e600: 8 total, 8 up, 7 in 2023-10-09T04:32:27.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:26 smithi181 ceph-mon[196979]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:26 smithi181 ceph-mon[196979]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:26 smithi181 ceph-mon[196979]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:26 smithi181 ceph-mon[196979]: failed to encode map e600 with expected crc 2023-10-09T04:32:27.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:26 smithi181 ceph-mon[196979]: pgmap v1561: 103 pgs: 9 remapped+peering, 94 active+clean; 574 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 314 KiB/s rd, 16 KiB/s wr, 410 op/s; 0 B/s, 2 objects/s recovering 2023-10-09T04:32:27.321 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:27.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:27.425 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:27.426 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:27.458 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:27.468 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:27.559 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:27.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:27.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:27.663 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:27.692 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:27.854 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:27.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:27.908 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: osdmap e601: 8 total, 8 up, 7 in 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.909 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/49837205' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: osdmap e601: 8 total, 8 up, 7 in 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: failed to encode map e601 with expected crc 2023-10-09T04:32:27.910 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/49837205' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:27.959 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:27.989 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:28.035 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:28.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:28.140 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:28.167 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:28.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: failed to encode map e601 with expected crc 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: failed to encode map e601 with expected crc 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: osdmap e601: 8 total, 8 up, 7 in 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: failed to encode map e601 with expected crc 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: failed to encode map e601 with expected crc 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: failed to encode map e601 with expected crc 2023-10-09T04:32:28.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/49837205' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:28.940 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T04:32:28.944 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import@snap1 2023-10-09T04:32:28.944 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 'protected: False' 2023-10-09T04:32:28.986 INFO:tasks.workunit.client.0.smithi163.stdout: protected: False 2023-10-09T04:32:28.987 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd info testimg_import@snap2 2023-10-09T04:32:28.987 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep 'protected: True' 2023-10-09T04:32:29.025 INFO:tasks.workunit.client.0.smithi163.stdout: protected: True 2023-10-09T04:32:29.025 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/snap_protect 2023-10-09T04:32:29.026 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T04:32:29.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:29.055 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/49837205' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[221067]: pgmap v1564: 103 pgs: 3 active+undersized+remapped, 9 remapped+peering, 1 active+remapped, 90 active+clean; 464 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 365 KiB/s rd, 23 KiB/s wr, 489 op/s; 69 KiB/s, 4 objects/s recovering 2023-10-09T04:32:29.056 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/49837205' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.057 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:28 smithi163 ceph-mon[213841]: pgmap v1564: 103 pgs: 3 active+undersized+remapped, 9 remapped+peering, 1 active+remapped, 90 active+clean; 464 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 365 KiB/s rd, 23 KiB/s wr, 489 op/s; 69 KiB/s, 4 objects/s recovering 2023-10-09T04:32:29.069 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T04:32:29.112 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap purge testimg 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/49837205' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: failed to encode map e602 with expected crc 2023-10-09T04:32:29.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:28 smithi181 ceph-mon[196979]: pgmap v1564: 103 pgs: 3 active+undersized+remapped, 9 remapped+peering, 1 active+remapped, 90 active+clean; 464 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 365 KiB/s rd, 23 KiB/s wr, 489 op/s; 69 KiB/s, 4 objects/s recovering 2023-10-09T04:32:29.471 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:29.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:29.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:29.575 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:29.606 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:29.654 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:29.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:29.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:29.759 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:29.788 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:29.834 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:29.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:29.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:29.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:29.968 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:30.010 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:30.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:30.032 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:29 smithi163 ceph-mon[221067]: failed to encode map e603 with expected crc 2023-10-09T04:32:30.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:29 smithi163 ceph-mon[221067]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T04:32:30.033 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:29 smithi163 ceph-mon[221067]: failed to encode map e603 with expected crc 2023-10-09T04:32:30.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:29 smithi163 ceph-mon[213841]: failed to encode map e603 with expected crc 2023-10-09T04:32:30.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:29 smithi163 ceph-mon[213841]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T04:32:30.033 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:29 smithi163 ceph-mon[213841]: failed to encode map e603 with expected crc 2023-10-09T04:32:30.086 INFO:tasks.thrashosds.thrasher:in_osds: [1, 7, 0, 4, 2, 5, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:32:30.086 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:32:30.086 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:32:30.086 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-10-09T04:32:30.087 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0 2023-10-09T04:32:30.116 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:30.142 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:30.184 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:30.186 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:29 smithi181 ceph-mon[196979]: failed to encode map e603 with expected crc 2023-10-09T04:32:30.186 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:29 smithi181 ceph-mon[196979]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T04:32:30.187 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:29 smithi181 ceph-mon[196979]: failed to encode map e603 with expected crc 2023-10-09T04:32:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:30.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:30.215 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:30.215 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:30.289 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:30.322 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:30.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:32:30.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:32:30.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:32:30.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:32:30.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:32:30.703 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.704 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:32:30.712 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:32:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:32:30] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.43.0" 2023-10-09T04:32:30.933 INFO:tasks.workunit.client.0.smithi163.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T04:32:30.943 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd snap purge testimg_import 2023-10-09T04:32:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[221067]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T04:32:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[221067]: failed to encode map e604 with expected crc 2023-10-09T04:32:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[221067]: failed to encode map e604 with expected crc 2023-10-09T04:32:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[221067]: pgmap v1567: 103 pgs: 3 active+undersized+remapped, 9 remapped+peering, 1 active+remapped, 90 active+clean; 464 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 51 KiB/s rd, 6.5 KiB/s wr, 79 op/s; 80 KiB/s, 2 objects/s recovering 2023-10-09T04:32:31.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/139404187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:31.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[213841]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T04:32:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[213841]: failed to encode map e604 with expected crc 2023-10-09T04:32:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[213841]: failed to encode map e604 with expected crc 2023-10-09T04:32:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[213841]: pgmap v1567: 103 pgs: 3 active+undersized+remapped, 9 remapped+peering, 1 active+remapped, 90 active+clean; 464 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 51 KiB/s rd, 6.5 KiB/s wr, 79 op/s; 80 KiB/s, 2 objects/s recovering 2023-10-09T04:32:31.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/139404187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:31.118 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:31.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:31.222 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:31.249 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:30 smithi181 ceph-mon[196979]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T04:32:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:30 smithi181 ceph-mon[196979]: failed to encode map e604 with expected crc 2023-10-09T04:32:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:30 smithi181 ceph-mon[196979]: failed to encode map e604 with expected crc 2023-10-09T04:32:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:30 smithi181 ceph-mon[196979]: pgmap v1567: 103 pgs: 3 active+undersized+remapped, 9 remapped+peering, 1 active+remapped, 90 active+clean; 464 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 51 KiB/s rd, 6.5 KiB/s wr, 79 op/s; 80 KiB/s, 2 objects/s recovering 2023-10-09T04:32:31.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/139404187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:31.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:32:31.394 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:31.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:31.498 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:31.524 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.626 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:31.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:31.752 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:31.762 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.775 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:32:31.810 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:31.862 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:31.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:31.949 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.992 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:31 smithi181 ceph-mon[196979]: 1.18 scrub starts 2023-10-09T04:32:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:31 smithi181 ceph-mon[196979]: 1.18 scrub ok 2023-10-09T04:32:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:31 smithi181 ceph-mon[196979]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:31 smithi181 ceph-mon[196979]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:31 smithi181 ceph-mon[196979]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T04:32:32.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:31 smithi181 ceph-mon[196979]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[221067]: 1.18 scrub starts 2023-10-09T04:32:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[221067]: 1.18 scrub ok 2023-10-09T04:32:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[221067]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[221067]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[221067]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T04:32:32.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[221067]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[213841]: 1.18 scrub starts 2023-10-09T04:32:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[213841]: 1.18 scrub ok 2023-10-09T04:32:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[213841]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[213841]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[213841]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T04:32:32.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:31 smithi163 ceph-mon[213841]: failed to encode map e605 with expected crc 2023-10-09T04:32:32.097 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:32.128 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:32.232 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:32.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:32.338 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:32.366 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:32.842 INFO:tasks.workunit.client.0.smithi163.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T04:32:32.849 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg 2023-10-09T04:32:32.952 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:32.955 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd remove testimg_import 2023-10-09T04:32:33.042 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:33.045 INFO:tasks.workunit.client.0.smithi163.stderr:+ tiered=0 2023-10-09T04:32:33.046 INFO:tasks.workunit.client.0.smithi163.stderr:+ ceph osd dump 2023-10-09T04:32:33.046 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep '^pool' 2023-10-09T04:32:33.046 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep ''\''rbd'\''' 2023-10-09T04:32:33.046 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep tier 2023-10-09T04:32:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:32 smithi181 ceph-mon[196979]: failed to encode map e606 with expected crc 2023-10-09T04:32:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:32 smithi181 ceph-mon[196979]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T04:32:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:32 smithi181 ceph-mon[196979]: failed to encode map e606 with expected crc 2023-10-09T04:32:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:32 smithi181 ceph-mon[196979]: pgmap v1570: 103 pgs: 103 active+clean; 674 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 182 KiB/s rd, 17 KiB/s wr, 240 op/s 2023-10-09T04:32:33.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:32 smithi181 ceph-mon[196979]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T04:32:33.322 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:33.323 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[221067]: failed to encode map e606 with expected crc 2023-10-09T04:32:33.323 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[221067]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T04:32:33.323 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[221067]: failed to encode map e606 with expected crc 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[221067]: pgmap v1570: 103 pgs: 103 active+clean; 674 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 182 KiB/s rd, 17 KiB/s wr, 240 op/s 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[221067]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[213841]: failed to encode map e606 with expected crc 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[213841]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[213841]: failed to encode map e606 with expected crc 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[213841]: pgmap v1570: 103 pgs: 103 active+clean; 674 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 182 KiB/s rd, 17 KiB/s wr, 240 op/s 2023-10-09T04:32:33.324 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:32 smithi163 ceph-mon[213841]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T04:32:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:33.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:33.426 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:33.461 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:33.527 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:33.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:33.532 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:33.621 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:32:33.631 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:33.697 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:33.808 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_667414/sparse1 2023-10-09T04:32:33.810 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:33.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:33.815 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T04:32:33.815 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T04:32:33.815 INFO:tasks.workunit.client.0.smithi163.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0051878 s, 202 MB/s 2023-10-09T04:32:33.816 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_667414/sparse2 2023-10-09T04:32:33.821 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T04:32:33.822 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T04:32:33.822 INFO:tasks.workunit.client.0.smithi163.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00520934 s, 201 MB/s 2023-10-09T04:32:33.822 INFO:tasks.workunit.client.0.smithi163.stderr:+ truncate /tmp/rbd_import_export_667414/sparse2 -s 2M 2023-10-09T04:32:33.873 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse1 2023-10-09T04:32:33.915 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:33.917 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 0% complete...failed. 2023-10-09T04:32:33.917 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:32:33.921 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T04:32:33.921 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_667414/sparse1 2023-10-09T04:32:33.935 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/664841906' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/664841906' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.936 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: failed to encode map e607 with expected crc 2023-10-09T04:32:33.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:32:33.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/664841906' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:32:33.937 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/664841906' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:32:33.963 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:33.975 INFO:teuthology.orchestra.run.smithi163.stdout:{ 2023-10-09T04:32:33.975 INFO:teuthology.orchestra.run.smithi163.stdout: "ops": [], 2023-10-09T04:32:33.975 INFO:teuthology.orchestra.run.smithi163.stdout: "num_ops": 0 2023-10-09T04:32:33.975 INFO:teuthology.orchestra.run.smithi163.stdout:} 2023-10-09T04:32:34.012 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T04:32:34.012 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:34.013 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:32:34.015 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T04:32:34.016 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep sparse1 2023-10-09T04:32:34.016 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T04:32:34.058 INFO:tasks.workunit.client.0.smithi163.stdout:sparse1 2 MiB 2 2023-10-09T04:32:34.058 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse1 2023-10-09T04:32:34.058 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse1 2023-10-09T04:32:34.059 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse1 2023-10-09T04:32:34.059 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:34.059 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:34.084 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:34.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:34.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:34.104 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.b8f4dd960e53 2023-10-09T04:32:34.106 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.b8f4dd960e53 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse1 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse1 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.b8f4dd960e53\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:34.107 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse1 2023-10-09T04:32:34.143 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:34.144 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:34.144 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:34.144 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:34.144 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:34.179 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects=1 2023-10-09T04:32:34.179 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 1 2023-10-09T04:32:34.179 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T04:32:34.179 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse1 /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:34.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:34.214 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: failed to encode map e607 with expected crc 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: failed to encode map e607 with expected crc 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: failed to encode map e607 with expected crc 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: failed to encode map e607 with expected crc 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/664841906' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:32:34.216 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/664841906' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:32:34.218 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667414/sparse1 /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:34.218 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_667414/sparse1 /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:34.218 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:34.220 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse1 --format %b 2023-10-09T04:32:34.221 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=2048 2023-10-09T04:32:34.222 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse1.out --format %b 2023-10-09T04:32:34.222 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=2048 2023-10-09T04:32:34.223 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:34.223 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:34.223 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:32:34.223 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:34.223 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse1 2023-10-09T04:32:34.265 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:34.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:34.312 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T04:32:34.315 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse2 2023-10-09T04:32:34.357 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 0% complete...failed. 2023-10-09T04:32:34.357 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T04:32:34.360 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T04:32:34.360 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_667414/sparse2 2023-10-09T04:32:34.370 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:34.401 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:34.425 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T04:32:34.426 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:34.426 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:32:34.429 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T04:32:34.429 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep sparse2 2023-10-09T04:32:34.429 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T04:32:34.468 INFO:tasks.workunit.client.0.smithi163.stdout:sparse2 2 MiB 2 2023-10-09T04:32:34.469 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse2 2023-10-09T04:32:34.469 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse2 2023-10-09T04:32:34.469 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse2 2023-10-09T04:32:34.469 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:34.469 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:34.502 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:34.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:34.507 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.b92d44942825 2023-10-09T04:32:34.508 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.b92d44942825 2023-10-09T04:32:34.508 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse2 2023-10-09T04:32:34.508 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.b92d44942825\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:34.508 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse2 2023-10-09T04:32:34.508 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:34.509 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse2 2023-10-09T04:32:34.509 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:34.509 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:34.547 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:34.547 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:34.547 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:34.547 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:34.547 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:34.578 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects=0 2023-10-09T04:32:34.578 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 0 2023-10-09T04:32:34.579 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T04:32:34.579 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse2 /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:34.605 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:34.618 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:34.621 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667414/sparse2 /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:34.622 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_667414/sparse2 /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:34.624 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse2 --format %b 2023-10-09T04:32:34.624 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=2048 2023-10-09T04:32:34.624 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse2.out --format %b 2023-10-09T04:32:34.626 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=2048 2023-10-09T04:32:34.626 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:34.626 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:34.626 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:32:34.626 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:34.627 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse2 2023-10-09T04:32:34.633 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:34.724 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T04:32:34.726 INFO:tasks.workunit.client.0.smithi163.stderr:+ truncate /tmp/rbd_import_export_667414/sparse1 -s 10M 2023-10-09T04:32:34.727 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T04:32:34.811 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:34.812 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:34.812 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:32:34.814 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T04:32:34.815 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep sparse1 2023-10-09T04:32:34.815 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T04:32:34.850 INFO:tasks.workunit.client.0.smithi163.stdout:sparse1 10 MiB 2 2023-10-09T04:32:34.851 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse1 2023-10-09T04:32:34.851 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse1 2023-10-09T04:32:34.851 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse1 2023-10-09T04:32:34.851 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:34.851 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:34.887 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.929424ddbd99 2023-10-09T04:32:34.888 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.929424ddbd99 2023-10-09T04:32:34.888 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse1 2023-10-09T04:32:34.888 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse1 2023-10-09T04:32:34.888 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.929424ddbd99\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:34.888 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:34.889 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse1 2023-10-09T04:32:34.889 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:34.889 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:34.924 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:34.924 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:34.924 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:34.924 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:34.924 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:34.951 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.5 primary-affinity to 0 (802) 2023-10-09T04:32:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:34 smithi181 ceph-mon[196979]: 1.7 scrub starts 2023-10-09T04:32:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:34 smithi181 ceph-mon[196979]: 1.7 scrub ok 2023-10-09T04:32:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:34 smithi181 ceph-mon[196979]: pgmap v1572: 103 pgs: 103 active+clean; 674 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 173 KiB/s rd, 16 KiB/s wr, 228 op/s 2023-10-09T04:32:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3134000023' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T04:32:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:34 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T04:32:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[221067]: 1.7 scrub starts 2023-10-09T04:32:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[221067]: 1.7 scrub ok 2023-10-09T04:32:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[221067]: pgmap v1572: 103 pgs: 103 active+clean; 674 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 173 KiB/s rd, 16 KiB/s wr, 228 op/s 2023-10-09T04:32:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3134000023' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T04:32:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T04:32:35.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[213841]: 1.7 scrub starts 2023-10-09T04:32:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[213841]: 1.7 scrub ok 2023-10-09T04:32:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[213841]: pgmap v1572: 103 pgs: 103 active+clean; 674 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 173 KiB/s rd, 16 KiB/s wr, 228 op/s 2023-10-09T04:32:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3134000023' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T04:32:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-10-09T04:32:35.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:35.545 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:35.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:35.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:35.678 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:35.725 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:35.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:35.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:35.858 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:35.903 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:35.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:35.968 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects=1 2023-10-09T04:32:35.968 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 1 2023-10-09T04:32:35.968 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T04:32:35.968 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse1 /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:36.005 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:36.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:36.009 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667414/sparse1 /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:36.009 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_667414/sparse1 /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:36.016 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse1 --format %b 2023-10-09T04:32:36.018 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=2048 2023-10-09T04:32:36.018 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse1.out --format %b 2023-10-09T04:32:36.019 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=2048 2023-10-09T04:32:36.019 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:36.019 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:36.019 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:32:36.019 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/sparse1.out 2023-10-09T04:32:36.020 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse1 2023-10-09T04:32:36.033 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.078 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:36.105 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:32:36.109 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T04:32:36.112 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_667414/sparse2 oflag=append conv=notrunc 2023-10-09T04:32:36.123 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T04:32:36.123 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T04:32:36.124 INFO:tasks.workunit.client.0.smithi163.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0104585 s, 201 MB/s 2023-10-09T04:32:36.124 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T04:32:36.183 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:36.185 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:36.208 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:36.208 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:36.209 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:32:36.212 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd ls -l 2023-10-09T04:32:36.212 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep sparse2 2023-10-09T04:32:36.212 INFO:tasks.workunit.client.0.smithi163.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[213841]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.220 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:35 smithi163 ceph-mon[221067]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.221 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.258 INFO:tasks.workunit.client.0.smithi163.stdout:sparse2 4 MiB 2 2023-10-09T04:32:36.259 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse2 2023-10-09T04:32:36.259 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse2 2023-10-09T04:32:36.259 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse2 2023-10-09T04:32:36.260 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:36.260 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-10-09T04:32:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T04:32:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:35 smithi181 ceph-mon[196979]: failed to encode map e608 with expected crc 2023-10-09T04:32:36.301 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.92bea84b72d3 2023-10-09T04:32:36.302 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.92bea84b72d3 2023-10-09T04:32:36.303 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.92bea84b72d3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:36.303 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:36.305 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse2 2023-10-09T04:32:36.306 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse2 2023-10-09T04:32:36.306 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse2 2023-10-09T04:32:36.306 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:36.306 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:36.327 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:36.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:36.351 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:36.351 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:36.351 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:36.351 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:36.351 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:36.386 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects='0 2023-10-09T04:32:36.386 INFO:tasks.workunit.client.0.smithi163.stderr:2 2023-10-09T04:32:36.386 INFO:tasks.workunit.client.0.smithi163.stderr:3' 2023-10-09T04:32:36.386 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 0 2 3 2023-10-09T04:32:36.386 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T04:32:36.386 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse2 /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:36.423 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T04:32:36.428 INFO:tasks.workunit.client.0.smithi163.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667414/sparse2 /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:36.428 INFO:tasks.workunit.client.0.smithi163.stderr:+ cmp -l /tmp/rbd_import_export_667414/sparse2 /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:36.431 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse2 --format %b 2023-10-09T04:32:36.432 INFO:tasks.workunit.client.0.smithi163.stderr:+ origsize=6144 2023-10-09T04:32:36.432 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse2.out --format %b 2023-10-09T04:32:36.432 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:36.433 INFO:tasks.workunit.client.0.smithi163.stderr:+ exportsize=6144 2023-10-09T04:32:36.434 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:36.434 INFO:tasks.workunit.client.0.smithi163.stderr:+ difference=0 2023-10-09T04:32:36.434 INFO:tasks.workunit.client.0.smithi163.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T04:32:36.434 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/sparse2.out 2023-10-09T04:32:36.434 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse2 2023-10-09T04:32:36.462 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.518 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:36.554 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T04:32:36.558 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T04:32:36.558 INFO:tasks.workunit.client.0.smithi163.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T04:32:36.558 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_667414/sparse1 sparse 2023-10-09T04:32:36.623 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:36.631 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T04:32:36.632 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:36.632 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T04:32:36.636 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse 2023-10-09T04:32:36.636 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse 2023-10-09T04:32:36.637 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse 2023-10-09T04:32:36.637 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:36.637 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:36.651 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.686 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.b9b7fe3a7d52 2023-10-09T04:32:36.686 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.b9b7fe3a7d52 2023-10-09T04:32:36.686 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse 2023-10-09T04:32:36.687 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse 2023-10-09T04:32:36.687 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:36.687 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.b9b7fe3a7d52\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:36.687 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:36.687 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:36.687 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse 2023-10-09T04:32:36.698 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:36.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:36.723 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:36.723 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:36.723 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:36.723 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:36.724 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:36.760 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects=1 2023-10-09T04:32:36.760 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 1 2023-10-09T04:32:36.760 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T04:32:36.760 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse 2023-10-09T04:32:36.806 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:36.837 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.862 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T04:32:36.866 INFO:tasks.workunit.client.0.smithi163.stdout:zeros import through stdin to sparse image 2023-10-09T04:32:36.867 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T04:32:36.867 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T04:32:36.867 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd import --new-format - sparse 2023-10-09T04:32:36.920 INFO:tasks.workunit.client.0.smithi163.stderr:4+0 records in 2023-10-09T04:32:36.920 INFO:tasks.workunit.client.0.smithi163.stderr:4+0 records out 2023-10-09T04:32:36.920 INFO:tasks.workunit.client.0.smithi163.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0522061 s, 80.3 MB/s 2023-10-09T04:32:36.929 INFO:tasks.workunit.client.0.smithi163.stderr: Importing image: 100% complete...done. 2023-10-09T04:32:36.929 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:36.933 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse 2023-10-09T04:32:36.933 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse 2023-10-09T04:32:36.933 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse 2023-10-09T04:32:36.933 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:36.933 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:36.960 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:32:36.988 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.930fdc543106 2023-10-09T04:32:36.989 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.930fdc543106 2023-10-09T04:32:36.989 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.930fdc543106\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:36.989 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse 2023-10-09T04:32:36.989 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse 2023-10-09T04:32:36.989 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse 2023-10-09T04:32:36.990 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:36.990 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:36.990 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:36.993 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:37.032 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:37.032 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:37.032 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:37.032 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:37.032 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: osdmap e609: 8 total, 8 up, 7 in 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: pgmap v1575: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 405 KiB/s rd, 255 KiB/s wr, 217 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3048540140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: osdmap e609: 8 total, 8 up, 7 in 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: pgmap v1575: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 405 KiB/s rd, 255 KiB/s wr, 217 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T04:32:37.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:37.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3048540140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:37.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:37.101 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects= 2023-10-09T04:32:37.101 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 2023-10-09T04:32:37.101 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T04:32:37.101 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse 2023-10-09T04:32:37.124 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:37.173 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:37.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:37.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:37.188 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:37.192 INFO:tasks.workunit.client.0.smithi163.stdout:zeros export to sparse file 2023-10-09T04:32:37.192 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo 'zeros export to sparse file' 2023-10-09T04:32:37.192 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T04:32:37.237 INFO:tasks.workunit.client.0.smithi163.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T04:32:37.240 INFO:tasks.workunit.client.0.smithi163.stderr:++ rbd info sparse 2023-10-09T04:32:37.240 INFO:tasks.workunit.client.0.smithi163.stderr:++ grep block_name_prefix 2023-10-09T04:32:37.240 INFO:tasks.workunit.client.0.smithi163.stderr:++ awk '{print $NF;}' 2023-10-09T04:32:37.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: osdmap e609: 8 total, 8 up, 7 in 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: failed to encode map e609 with expected crc 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: pgmap v1575: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 405 KiB/s rd, 255 KiB/s wr, 217 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3048540140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:37.276 INFO:tasks.workunit.client.0.smithi163.stderr:+ prefix=rbd_data.933089f2bddc 2023-10-09T04:32:37.276 INFO:tasks.workunit.client.0.smithi163.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T04:32:37.277 INFO:tasks.workunit.client.0.smithi163.stderr:++ get_image_data_pool sparse 2023-10-09T04:32:37.277 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse 2023-10-09T04:32:37.277 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse 2023-10-09T04:32:37.277 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep 'data_pool: ' 2023-10-09T04:32:37.277 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T04:32:37.278 DEBUG:teuthology.orchestra.run.smithi163:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:37.306 INFO:teuthology.orchestra.run.smithi163.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:37.314 INFO:tasks.workunit.client.0.smithi163.stderr:++ data_pool= 2023-10-09T04:32:37.315 INFO:tasks.workunit.client.0.smithi163.stderr:++ '[' -z ']' 2023-10-09T04:32:37.315 INFO:tasks.workunit.client.0.smithi163.stderr:++ data_pool=rbd 2023-10-09T04:32:37.315 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo rbd 2023-10-09T04:32:37.315 INFO:tasks.workunit.client.0.smithi163.stderr:+ rados -p rbd put rbd_data.933089f2bddc.000000000000 - 2023-10-09T04:32:37.348 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records in 2023-10-09T04:32:37.348 INFO:tasks.workunit.client.0.smithi163.stderr:1+0 records out 2023-10-09T04:32:37.348 INFO:tasks.workunit.client.0.smithi163.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0707012 s, 59.3 MB/s 2023-10-09T04:32:37.373 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects sparse 2023-10-09T04:32:37.373 INFO:tasks.workunit.client.0.smithi163.stderr:++ image=sparse 2023-10-09T04:32:37.373 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rbd info sparse 2023-10-09T04:32:37.374 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep block_name_prefix 2023-10-09T04:32:37.374 INFO:tasks.workunit.client.0.smithi163.stderr:+++ awk '{print $NF;}' 2023-10-09T04:32:37.409 INFO:tasks.workunit.client.0.smithi163.stderr:++ prefix=rbd_data.933089f2bddc 2023-10-09T04:32:37.410 INFO:tasks.workunit.client.0.smithi163.stderr:+++ grep rbd_data.933089f2bddc 2023-10-09T04:32:37.410 INFO:tasks.workunit.client.0.smithi163.stderr:++++ get_image_data_pool sparse 2023-10-09T04:32:37.411 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sed -e 's/rbd_data.933089f2bddc\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T04:32:37.411 INFO:tasks.workunit.client.0.smithi163.stderr:++++ image=sparse 2023-10-09T04:32:37.411 INFO:tasks.workunit.client.0.smithi163.stderr:+++ sort -u 2023-10-09T04:32:37.411 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ rbd info sparse 2023-10-09T04:32:37.411 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ grep 'data_pool: ' 2023-10-09T04:32:37.411 INFO:tasks.workunit.client.0.smithi163.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T04:32:37.447 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool= 2023-10-09T04:32:37.447 INFO:tasks.workunit.client.0.smithi163.stderr:++++ '[' -z ']' 2023-10-09T04:32:37.447 INFO:tasks.workunit.client.0.smithi163.stderr:++++ data_pool=rbd 2023-10-09T04:32:37.447 INFO:tasks.workunit.client.0.smithi163.stderr:++++ echo rbd 2023-10-09T04:32:37.448 INFO:tasks.workunit.client.0.smithi163.stderr:+++ rados ls -p rbd 2023-10-09T04:32:37.479 INFO:tasks.workunit.client.0.smithi163.stderr:++ objects=0 2023-10-09T04:32:37.480 INFO:tasks.workunit.client.0.smithi163.stderr:++ echo 0 2023-10-09T04:32:37.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T04:32:37.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/sparse 2023-10-09T04:32:37.480 INFO:tasks.workunit.client.0.smithi163.stderr:rm: cannot remove '/tmp/rbd_import_export_667414/sparse': No such file or directory 2023-10-09T04:32:37.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T04:32:37.480 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd export sparse /tmp/rbd_import_export_667414/sparse 2023-10-09T04:32:37.515 INFO:tasks.workunit.client.0.smithi163.stderr: Exporting image: 100% complete...done. 2023-10-09T04:32:37.519 INFO:tasks.workunit.client.0.smithi163.stderr:++ stat /tmp/rbd_import_export_667414/sparse --format=%b 2023-10-09T04:32:37.519 INFO:tasks.workunit.client.0.smithi163.stderr:+ '[' 0 = 0 ']' 2023-10-09T04:32:37.520 INFO:tasks.workunit.client.0.smithi163.stderr:+ rbd rm sparse 2023-10-09T04:32:37.604 INFO:tasks.workunit.client.0.smithi163.stderr: Removing image: 100% complete...done. 2023-10-09T04:32:37.607 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm /tmp/rbd_import_export_667414/sparse /tmp/rbd_import_export_667414/sparse1 /tmp/rbd_import_export_667414/sparse2 /tmp/rbd_import_export_667414/sparse3 2023-10-09T04:32:37.609 INFO:tasks.workunit.client.0.smithi163.stderr:rm: cannot remove '/tmp/rbd_import_export_667414/sparse3': No such file or directory 2023-10-09T04:32:37.610 INFO:tasks.workunit.client.0.smithi163.stderr:+ true 2023-10-09T04:32:37.610 INFO:tasks.workunit.client.0.smithi163.stdout:OK 2023-10-09T04:32:37.611 INFO:tasks.workunit.client.0.smithi163.stderr:+ echo OK 2023-10-09T04:32:37.611 INFO:tasks.workunit.client.0.smithi163.stderr:+ rm -rf /tmp/rbd_import_export_667414 2023-10-09T04:32:37.612 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T04:32:37.612 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T04:32:37.691 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T04:32:37.692 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T04:32:38.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:32:38.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:38.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:32:38.250 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3048540140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: osdmap e610: 8 total, 8 up, 7 in 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[213841]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:32:38.251 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3048540140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:38.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: osdmap e610: 8 total, 8 up, 7 in 2023-10-09T04:32:38.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.252 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:37 smithi163 ceph-mon[221067]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T04:32:38.267 DEBUG:teuthology.parallel:result is None 2023-10-09T04:32:38.268 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3048540140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: osdmap e610: 8 total, 8 up, 7 in 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: failed to encode map e610 with expected crc 2023-10-09T04:32:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:37 smithi181 ceph-mon[196979]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T04:32:38.291 INFO:teuthology.orchestra.run.smithi163.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:38.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi163 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:38.306 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:32:38.307 DEBUG:teuthology.orchestra.run.smithi163:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:32:38.335 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:38.358 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:32:38.358 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:32:38.358 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T04:32:38.358 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T04:32:38.359 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T04:32:38.359 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T04:32:38.359 DEBUG:teuthology.orchestra.run.smithi163:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:32:38.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:32:38.416 INFO:teuthology.orchestra.run.smithi163.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T04:32:38.417 DEBUG:teuthology.orchestra.run.smithi163:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T04:32:38.426 INFO:teuthology.orchestra.run.smithi181.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:38.468 INFO:teuthology.orchestra.run.smithi181.stderr:Error: no container with name or ID ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:38.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:38.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi181 with status 125: 'sudo docker kill -s 1 ceph-8628d772-6656-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:38.477 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T04:32:38.477 DEBUG:teuthology.orchestra.run.smithi163:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T04:32:38.536 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T04:32:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:38 smithi181 ceph-mon[196979]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:38 smithi181 ceph-mon[196979]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:38 smithi181 ceph-mon[196979]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:38 smithi181 ceph-mon[196979]: pgmap v1578: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 366 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T04:32:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:38 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1650572803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[213841]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[213841]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[213841]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[213841]: pgmap v1578: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 366 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1650572803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[221067]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[221067]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[221067]: failed to encode map e611 with expected crc 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[221067]: pgmap v1578: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 366 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T04:32:39.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:38 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1650572803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:39.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.0: {} 2023-10-09T04:32:39.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.1: {} 2023-10-09T04:32:39.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.2: {} 2023-10-09T04:32:39.642 INFO:teuthology.orchestra.run.smithi163.stdout:osd.3: {} 2023-10-09T04:32:39.643 INFO:teuthology.orchestra.run.smithi163.stdout:osd.4: {} 2023-10-09T04:32:39.643 INFO:teuthology.orchestra.run.smithi163.stdout:osd.5: {} 2023-10-09T04:32:39.643 INFO:teuthology.orchestra.run.smithi163.stdout:osd.6: {} 2023-10-09T04:32:39.643 INFO:teuthology.orchestra.run.smithi163.stdout:osd.7: {} 2023-10-09T04:32:39.644 INFO:teuthology.orchestra.run.smithi163.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.644 INFO:teuthology.orchestra.run.smithi163.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.644 INFO:teuthology.orchestra.run.smithi163.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.644 INFO:teuthology.orchestra.run.smithi163.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.644 INFO:teuthology.orchestra.run.smithi163.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.645 INFO:teuthology.orchestra.run.smithi163.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.645 INFO:teuthology.orchestra.run.smithi163.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:32:39.645 INFO:teuthology.orchestra.run.smithi163.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:32:40.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:39 smithi163 ceph-mon[213841]: failed to encode map e611 with expected crc 2023-10-09T04:32:40.111 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:40.111 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:39 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:40.111 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:39 smithi163 ceph-mon[221067]: failed to encode map e611 with expected crc 2023-10-09T04:32:40.112 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:40.112 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:39 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:39 smithi181 ceph-mon[196979]: failed to encode map e611 with expected crc 2023-10-09T04:32:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:39 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:40.310 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:40.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:32:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:32:40] "GET /metrics HTTP/1.1" 200 34224 "" "Prometheus/2.43.0" 2023-10-09T04:32:41.137 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T04:32:41.137 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T04:32:41.137 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T04:32:41.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:41 smithi163 ceph-mon[221067]: pgmap v1579: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 991 KiB/s rd, 1001 KiB/s wr, 281 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:32:41.443 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:41 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:41.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:41 smithi163 ceph-mon[213841]: pgmap v1579: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 991 KiB/s rd, 1001 KiB/s wr, 281 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:32:41.443 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:41 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:41.464 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:32:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:41 smithi181 ceph-mon[196979]: pgmap v1579: 103 pgs: 14 peering, 89 active+clean; 582 KiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 991 KiB/s rd, 1001 KiB/s wr, 281 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:32:41.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:41 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:42.496 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:42.650 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:42 smithi181 ceph-mon[196979]: pgmap v1580: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.0 MiB/s rd, 1.8 MiB/s wr, 295 op/s 2023-10-09T04:32:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:42 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:32:42.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:42 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:32:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:42 smithi163 ceph-mon[213841]: pgmap v1580: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.0 MiB/s rd, 1.8 MiB/s wr, 295 op/s 2023-10-09T04:32:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:42 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:32:42.832 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:42 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:32:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:42 smithi163 ceph-mon[221067]: pgmap v1580: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 1.0 MiB/s rd, 1.8 MiB/s wr, 295 op/s 2023-10-09T04:32:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:42 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:32:42.832 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:42 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:32:43.865 INFO:teuthology.orchestra.run.smithi163.stderr:noscrub is unset 2023-10-09T04:32:44.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:43 smithi163 ceph-mon[221067]: failed to encode map e612 with expected crc 2023-10-09T04:32:44.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:43 smithi163 ceph-mon[221067]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T04:32:44.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:43 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4200327783' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:44.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:43 smithi163 ceph-mon[213841]: failed to encode map e612 with expected crc 2023-10-09T04:32:44.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:43 smithi163 ceph-mon[213841]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T04:32:44.095 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:43 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4200327783' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:43 smithi181 ceph-mon[196979]: failed to encode map e612 with expected crc 2023-10-09T04:32:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:43 smithi181 ceph-mon[196979]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T04:32:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:43 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4200327783' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:45.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[221067]: failed to encode map e613 with expected crc 2023-10-09T04:32:45.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4200327783' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:45.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[221067]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T04:32:45.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[221067]: failed to encode map e613 with expected crc 2023-10-09T04:32:45.265 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[221067]: pgmap v1583: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 312 KiB/s rd, 980 KiB/s wr, 194 op/s 2023-10-09T04:32:45.265 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[213841]: failed to encode map e613 with expected crc 2023-10-09T04:32:45.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4200327783' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:45.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[213841]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T04:32:45.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[213841]: failed to encode map e613 with expected crc 2023-10-09T04:32:45.266 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:44 smithi163 ceph-mon[213841]: pgmap v1583: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 312 KiB/s rd, 980 KiB/s wr, 194 op/s 2023-10-09T04:32:45.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:44 smithi181 ceph-mon[196979]: failed to encode map e613 with expected crc 2023-10-09T04:32:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:44 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4200327783' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:44 smithi181 ceph-mon[196979]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T04:32:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:44 smithi181 ceph-mon[196979]: failed to encode map e613 with expected crc 2023-10-09T04:32:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:44 smithi181 ceph-mon[196979]: pgmap v1583: 103 pgs: 103 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 312 KiB/s rd, 980 KiB/s wr, 194 op/s 2023-10-09T04:32:45.441 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:32:45.899 INFO:teuthology.orchestra.run.smithi163.stderr:marked in osd.6. 2023-10-09T04:32:45.982 INFO:teuthology.orchestra.run.smithi163.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:45.983 INFO:teuthology.orchestra.run.smithi163.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:46.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:45 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/450616729' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:32:46.009 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:45 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:32:46.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:45 smithi163 conmon[213817]: 2023-10-09T04:32:45.760+0000 7fe16b5bb700 -1 mon.a@0(leader).osd e613 definitely_dead 0 2023-10-09T04:32:46.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:45 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/450616729' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:32:46.009 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:45 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:32:46.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:45 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/450616729' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:32:46.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:45 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: pgmap v1585: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 328 KiB/s rd, 1.0 MiB/s wr, 203 op/s 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T04:32:47.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: pgmap v1585: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 328 KiB/s rd, 1.0 MiB/s wr, 203 op/s 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:47.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:47.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:32:47.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:32:47.168 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: osdmap e614: 8 total, 8 up, 8 in 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: failed to encode map e614 with expected crc 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: pgmap v1585: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 328 KiB/s rd, 1.0 MiB/s wr, 203 op/s 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:47.169 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:47.206 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T04:32:47.206 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T04:32:47.206 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 71 2023-10-09T04:32:47.244 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:47.485 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:47.864 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T04:32:48.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:48.020 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: osdmap e615: 8 total, 8 up, 8 in 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: failed to encode map e615 with expected crc 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:48.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:49.195 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.196 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:48 smithi181 ceph-mon[196979]: pgmap v1588: 103 pgs: 8 peering, 95 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[213841]: pgmap v1588: 103 pgs: 8 peering, 95 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: osdmap e616: 8 total, 8 up, 8 in 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: failed to encode map e616 with expected crc 2023-10-09T04:32:49.334 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:48 smithi163 ceph-mon[221067]: pgmap v1588: 103 pgs: 8 peering, 95 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:32:49.406 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:49.406 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:50.051 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.082 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:50.084 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:49 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:50.219 INFO:teuthology.orchestra.run.smithi163.stderr:Error EINVAL: specified pgp_num 71 > pg_num 31 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: osdmap e617: 8 total, 8 up, 8 in 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: failed to encode map e617 with expected crc 2023-10-09T04:32:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:50.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:49 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:50.435 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:50.896 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:32:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:32:50] "GET /metrics HTTP/1.1" 200 34228 "" "Prometheus/2.43.0" 2023-10-09T04:32:50.909 INFO:teuthology.orchestra.run.smithi163.stderr:nodeep-scrub is unset 2023-10-09T04:32:50.919 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:32:50.920 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4218374699' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:51.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:51.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: pgmap v1591: 103 pgs: 8 peering, 95 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:32:51.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1194415324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-10-09T04:32:51.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4218374699' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: pgmap v1591: 103 pgs: 8 peering, 95 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1194415324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: osdmap e618: 8 total, 8 up, 8 in 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: failed to encode map e618 with expected crc 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4218374699' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: pgmap v1591: 103 pgs: 8 peering, 95 active+clean; 4.6 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1194415324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-10-09T04:32:51.334 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:32:51.550 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:51.904 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:51.904 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:51 smithi181 ceph-mon[196979]: failed to encode map e619 with expected crc 2023-10-09T04:32:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:51 smithi181 ceph-mon[196979]: failed to encode map e619 with expected crc 2023-10-09T04:32:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:51 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:52.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:51 smithi181 ceph-mon[196979]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T04:32:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[213841]: failed to encode map e619 with expected crc 2023-10-09T04:32:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[213841]: failed to encode map e619 with expected crc 2023-10-09T04:32:52.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:52.083 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[213841]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T04:32:52.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[221067]: failed to encode map e619 with expected crc 2023-10-09T04:32:52.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[221067]: failed to encode map e619 with expected crc 2023-10-09T04:32:52.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:52.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:51 smithi163 ceph-mon[221067]: osdmap e619: 8 total, 8 up, 8 in 2023-10-09T04:32:52.509 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:32:52.859 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:52.989 INFO:teuthology.orchestra.run.smithi163.stdout:mon.a: {} 2023-10-09T04:32:52.999 INFO:teuthology.orchestra.run.smithi163.stdout:mon.b: {} 2023-10-09T04:32:53.008 INFO:teuthology.orchestra.run.smithi163.stdout:mon.c: {} 2023-10-09T04:32:53.010 INFO:teuthology.orchestra.run.smithi163.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T04:32:53.010 INFO:teuthology.orchestra.run.smithi163.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T04:32:53.010 INFO:teuthology.orchestra.run.smithi163.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T04:32:53.182 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:52 smithi163 ceph-mon[221067]: failed to encode map e619 with expected crc 2023-10-09T04:32:53.182 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:52 smithi163 ceph-mon[221067]: pgmap v1593: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1.0 MiB/s, 8 objects/s recovering 2023-10-09T04:32:53.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:52 smithi163 ceph-mon[213841]: failed to encode map e619 with expected crc 2023-10-09T04:32:53.183 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:52 smithi163 ceph-mon[213841]: pgmap v1593: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1.0 MiB/s, 8 objects/s recovering 2023-10-09T04:32:53.262 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:52 smithi181 ceph-mon[196979]: failed to encode map e619 with expected crc 2023-10-09T04:32:53.262 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:52 smithi181 ceph-mon[196979]: pgmap v1593: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1.0 MiB/s, 8 objects/s recovering 2023-10-09T04:32:53.639 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:32:54.027 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:54.128 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:53 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2020303727' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:32:54.240 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:53 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2020303727' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:32:54.240 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:53 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2020303727' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:32:54.350 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:54.350 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:54.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:32:55.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:54 smithi181 ceph-mon[196979]: pgmap v1594: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 737 KiB/s, 6 objects/s recovering 2023-10-09T04:32:55.275 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:54 smithi163 ceph-mon[221067]: pgmap v1594: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 737 KiB/s, 6 objects/s recovering 2023-10-09T04:32:55.276 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:54 smithi163 ceph-mon[213841]: pgmap v1594: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 737 KiB/s, 6 objects/s recovering 2023-10-09T04:32:55.288 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:55.926 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:32:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:55 smithi181 ceph-mon[196979]: 1.7 deep-scrub starts 2023-10-09T04:32:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:55 smithi181 ceph-mon[196979]: 1.7 deep-scrub ok 2023-10-09T04:32:56.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:55 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2894491452' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:32:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:55 smithi163 ceph-mon[221067]: 1.7 deep-scrub starts 2023-10-09T04:32:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:55 smithi163 ceph-mon[221067]: 1.7 deep-scrub ok 2023-10-09T04:32:56.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:55 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2894491452' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:32:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:55 smithi163 ceph-mon[213841]: 1.7 deep-scrub starts 2023-10-09T04:32:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:55 smithi163 ceph-mon[213841]: 1.7 deep-scrub ok 2023-10-09T04:32:56.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:55 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2894491452' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:32:56.564 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T04:32:56.744 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:56.744 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:56.909 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2894491452' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:32:57.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: osdmap e620: 8 total, 8 up, 8 in 2023-10-09T04:32:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:56 smithi181 ceph-mon[196979]: pgmap v1596: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 708 KiB/s, 5 objects/s recovering 2023-10-09T04:32:57.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2894491452' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:32:57.283 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: osdmap e620: 8 total, 8 up, 8 in 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[221067]: pgmap v1596: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 708 KiB/s, 5 objects/s recovering 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2894491452' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: osdmap e620: 8 total, 8 up, 8 in 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.284 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: failed to encode map e620 with expected crc 2023-10-09T04:32:57.285 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:56 smithi163 ceph-mon[213841]: pgmap v1596: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 708 KiB/s, 5 objects/s recovering 2023-10-09T04:32:57.375 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:32:57.716 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:58.852 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T04:32:58.933 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:58 smithi181 ceph-mon[196979]: pgmap v1597: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 552 KiB/s, 4 objects/s recovering 2023-10-09T04:32:58.934 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:58 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3895572122' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:32:59.140 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:59.140 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:58 smithi163 ceph-mon[221067]: pgmap v1597: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 552 KiB/s, 4 objects/s recovering 2023-10-09T04:32:59.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:58 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3895572122' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:32:59.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:58 smithi163 ceph-mon[213841]: pgmap v1597: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 552 KiB/s, 4 objects/s recovering 2023-10-09T04:32:59.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:58 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3895572122' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:32:59.471 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:32:59.715 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:32:59.816 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:59 smithi181 ceph-mon[196979]: failed to encode map e621 with expected crc 2023-10-09T04:33:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:59 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3895572122' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:33:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:59 smithi181 ceph-mon[196979]: osdmap e621: 8 total, 8 up, 8 in 2023-10-09T04:33:00.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:32:59 smithi181 ceph-mon[196979]: failed to encode map e621 with expected crc 2023-10-09T04:33:00.066 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:00.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[221067]: failed to encode map e621 with expected crc 2023-10-09T04:33:00.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3895572122' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:33:00.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[221067]: osdmap e621: 8 total, 8 up, 8 in 2023-10-09T04:33:00.197 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[221067]: failed to encode map e621 with expected crc 2023-10-09T04:33:00.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[213841]: failed to encode map e621 with expected crc 2023-10-09T04:33:00.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3895572122' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:33:00.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[213841]: osdmap e621: 8 total, 8 up, 8 in 2023-10-09T04:33:00.198 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:32:59 smithi163 ceph-mon[213841]: failed to encode map e621 with expected crc 2023-10-09T04:33:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:00 smithi181 ceph-mon[196979]: failed to encode map e622 with expected crc 2023-10-09T04:33:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:00 smithi181 ceph-mon[196979]: osdmap e622: 8 total, 8 up, 8 in 2023-10-09T04:33:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:00 smithi181 ceph-mon[196979]: failed to encode map e622 with expected crc 2023-10-09T04:33:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:00 smithi181 ceph-mon[196979]: pgmap v1600: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:01.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:00 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:01.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[221067]: failed to encode map e622 with expected crc 2023-10-09T04:33:01.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[221067]: osdmap e622: 8 total, 8 up, 8 in 2023-10-09T04:33:01.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[221067]: failed to encode map e622 with expected crc 2023-10-09T04:33:01.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[221067]: pgmap v1600: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:01.067 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:01.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[213841]: failed to encode map e622 with expected crc 2023-10-09T04:33:01.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[213841]: osdmap e622: 8 total, 8 up, 8 in 2023-10-09T04:33:01.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[213841]: failed to encode map e622 with expected crc 2023-10-09T04:33:01.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[213841]: pgmap v1600: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:01.068 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:00 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:01.068 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:33:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:33:00] "GET /metrics HTTP/1.1" 200 34228 "" "Prometheus/2.43.0" 2023-10-09T04:33:01.582 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:01.582 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:01.863 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:33:02.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:01 smithi181 ceph-mon[196979]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:01 smithi181 ceph-mon[196979]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:01 smithi181 ceph-mon[196979]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:01 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1857749456' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[221067]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[221067]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[221067]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1857749456' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:02.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[213841]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[213841]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[213841]: failed to encode map e622 with expected crc 2023-10-09T04:33:02.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:01 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1857749456' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:02.480 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T04:33:02.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:33:02.830 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:02.855 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:03.100 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:02 smithi181 ceph-mon[196979]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:02 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1857749456' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:33:03.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:02 smithi181 ceph-mon[196979]: osdmap e623: 8 total, 8 up, 8 in 2023-10-09T04:33:03.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:02 smithi181 ceph-mon[196979]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:02 smithi181 ceph-mon[196979]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.101 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:02 smithi181 ceph-mon[196979]: pgmap v1602: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:03.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[221067]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1857749456' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:33:03.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[221067]: osdmap e623: 8 total, 8 up, 8 in 2023-10-09T04:33:03.209 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[221067]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[221067]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[221067]: pgmap v1602: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[213841]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1857749456' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[213841]: osdmap e623: 8 total, 8 up, 8 in 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[213841]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[213841]: failed to encode map e623 with expected crc 2023-10-09T04:33:03.210 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:02 smithi163 ceph-mon[213841]: pgmap v1602: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:04.131 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:03 smithi163 ceph-mon[221067]: failed to encode map e623 with expected crc 2023-10-09T04:33:04.132 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:03 smithi163 ceph-mon[213841]: failed to encode map e623 with expected crc 2023-10-09T04:33:04.194 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:03 smithi181 ceph-mon[196979]: failed to encode map e623 with expected crc 2023-10-09T04:33:04.419 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:04.419 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:04.883 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T04:33:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:04 smithi181 ceph-mon[196979]: pgmap v1603: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:05.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2191013545' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:05.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:33:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:04 smithi163 ceph-mon[221067]: pgmap v1603: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:05.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2191013545' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:04 smithi163 ceph-mon[213841]: pgmap v1603: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:05.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2191013545' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:05.401 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:05.538 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:33:05.894 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:06.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:05 smithi181 ceph-mon[196979]: failed to encode map e624 with expected crc 2023-10-09T04:33:06.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:05 smithi181 ceph-mon[196979]: failed to encode map e624 with expected crc 2023-10-09T04:33:06.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:05 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2191013545' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:33:06.090 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:05 smithi181 ceph-mon[196979]: osdmap e624: 8 total, 8 up, 8 in 2023-10-09T04:33:06.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[221067]: failed to encode map e624 with expected crc 2023-10-09T04:33:06.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[221067]: failed to encode map e624 with expected crc 2023-10-09T04:33:06.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2191013545' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:33:06.245 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[221067]: osdmap e624: 8 total, 8 up, 8 in 2023-10-09T04:33:06.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[213841]: failed to encode map e624 with expected crc 2023-10-09T04:33:06.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[213841]: failed to encode map e624 with expected crc 2023-10-09T04:33:06.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2191013545' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:33:06.246 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:05 smithi163 ceph-mon[213841]: osdmap e624: 8 total, 8 up, 8 in 2023-10-09T04:33:07.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: osdmap e625: 8 total, 8 up, 8 in 2023-10-09T04:33:07.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.102 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.103 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:06 smithi181 ceph-mon[196979]: pgmap v1606: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: osdmap e625: 8 total, 8 up, 8 in 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.149 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[221067]: pgmap v1606: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: osdmap e625: 8 total, 8 up, 8 in 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: failed to encode map e625 with expected crc 2023-10-09T04:33:07.150 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:06 smithi163 ceph-mon[213841]: pgmap v1606: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:07.327 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:07.327 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:07.909 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:33:08.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:33:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:07 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/374449156' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:08.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:07 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:07 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/374449156' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:08.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:07 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:07 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/374449156' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:08.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:07 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:08.360 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:08.587 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T04:33:08.933 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:09.152 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: 1.12 scrub starts 2023-10-09T04:33:09.152 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: 1.12 scrub ok 2023-10-09T04:33:09.152 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: osdmap e626: 8 total, 8 up, 8 in 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: pgmap v1608: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[221067]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: 1.12 scrub starts 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: 1.12 scrub ok 2023-10-09T04:33:09.153 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:33:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: osdmap e626: 8 total, 8 up, 8 in 2023-10-09T04:33:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: pgmap v1608: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.154 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:08 smithi163 ceph-mon[213841]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: 1.12 scrub starts 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: 1.12 scrub ok 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: osdmap e626: 8 total, 8 up, 8 in 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: pgmap v1608: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:08 smithi181 ceph-mon[196979]: failed to encode map e626 with expected crc 2023-10-09T04:33:09.953 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:09.953 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:10.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:09 smithi181 ceph-mon[196979]: failed to encode map e626 with expected crc 2023-10-09T04:33:10.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:09 smithi181 ceph-mon[196979]: failed to encode map e626 with expected crc 2023-10-09T04:33:10.257 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:09 smithi163 ceph-mon[221067]: failed to encode map e626 with expected crc 2023-10-09T04:33:10.257 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:09 smithi163 ceph-mon[221067]: failed to encode map e626 with expected crc 2023-10-09T04:33:10.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:09 smithi163 ceph-mon[213841]: failed to encode map e626 with expected crc 2023-10-09T04:33:10.257 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:09 smithi163 ceph-mon[213841]: failed to encode map e626 with expected crc 2023-10-09T04:33:10.606 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:33:10.911 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:33:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:33:10] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2023-10-09T04:33:10.923 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T04:33:10.956 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:10 smithi181 ceph-mon[196979]: pgmap v1609: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:11.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2640029707' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:10 smithi163 ceph-mon[213841]: pgmap v1609: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:11.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2640029707' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:10 smithi163 ceph-mon[221067]: pgmap v1609: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2640029707' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:33:11.603 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:33:11.918 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: failed to encode map e627 with expected crc 2023-10-09T04:33:11.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2640029707' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:33:11.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: osdmap e627: 8 total, 8 up, 8 in 2023-10-09T04:33:11.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: failed to encode map e627 with expected crc 2023-10-09T04:33:11.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: failed to encode map e627 with expected crc 2023-10-09T04:33:11.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: failed to encode map e627 with expected crc 2023-10-09T04:33:11.919 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[213841]: failed to encode map e627 with expected crc 2023-10-09T04:33:11.958 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2640029707' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: osdmap e627: 8 total, 8 up, 8 in 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.177 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:11 smithi163 ceph-mon[221067]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2640029707' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: osdmap e627: 8 total, 8 up, 8 in 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:11 smithi181 ceph-mon[196979]: failed to encode map e627 with expected crc 2023-10-09T04:33:12.484 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:12.484 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:13.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: osdmap e628: 8 total, 8 up, 8 in 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.232 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[221067]: pgmap v1612: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: osdmap e628: 8 total, 8 up, 8 in 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.233 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:12 smithi163 ceph-mon[213841]: pgmap v1612: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:13.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: osdmap e628: 8 total, 8 up, 8 in 2023-10-09T04:33:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: failed to encode map e628 with expected crc 2023-10-09T04:33:13.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:12 smithi181 ceph-mon[196979]: pgmap v1612: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:13.494 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:13.937 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:33:14.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2220401983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:14.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2220401983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:14.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2220401983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:14.562 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T04:33:14.914 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:14.985 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:14.985 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: 1.14 scrub starts 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: 1.14 scrub ok 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: failed to encode map e629 with expected crc 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: failed to encode map e629 with expected crc 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2220401983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: osdmap e629: 8 total, 8 up, 8 in 2023-10-09T04:33:15.174 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:14 smithi181 ceph-mon[196979]: pgmap v1614: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:15.191 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: 1.14 scrub starts 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: 1.14 scrub ok 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: failed to encode map e629 with expected crc 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: failed to encode map e629 with expected crc 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2220401983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: osdmap e629: 8 total, 8 up, 8 in 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[221067]: pgmap v1614: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:15.192 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: 1.14 scrub starts 2023-10-09T04:33:15.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: 1.14 scrub ok 2023-10-09T04:33:15.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: failed to encode map e629 with expected crc 2023-10-09T04:33:15.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: failed to encode map e629 with expected crc 2023-10-09T04:33:15.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2220401983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:33:15.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: osdmap e629: 8 total, 8 up, 8 in 2023-10-09T04:33:15.193 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:14 smithi163 ceph-mon[213841]: pgmap v1614: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:15.621 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:33:15.962 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:33:17.221 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T04:33:17.283 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:17 smithi181 ceph-mon[196979]: pgmap v1615: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-10-09T04:33:17.283 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:17 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1634865039' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:17.283 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:17 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:17.513 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:17.513 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:17 smithi163 ceph-mon[221067]: pgmap v1615: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-10-09T04:33:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:17 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1634865039' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:17.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:17 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:17 smithi163 ceph-mon[213841]: pgmap v1615: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-10-09T04:33:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:17 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1634865039' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:17.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:17 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:33:17.870 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:33:18.226 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[213841]: failed to encode map e630 with expected crc 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[213841]: failed to encode map e630 with expected crc 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[213841]: osdmap e630: 8 total, 8 up, 8 in 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[221067]: failed to encode map e630 with expected crc 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[221067]: failed to encode map e630 with expected crc 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:33:18.465 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:18 smithi163 ceph-mon[221067]: osdmap e630: 8 total, 8 up, 8 in 2023-10-09T04:33:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:18 smithi181 ceph-mon[196979]: failed to encode map e630 with expected crc 2023-10-09T04:33:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:18 smithi181 ceph-mon[196979]: failed to encode map e630 with expected crc 2023-10-09T04:33:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:18 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:33:18.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:18 smithi181 ceph-mon[196979]: osdmap e630: 8 total, 8 up, 8 in 2023-10-09T04:33:19.488 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[221067]: pgmap v1617: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[221067]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[221067]: osdmap e631: 8 total, 8 up, 8 in 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[221067]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[221067]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[221067]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[213841]: pgmap v1617: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[213841]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[213841]: osdmap e631: 8 total, 8 up, 8 in 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[213841]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.489 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[213841]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.490 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:19 smithi163 ceph-mon[213841]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:19 smithi181 ceph-mon[196979]: pgmap v1617: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-10-09T04:33:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:19 smithi181 ceph-mon[196979]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:19 smithi181 ceph-mon[196979]: osdmap e631: 8 total, 8 up, 8 in 2023-10-09T04:33:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:19 smithi181 ceph-mon[196979]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:19 smithi181 ceph-mon[196979]: failed to encode map e631 with expected crc 2023-10-09T04:33:19.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:19 smithi181 ceph-mon[196979]: failed to encode map e631 with expected crc 2023-10-09T04:33:20.230 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:33:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:20 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:20.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:20 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3743835555' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:20 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:20.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:20 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3743835555' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:20 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:20.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:20 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3743835555' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:20.843 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T04:33:21.082 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:33:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:33:20] "GET /metrics HTTP/1.1" 200 34238 "" "Prometheus/2.43.0" 2023-10-09T04:33:21.195 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:21 smithi181 ceph-mon[196979]: pgmap v1619: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-10-09T04:33:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:21 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:21 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3743835555' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:33:21.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:21 smithi181 ceph-mon[196979]: osdmap e632: 8 total, 8 up, 8 in 2023-10-09T04:33:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[221067]: pgmap v1619: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-10-09T04:33:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3743835555' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:33:21.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[221067]: osdmap e632: 8 total, 8 up, 8 in 2023-10-09T04:33:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[213841]: pgmap v1619: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-10-09T04:33:21.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:21.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3743835555' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:33:21.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:21 smithi163 ceph-mon[213841]: osdmap e632: 8 total, 8 up, 8 in 2023-10-09T04:33:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:22 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:22 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:22 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:22 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:22 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:22 smithi181 ceph-mon[196979]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.555 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[213841]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:22.556 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:22 smithi163 ceph-mon[221067]: failed to encode map e632 with expected crc 2023-10-09T04:33:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:23 smithi181 ceph-mon[196979]: pgmap v1621: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:23 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2959454045' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:23.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:23 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:23 smithi163 ceph-mon[221067]: pgmap v1621: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:23 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2959454045' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:23.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:23 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:23 smithi163 ceph-mon[213841]: pgmap v1621: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:23.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:23 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2959454045' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:23.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:23 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:33:23.861 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T04:33:24.555 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:33:24.913 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: osdmap e633: 8 total, 8 up, 8 in 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: pgmap v1623: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: 1.7 scrub starts 2023-10-09T04:33:25.127 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[213841]: 1.7 scrub ok 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: osdmap e633: 8 total, 8 up, 8 in 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: pgmap v1623: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: 1.7 scrub starts 2023-10-09T04:33:25.128 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:24 smithi163 ceph-mon[221067]: 1.7 scrub ok 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: osdmap e633: 8 total, 8 up, 8 in 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: failed to encode map e633 with expected crc 2023-10-09T04:33:25.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: pgmap v1623: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:25.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: 1.7 scrub starts 2023-10-09T04:33:25.270 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:24 smithi181 ceph-mon[196979]: 1.7 scrub ok 2023-10-09T04:33:27.211 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:33:27.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:27 smithi181 ceph-mon[196979]: pgmap v1624: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:27.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:27 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1182329479' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:27 smithi163 ceph-mon[221067]: pgmap v1624: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:27.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:27 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1182329479' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:27 smithi163 ceph-mon[213841]: pgmap v1624: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:27.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:27 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1182329479' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:27.928 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T04:33:28.281 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:28.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1182329479' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:33:28.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[221067]: osdmap e634: 8 total, 8 up, 8 in 2023-10-09T04:33:28.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[221067]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[221067]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.493 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[221067]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.493 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1182329479' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:33:28.494 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[213841]: osdmap e634: 8 total, 8 up, 8 in 2023-10-09T04:33:28.494 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[213841]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.494 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[213841]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.494 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:28 smithi163 ceph-mon[213841]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:28 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1182329479' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:33:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:28 smithi181 ceph-mon[196979]: osdmap e634: 8 total, 8 up, 8 in 2023-10-09T04:33:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:28 smithi181 ceph-mon[196979]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:28 smithi181 ceph-mon[196979]: failed to encode map e634 with expected crc 2023-10-09T04:33:28.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:28 smithi181 ceph-mon[196979]: failed to encode map e634 with expected crc 2023-10-09T04:33:29.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:29 smithi181 ceph-mon[196979]: pgmap v1626: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:29.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:29 smithi181 ceph-mon[196979]: failed to encode map e634 with expected crc 2023-10-09T04:33:29.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:29 smithi163 ceph-mon[221067]: pgmap v1626: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:29.560 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:29 smithi163 ceph-mon[221067]: failed to encode map e634 with expected crc 2023-10-09T04:33:29.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:29 smithi163 ceph-mon[213841]: pgmap v1626: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:29.561 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:29 smithi163 ceph-mon[213841]: failed to encode map e634 with expected crc 2023-10-09T04:33:30.234 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T04:33:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:30 smithi181 ceph-mon[196979]: failed to encode map e634 with expected crc 2023-10-09T04:33:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:30 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/231844113' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:30.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:30 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:30 smithi163 ceph-mon[221067]: failed to encode map e634 with expected crc 2023-10-09T04:33:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:30 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/231844113' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:30.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:30 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:30 smithi163 ceph-mon[213841]: failed to encode map e634 with expected crc 2023-10-09T04:33:30.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:30 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/231844113' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:30.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:30 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:33:30.949 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:33:30 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:33:30] "GET /metrics HTTP/1.1" 200 34238 "" "Prometheus/2.43.0" 2023-10-09T04:33:30.968 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:33:31.323 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:31.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[221067]: pgmap v1627: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:31.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[221067]: failed to encode map e635 with expected crc 2023-10-09T04:33:31.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[221067]: failed to encode map e635 with expected crc 2023-10-09T04:33:31.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:33:31.517 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[221067]: osdmap e635: 8 total, 8 up, 8 in 2023-10-09T04:33:31.517 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[213841]: pgmap v1627: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[213841]: failed to encode map e635 with expected crc 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[213841]: failed to encode map e635 with expected crc 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:31 smithi163 ceph-mon[213841]: osdmap e635: 8 total, 8 up, 8 in 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:31 smithi181 ceph-mon[196979]: pgmap v1627: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:31 smithi181 ceph-mon[196979]: failed to encode map e635 with expected crc 2023-10-09T04:33:31.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:31 smithi181 ceph-mon[196979]: failed to encode map e635 with expected crc 2023-10-09T04:33:31.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:31 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:33:31.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:31 smithi181 ceph-mon[196979]: osdmap e635: 8 total, 8 up, 8 in 2023-10-09T04:33:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:32 smithi181 ceph-mon[196979]: osdmap e636: 8 total, 8 up, 8 in 2023-10-09T04:33:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:32 smithi181 ceph-mon[196979]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:32 smithi181 ceph-mon[196979]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:32 smithi181 ceph-mon[196979]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[221067]: osdmap e636: 8 total, 8 up, 8 in 2023-10-09T04:33:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[221067]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[221067]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[221067]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[213841]: osdmap e636: 8 total, 8 up, 8 in 2023-10-09T04:33:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[213841]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[213841]: failed to encode map e636 with expected crc 2023-10-09T04:33:32.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:32 smithi163 ceph-mon[213841]: failed to encode map e636 with expected crc 2023-10-09T04:33:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:33 smithi181 ceph-mon[196979]: pgmap v1630: 103 pgs: 19 peering, 84 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:33 smithi181 ceph-mon[196979]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:33:33.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:33 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2678023571' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:33 smithi163 ceph-mon[213841]: pgmap v1630: 103 pgs: 19 peering, 84 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:33.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:33 smithi163 ceph-mon[213841]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:33:33.583 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:33 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2678023571' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:33 smithi163 ceph-mon[221067]: pgmap v1630: 103 pgs: 19 peering, 84 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:33 smithi163 ceph-mon[221067]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:33:33.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:33 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2678023571' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:33.855 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:33:34.542 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T04:33:34.890 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: failed to encode map e637 with expected crc 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: failed to encode map e637 with expected crc 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2678023571' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: osdmap e637: 8 total, 8 up, 8 in 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: pgmap v1632: 103 pgs: 19 peering, 84 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:35.107 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: failed to encode map e637 with expected crc 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: failed to encode map e637 with expected crc 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2678023571' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: osdmap e637: 8 total, 8 up, 8 in 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: pgmap v1632: 103 pgs: 19 peering, 84 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:35.108 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:34 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: failed to encode map e637 with expected crc 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: failed to encode map e637 with expected crc 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2678023571' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: osdmap e637: 8 total, 8 up, 8 in 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: pgmap v1632: 103 pgs: 19 peering, 84 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:35.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:34 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:36.119 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:35 smithi163 ceph-mon[221067]: failed to encode map e637 with expected crc 2023-10-09T04:33:36.119 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:35 smithi163 ceph-mon[213841]: failed to encode map e637 with expected crc 2023-10-09T04:33:36.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:35 smithi181 ceph-mon[196979]: failed to encode map e637 with expected crc 2023-10-09T04:33:36.865 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T04:33:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:36 smithi181 ceph-mon[196979]: failed to encode map e637 with expected crc 2023-10-09T04:33:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:36 smithi181 ceph-mon[196979]: pgmap v1633: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:36 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2988742125' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:37.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:36 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[213841]: failed to encode map e637 with expected crc 2023-10-09T04:33:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[213841]: pgmap v1633: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2988742125' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:37.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[221067]: failed to encode map e637 with expected crc 2023-10-09T04:33:37.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[221067]: pgmap v1633: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:37.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2988742125' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:37.333 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:36 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:33:37.586 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:33:37.948 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:38.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[221067]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 11 pgs peering) 2023-10-09T04:33:38.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[221067]: Cluster is now healthy 2023-10-09T04:33:38.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[221067]: failed to encode map e638 with expected crc 2023-10-09T04:33:38.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[221067]: failed to encode map e638 with expected crc 2023-10-09T04:33:38.141 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[221067]: osdmap e638: 8 total, 8 up, 8 in 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[213841]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 11 pgs peering) 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[213841]: Cluster is now healthy 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[213841]: failed to encode map e638 with expected crc 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[213841]: failed to encode map e638 with expected crc 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:33:38.142 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:37 smithi163 ceph-mon[213841]: osdmap e638: 8 total, 8 up, 8 in 2023-10-09T04:33:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:37 smithi181 ceph-mon[196979]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 11 pgs peering) 2023-10-09T04:33:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:37 smithi181 ceph-mon[196979]: Cluster is now healthy 2023-10-09T04:33:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:37 smithi181 ceph-mon[196979]: failed to encode map e638 with expected crc 2023-10-09T04:33:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:37 smithi181 ceph-mon[196979]: failed to encode map e638 with expected crc 2023-10-09T04:33:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:37 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:33:38.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:37 smithi181 ceph-mon[196979]: osdmap e638: 8 total, 8 up, 8 in 2023-10-09T04:33:39.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[213841]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.218 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[213841]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[213841]: osdmap e639: 8 total, 8 up, 8 in 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[213841]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[213841]: pgmap v1636: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[221067]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[221067]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[221067]: osdmap e639: 8 total, 8 up, 8 in 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[221067]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.219 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:38 smithi163 ceph-mon[221067]: pgmap v1636: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:38 smithi181 ceph-mon[196979]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:38 smithi181 ceph-mon[196979]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:38 smithi181 ceph-mon[196979]: osdmap e639: 8 total, 8 up, 8 in 2023-10-09T04:33:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:38 smithi181 ceph-mon[196979]: failed to encode map e639 with expected crc 2023-10-09T04:33:39.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:38 smithi181 ceph-mon[196979]: pgmap v1636: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:39.882 INFO:teuthology.orchestra.run.smithi163.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:33:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:39 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/3013916520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:40.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:39 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:39 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/3013916520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:40.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:39 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:39 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/3013916520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:40.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:39 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:40.497 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T04:33:40.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:33:40 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:33:40] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2023-10-09T04:33:40.845 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:41.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[221067]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:41.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[221067]: osdmap e640: 8 total, 8 up, 8 in 2023-10-09T04:33:41.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[221067]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[221067]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.248 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[221067]: pgmap v1638: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:41.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[213841]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:41.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[213841]: osdmap e640: 8 total, 8 up, 8 in 2023-10-09T04:33:41.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[213841]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[213841]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.249 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:40 smithi163 ceph-mon[213841]: pgmap v1638: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:41.268 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:40 smithi181 ceph-mon[196979]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:40 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:40 smithi181 ceph-mon[196979]: osdmap e640: 8 total, 8 up, 8 in 2023-10-09T04:33:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:40 smithi181 ceph-mon[196979]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:40 smithi181 ceph-mon[196979]: failed to encode map e640 with expected crc 2023-10-09T04:33:41.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:40 smithi181 ceph-mon[196979]: pgmap v1638: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:42.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:41 smithi163 ceph-mon[213841]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:41 smithi163 ceph-mon[213841]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.126 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:41 smithi163 ceph-mon[213841]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.127 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:41 smithi163 ceph-mon[221067]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.127 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:41 smithi163 ceph-mon[221067]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.127 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:41 smithi163 ceph-mon[221067]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:41 smithi181 ceph-mon[196979]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:41 smithi181 ceph-mon[196979]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:41 smithi181 ceph-mon[196979]: failed to encode map e640 with expected crc 2023-10-09T04:33:42.893 INFO:teuthology.orchestra.run.smithi163.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T04:33:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:42 smithi181 ceph-mon[196979]: pgmap v1639: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:42 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2657227435' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:43.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:42 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:42 smithi163 ceph-mon[213841]: pgmap v1639: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:42 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2657227435' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:43.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:42 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:42 smithi163 ceph-mon[221067]: pgmap v1639: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:42 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2657227435' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:43.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:42 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:33:43.532 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T04:33:43.532 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T04:33:43.532 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T04:33:43.532 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T04:33:43.532 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:33:43.532 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:33:43.884 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:44.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:43 smithi163 ceph-mon[221067]: failed to encode map e641 with expected crc 2023-10-09T04:33:44.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:43 smithi163 ceph-mon[221067]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:44.254 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:43 smithi163 ceph-mon[221067]: osdmap e641: 8 total, 8 up, 8 in 2023-10-09T04:33:44.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:43 smithi163 ceph-mon[213841]: failed to encode map e641 with expected crc 2023-10-09T04:33:44.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:43 smithi163 ceph-mon[213841]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:44.254 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:43 smithi163 ceph-mon[213841]: osdmap e641: 8 total, 8 up, 8 in 2023-10-09T04:33:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:43 smithi181 ceph-mon[196979]: failed to encode map e641 with expected crc 2023-10-09T04:33:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:43 smithi181 ceph-mon[196979]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:44.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:43 smithi181 ceph-mon[196979]: osdmap e641: 8 total, 8 up, 8 in 2023-10-09T04:33:45.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[221067]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[221067]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[221067]: osdmap e642: 8 total, 8 up, 8 in 2023-10-09T04:33:45.142 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[221067]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.143 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[221067]: pgmap v1642: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:45.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[213841]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[213841]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[213841]: osdmap e642: 8 total, 8 up, 8 in 2023-10-09T04:33:45.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[213841]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.143 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:44 smithi163 ceph-mon[213841]: pgmap v1642: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:44 smithi181 ceph-mon[196979]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:44 smithi181 ceph-mon[196979]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:44 smithi181 ceph-mon[196979]: osdmap e642: 8 total, 8 up, 8 in 2023-10-09T04:33:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:44 smithi181 ceph-mon[196979]: failed to encode map e642 with expected crc 2023-10-09T04:33:45.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:44 smithi181 ceph-mon[196979]: pgmap v1642: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:45.504 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:33:45.504 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":642,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T04:33:43.886151+0000","last_up_change":"2023-10-09T04:03:36.195924+0000","last_in_change":"2023-10-09T04:32:45.858461+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"301","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"292","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"607","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":607,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T03:54:11.510171+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T03:54:12.437589+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":621,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6803","nonce":658095252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6805","nonce":658095252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6809","nonce":658095252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6807","nonce":658095252}]},"public_addr":"172.21.15.163:6803/658095252","cluster_addr":"172.21.15.163:6805/658095252","heartbeat_back_addr":"172.21.15.163:6809/658095252","heartbeat_front_addr":"172.21.15.163:6807/658095252","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":624,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6811","nonce":3761898215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6813","nonce":3761898215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6817","nonce":3761898215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6815","nonce":3761898215}]},"public_addr":"172.21.15.163:6811/3761898215","cluster_addr":"172.21.15.163:6813/3761898215","heartbeat_back_addr":"172.21.15.163:6817/3761898215","heartbeat_front_addr":"172.21.15.163:6815/3761898215","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":627,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6819","nonce":96613669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6821","nonce":96613669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6825","nonce":96613669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6823","nonce":96613669}]},"public_addr":"172.21.15.163:6819/96613669","cluster_addr":"172.21.15.163:6821/96613669","heartbeat_back_addr":"172.21.15.163:6825/96613669","heartbeat_front_addr":"172.21.15.163:6823/96613669","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":95,"up_from":99,"up_thru":630,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6827","nonce":3467616858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6829","nonce":3467616858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6833","nonce":3467616858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6831","nonce":3467616858}]},"public_addr":"172.21.15.163:6827/3467616858","cluster_addr":"172.21.15.163:6829/3467616858","heartbeat_back_addr":"172.21.15.163:6833/3467616858","heartbeat_front_addr":"172.21.15.163:6831/3467616858","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":617,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":635,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":638,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":641,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.e","mappings":[{"from":4,"to":2}]},{"pgid":"3.19","mappings":[{"from":4,"to":0}]},{"pgid":"3.1b","mappings":[{"from":4,"to":1}]},{"pgid":"4.4","mappings":[{"from":1,"to":0}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":7}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/1229473398":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.181:6834/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.181:0/1808329322":"2023-10-10T03:57:13.005970+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1847962187":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/3524034575":"2023-10-10T03:56:35.412470+0000","172.21.15.181:6835/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/814100112":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/2543837887":"2023-10-10T03:57:13.005970+0000","172.21.15.163:6801/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.181:0/647401329":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.181:0/3987314134":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2462309240":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6800/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6801/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.181:0/1614027396":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/3405897408":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1361101496":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/1275790360":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/4218140698":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1802556470":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2380156011":"2023-10-10T03:57:49.149190+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:33:46.148 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:33:46.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:33:46.497 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:46 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:46.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:46 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1482702754' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:33:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:46 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:46.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:46 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1482702754' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:33:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:46 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:46.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:46 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1482702754' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:33:47.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:47 smithi163 ceph-mon[221067]: pgmap v1643: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:47.750 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:47 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:47.750 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:47 smithi163 ceph-mon[213841]: pgmap v1643: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:47.750 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:47 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:47 smithi181 ceph-mon[196979]: pgmap v1643: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:47.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:47 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:48.152 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:33:48.153 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":642,"fsid":"8628d772-6656-11ee-8db5-212e2dc638e7","created":"2023-10-09T03:48:06.379138+0000","modified":"2023-10-09T04:33:43.886151+0000","last_up_change":"2023-10-09T04:03:36.195924+0000","last_in_change":"2023-10-09T04:32:45.858461+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T03:50:59.539545+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"301","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"292","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T03:53:18.822166+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"607","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":607,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T03:54:11.510171+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T03:54:12.437589+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9f39300f-5c82-4020-b75d-28b9705365f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":80,"up_from":84,"up_thru":621,"down_at":81,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6803","nonce":658095252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6805","nonce":658095252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6808","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6809","nonce":658095252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":658095252},{"type":"v1","addr":"172.21.15.163:6807","nonce":658095252}]},"public_addr":"172.21.15.163:6803/658095252","cluster_addr":"172.21.15.163:6805/658095252","heartbeat_back_addr":"172.21.15.163:6809/658095252","heartbeat_front_addr":"172.21.15.163:6807/658095252","state":["exists","up"]},{"osd":1,"uuid":"07dc6811-c7f1-40ce-83b3-46a33ff57f0a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":85,"up_from":89,"up_thru":624,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6810","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6811","nonce":3761898215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6812","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6813","nonce":3761898215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6816","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6817","nonce":3761898215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6814","nonce":3761898215},{"type":"v1","addr":"172.21.15.163:6815","nonce":3761898215}]},"public_addr":"172.21.15.163:6811/3761898215","cluster_addr":"172.21.15.163:6813/3761898215","heartbeat_back_addr":"172.21.15.163:6817/3761898215","heartbeat_front_addr":"172.21.15.163:6815/3761898215","state":["exists","up"]},{"osd":2,"uuid":"d5a25867-ce16-4bfd-8356-c346de1fcbf6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":90,"up_from":94,"up_thru":627,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6818","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6819","nonce":96613669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6820","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6821","nonce":96613669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6824","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6825","nonce":96613669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6822","nonce":96613669},{"type":"v1","addr":"172.21.15.163:6823","nonce":96613669}]},"public_addr":"172.21.15.163:6819/96613669","cluster_addr":"172.21.15.163:6821/96613669","heartbeat_back_addr":"172.21.15.163:6825/96613669","heartbeat_front_addr":"172.21.15.163:6823/96613669","state":["exists","up"]},{"osd":3,"uuid":"6d47d9fe-d7d7-4505-abf8-22ac463ee305","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":95,"up_from":99,"up_thru":630,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6826","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6827","nonce":3467616858}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6828","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6829","nonce":3467616858}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6832","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6833","nonce":3467616858}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6830","nonce":3467616858},{"type":"v1","addr":"172.21.15.163:6831","nonce":3467616858}]},"public_addr":"172.21.15.163:6827/3467616858","cluster_addr":"172.21.15.163:6829/3467616858","heartbeat_back_addr":"172.21.15.163:6833/3467616858","heartbeat_front_addr":"172.21.15.163:6831/3467616858","state":["exists","up"]},{"osd":4,"uuid":"ad58407a-b12d-4543-8092-f0ba514729f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":617,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6801","nonce":2565848687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6803","nonce":2565848687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6807","nonce":2565848687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2565848687},{"type":"v1","addr":"172.21.15.181:6805","nonce":2565848687}]},"public_addr":"172.21.15.181:6801/2565848687","cluster_addr":"172.21.15.181:6803/2565848687","heartbeat_back_addr":"172.21.15.181:6807/2565848687","heartbeat_front_addr":"172.21.15.181:6805/2565848687","state":["exists","up"]},{"osd":5,"uuid":"21ccce9f-28b2-4624-8a70-2f726e7c45d1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":635,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6809","nonce":2772326307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6811","nonce":2772326307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6815","nonce":2772326307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":2772326307},{"type":"v1","addr":"172.21.15.181:6813","nonce":2772326307}]},"public_addr":"172.21.15.181:6809/2772326307","cluster_addr":"172.21.15.181:6811/2772326307","heartbeat_back_addr":"172.21.15.181:6815/2772326307","heartbeat_front_addr":"172.21.15.181:6813/2772326307","state":["exists","up"]},{"osd":6,"uuid":"69caed62-5f69-4fcd-bbbd-5369ec7bc6e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":638,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6817","nonce":4236482149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6819","nonce":4236482149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6823","nonce":4236482149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":4236482149},{"type":"v1","addr":"172.21.15.181:6821","nonce":4236482149}]},"public_addr":"172.21.15.181:6817/4236482149","cluster_addr":"172.21.15.181:6819/4236482149","heartbeat_back_addr":"172.21.15.181:6823/4236482149","heartbeat_front_addr":"172.21.15.181:6821/4236482149","state":["exists","up"]},{"osd":7,"uuid":"3dc4fa53-2d90-441e-a825-74bd4da41f4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":641,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6825","nonce":2762148668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6827","nonce":2762148668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6831","nonce":2762148668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":2762148668},{"type":"v1","addr":"172.21.15.181:6829","nonce":2762148668}]},"public_addr":"172.21.15.181:6825/2762148668","cluster_addr":"172.21.15.181:6827/2762148668","heartbeat_back_addr":"172.21.15.181:6831/2762148668","heartbeat_front_addr":"172.21.15.181:6829/2762148668","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:11.457632+0000","dead_epoch":80},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:33.629887+0000","dead_epoch":85},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:50:55.616394+0000","dead_epoch":90},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:18.180997+0000","dead_epoch":95},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:38.653524+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:51:59.977867+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:21.215452+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T03:52:42.737103+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.e","mappings":[{"from":4,"to":2}]},{"pgid":"3.19","mappings":[{"from":4,"to":0}]},{"pgid":"3.1b","mappings":[{"from":4,"to":1}]},{"pgid":"4.4","mappings":[{"from":1,"to":0}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":7}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.163:0/983322351":"2023-10-10T03:48:32.847175+0000","172.21.15.181:0/3987314134":"2023-10-10T03:57:13.005970+0000","172.21.15.163:6800/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6800/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6800/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2462309240":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/343886015":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1847962187":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/3907401791":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1713944666":"2023-10-10T03:53:17.088151+0000","172.21.15.163:0/1802556470":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/2266209559":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/3524034575":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6801/277872466":"2023-10-10T03:57:49.149190+0000","172.21.15.181:6835/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/814100112":"2023-10-10T03:57:13.005970+0000","172.21.15.181:0/2543837887":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/4218140698":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1275790360":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6801/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/1488239398":"2023-10-10T03:48:32.847175+0000","172.21.15.163:6801/65515623":"2023-10-10T03:48:32.847175+0000","172.21.15.163:0/2134854516":"2023-10-10T03:48:55.215367+0000","172.21.15.181:6834/704384906":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2705338472":"2023-10-10T03:53:17.088151+0000","172.21.15.163:6801/330134886":"2023-10-10T03:53:17.088151+0000","172.21.15.181:0/1808329322":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/2380156011":"2023-10-10T03:57:49.149190+0000","172.21.15.181:0/647401329":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/1229473398":"2023-10-10T03:57:49.149190+0000","172.21.15.163:6800/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.163:6800/3683659572":"2023-10-10T03:48:55.215367+0000","172.21.15.163:6801/55760003":"2023-10-10T03:56:35.412470+0000","172.21.15.181:0/1614027396":"2023-10-10T03:57:13.005970+0000","172.21.15.163:0/3405897408":"2023-10-10T03:57:49.149190+0000","172.21.15.163:0/1361101496":"2023-10-10T03:56:35.412470+0000","172.21.15.163:0/2271486409":"2023-10-10T03:48:55.215367+0000","172.21.15.163:0/611815376":"2023-10-10T03:48:32.847175+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:33:48.779 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:33:48.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:33:48.780 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:33:48.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:33:48.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:33:48.781 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:33:48.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:33:48.782 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:33:49.094 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:48 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1834255467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:33:49.094 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:48 smithi163 ceph-mon[213841]: pgmap v1644: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:49.094 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:48 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1834255467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:33:49.095 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:48 smithi163 ceph-mon[221067]: pgmap v1644: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:48 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1834255467' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:33:49.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:48 smithi181 ceph-mon[196979]: pgmap v1644: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:33:49.500 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:49.899 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:49.900 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:49.901 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:50.229 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:50.234 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:50.240 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:50.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:50.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.636 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.636 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:50.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.637 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:50 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.666 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:50.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:50.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:50 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:33:51.074 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:33:50 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:33:50] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2023-10-09T04:33:51.760 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:51 smithi163 ceph-mon[221067]: pgmap v1645: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:51.761 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:51 smithi163 ceph-mon[213841]: pgmap v1645: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:51.768 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:51 smithi181 ceph-mon[196979]: pgmap v1645: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:53.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:53 smithi163 ceph-mon[221067]: pgmap v1646: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:53.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:53 smithi163 ceph-mon[213841]: pgmap v1646: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:53.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:53 smithi181 ceph-mon[196979]: pgmap v1646: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:54.472 INFO:teuthology.orchestra.run.smithi163.stdout:133143986686 2023-10-09T04:33:54.472 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:33:55.541 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:55 smithi163 ceph-mon[213841]: pgmap v1647: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:55.542 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:55 smithi163 ceph-mon[221067]: pgmap v1647: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:55.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:55 smithi181 ceph-mon[196979]: pgmap v1647: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:57.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:57 smithi163 ceph-mon[213841]: pgmap v1648: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:57.742 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:57 smithi163 ceph-mon[221067]: pgmap v1648: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:57.769 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:57 smithi181 ceph-mon[196979]: pgmap v1648: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:59.027 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:33:58 smithi163 ceph-mon[221067]: pgmap v1649: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:59.028 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:33:58 smithi163 ceph-mon[213841]: pgmap v1649: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:59.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:33:58 smithi181 ceph-mon[196979]: pgmap v1649: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:33:59.306 INFO:teuthology.orchestra.run.smithi163.stdout:360777253249 2023-10-09T04:33:59.307 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:33:59.607 INFO:teuthology.orchestra.run.smithi163.stdout:236223201778 2023-10-09T04:33:59.607 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:33:59.644 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:59.649 INFO:teuthology.orchestra.run.smithi163.stdout:167503725051 2023-10-09T04:33:59.650 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:34:00.636 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:00 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:34:00] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2023-10-09T04:34:00.644 INFO:teuthology.orchestra.run.smithi163.stdout:425201762671 2023-10-09T04:34:00.644 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:34:01.094 INFO:teuthology.orchestra.run.smithi163.stdout:201863463414 2023-10-09T04:34:01.094 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:34:01.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:01 smithi163 ceph-mon[221067]: pgmap v1650: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:01.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:01 smithi163 ceph-mon[213841]: pgmap v1650: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:01.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:01 smithi181 ceph-mon[196979]: pgmap v1650: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:02.820 INFO:teuthology.orchestra.run.smithi163.stdout:403726926197 2023-10-09T04:34:02.821 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:34:03.042 INFO:teuthology.orchestra.run.smithi163.stdout:382252089724 2023-10-09T04:34:03.043 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:34:03.310 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:03.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:03 smithi163 ceph-mon[221067]: pgmap v1651: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:03.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:03.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:03 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:03.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:03 smithi163 ceph-mon[213841]: pgmap v1651: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:03.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:03.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:03 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:03.362 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:03.446 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:03.474 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:03.514 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:03.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:03 smithi181 ceph-mon[196979]: pgmap v1651: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:03.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:03 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:03.649 INFO:teuthology.orchestra.run.smithi163.stdout:133143986688 2023-10-09T04:34:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:04 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:04.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:04 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1252951864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:34:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:04.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:04 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:04.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:04 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1252951864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:34:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:04 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:04.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:04 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1252951864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:34:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:05 smithi181 ceph-mon[196979]: pgmap v1652: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:05.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:05 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:05.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:05 smithi163 ceph-mon[221067]: pgmap v1652: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:05.527 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:05 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:05.527 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:05 smithi163 ceph-mon[213841]: pgmap v1652: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:05.528 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:05 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:06.106 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:06.257 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:06.288 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986686 got 133143986688 for osd.4 2023-10-09T04:34:06.289 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:07.320 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:07 smithi163 ceph-mon[213841]: pgmap v1653: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:07.320 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:07 smithi163 ceph-mon[221067]: pgmap v1653: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:07.520 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:07 smithi181 ceph-mon[196979]: pgmap v1653: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:07.955 INFO:teuthology.orchestra.run.smithi163.stdout:167503725053 2023-10-09T04:34:08.385 INFO:teuthology.orchestra.run.smithi163.stdout:236223201780 2023-10-09T04:34:09.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1543405602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:34:09.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:08 smithi163 ceph-mon[213841]: pgmap v1654: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:09.082 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:08 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1984457302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:34:09.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1543405602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:34:09.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:08 smithi163 ceph-mon[221067]: pgmap v1654: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:09.083 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:08 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1984457302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:34:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1543405602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:34:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:08 smithi181 ceph-mon[196979]: pgmap v1654: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:09.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:08 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1984457302' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:34:09.385 INFO:teuthology.orchestra.run.smithi163.stdout:425201762673 2023-10-09T04:34:09.877 INFO:teuthology.orchestra.run.smithi163.stdout:201863463416 2023-10-09T04:34:09.970 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:09 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2238578230' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:34:09.971 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:09 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2238578230' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:34:10.002 INFO:teuthology.orchestra.run.smithi163.stdout:360777253251 2023-10-09T04:34:10.116 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725051 got 167503725053 for osd.5 2023-10-09T04:34:10.116 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:10.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:09 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2238578230' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:34:10.832 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:10 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:34:10] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2023-10-09T04:34:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/4038445840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:34:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:10 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/2949024378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:34:11.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:10 smithi181 ceph-mon[196979]: pgmap v1655: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/4038445840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:34:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:10 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/2949024378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:34:11.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:10 smithi163 ceph-mon[221067]: pgmap v1655: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:11.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/4038445840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:34:11.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:10 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/2949024378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:34:11.333 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:10 smithi163 ceph-mon[213841]: pgmap v1655: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:11.436 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762671 got 425201762673 for osd.3 2023-10-09T04:34:11.436 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:11.491 INFO:tasks.cephadm.ceph_manager.ceph:need seq 360777253249 got 360777253251 for osd.0 2023-10-09T04:34:11.492 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:11.733 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201778 got 236223201780 for osd.7 2023-10-09T04:34:11.734 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:11.855 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463414 got 201863463416 for osd.6 2023-10-09T04:34:11.855 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:13.068 INFO:teuthology.orchestra.run.smithi163.stdout:382252089726 2023-10-09T04:34:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:13 smithi163 ceph-mon[221067]: pgmap v1656: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:13.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:13 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/948077030' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:34:13.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:13 smithi163 ceph-mon[213841]: pgmap v1656: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:13.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:13 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/948077030' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:34:13.468 INFO:teuthology.orchestra.run.smithi163.stdout:403726926200 2023-10-09T04:34:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:13 smithi181 ceph-mon[196979]: pgmap v1656: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:13.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:13 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/948077030' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:34:14.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:14 smithi181 ceph-mon[196979]: from='client.? 172.21.15.163:0/1941728960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:34:14.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:14 smithi163 ceph-mon[213841]: from='client.? 172.21.15.163:0/1941728960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:34:14.582 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:14 smithi163 ceph-mon[221067]: from='client.? 172.21.15.163:0/1941728960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:34:14.735 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089724 got 382252089726 for osd.1 2023-10-09T04:34:14.735 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:14.799 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926197 got 403726926200 for osd.2 2023-10-09T04:34:14.799 DEBUG:teuthology.parallel:result is None 2023-10-09T04:34:14.799 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:34:14.799 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:34:15.194 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:15.518 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:15 smithi181 ceph-mon[196979]: pgmap v1657: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:15.582 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:15 smithi163 ceph-mon[213841]: pgmap v1657: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:15.583 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:15 smithi163 ceph-mon[221067]: pgmap v1657: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:17.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[221067]: pgmap v1658: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:17.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.212 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:17.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[213841]: pgmap v1658: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:17.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.213 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:17.381 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:34:17.383 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:34:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:17 smithi181 ceph-mon[196979]: pgmap v1658: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:17.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:17 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.534 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:17.535 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:17.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:17.535 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:17 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' 2023-10-09T04:34:18.041 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1658,"stamp":"2023-10-09T04:34:16.162188+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5923,"num_read_kb":10077,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":170,"num_bytes_recovered":2327398,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7421692,"kb_used_data":71036,"kb_used_omap":126,"kb_used_meta":7350465,"kb_avail":742539524,"statfs":{"total":767960285184,"available":760360472576,"internally_reserved":0,"allocated":72740864,"data_stored":55188240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7526876795},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003225"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":739,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102160+0000","last_change":"2023-10-09T04:33:31.249953+0000","last_active":"2023-10-09T04:33:43.102160+0000","last_peered":"2023-10-09T04:33:43.102160+0000","last_clean":"2023-10-09T04:33:43.102160+0000","last_became_active":"2023-10-09T04:33:31.248959+0000","last_became_peered":"2023-10-09T04:33:31.248959+0000","last_unstale":"2023-10-09T04:33:43.102160+0000","last_undegraded":"2023-10-09T04:33:43.102160+0000","last_fullsized":"2023-10-09T04:33:43.102160+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:32:30.709742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:32:30.709742+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:36.195749+0000","scrub_duration":0.001770293,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896242+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:33:42.896242+0000","last_peered":"2023-10-09T04:33:42.896242+0000","last_clean":"2023-10-09T04:33:42.896242+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:33:42.896242+0000","last_undegraded":"2023-10-09T04:33:42.896242+0000","last_fullsized":"2023-10-09T04:33:42.896242+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":376,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007052+0000","last_change":"2023-10-09T04:33:18.355505+0000","last_active":"2023-10-09T04:33:43.007052+0000","last_peered":"2023-10-09T04:33:43.007052+0000","last_clean":"2023-10-09T04:33:43.007052+0000","last_became_active":"2023-10-09T04:33:18.355208+0000","last_became_peered":"2023-10-09T04:33:18.355208+0000","last_unstale":"2023-10-09T04:33:43.007052+0000","last_undegraded":"2023-10-09T04:33:43.007052+0000","last_fullsized":"2023-10-09T04:33:43.007052+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:00.226450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":609,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102275+0000","last_change":"2023-10-09T04:33:31.249327+0000","last_active":"2023-10-09T04:33:43.102275+0000","last_peered":"2023-10-09T04:33:43.102275+0000","last_clean":"2023-10-09T04:33:43.102275+0000","last_became_active":"2023-10-09T04:33:31.248273+0000","last_became_peered":"2023-10-09T04:33:31.248273+0000","last_unstale":"2023-10-09T04:33:43.102275+0000","last_undegraded":"2023-10-09T04:33:43.102275+0000","last_fullsized":"2023-10-09T04:33:43.102275+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:16.054718+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":598,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101438+0000","last_change":"2023-10-09T04:33:05.892500+0000","last_active":"2023-10-09T04:33:43.101438+0000","last_peered":"2023-10-09T04:33:43.101438+0000","last_clean":"2023-10-09T04:33:43.101438+0000","last_became_active":"2023-10-09T04:33:05.892172+0000","last_became_peered":"2023-10-09T04:33:05.892172+0000","last_unstale":"2023-10-09T04:33:43.101438+0000","last_undegraded":"2023-10-09T04:33:43.101438+0000","last_fullsized":"2023-10-09T04:33:43.101438+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:13.971102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":268,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895702+0000","last_change":"2023-10-09T04:32:24.897913+0000","last_active":"2023-10-09T04:33:42.895702+0000","last_peered":"2023-10-09T04:33:42.895702+0000","last_clean":"2023-10-09T04:33:42.895702+0000","last_became_active":"2023-10-09T04:32:24.897593+0000","last_became_peered":"2023-10-09T04:32:24.897593+0000","last_unstale":"2023-10-09T04:33:42.895702+0000","last_undegraded":"2023-10-09T04:33:42.895702+0000","last_fullsized":"2023-10-09T04:33:42.895702+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:23.688984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":634,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896115+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:33:42.896115+0000","last_peered":"2023-10-09T04:33:42.896115+0000","last_clean":"2023-10-09T04:33:42.896115+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:33:42.896115+0000","last_undegraded":"2023-10-09T04:33:42.896115+0000","last_fullsized":"2023-10-09T04:33:42.896115+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006626+0000","last_change":"2023-10-09T04:33:18.355441+0000","last_active":"2023-10-09T04:33:43.006626+0000","last_peered":"2023-10-09T04:33:43.006626+0000","last_clean":"2023-10-09T04:33:43.006626+0000","last_became_active":"2023-10-09T04:33:18.355123+0000","last_became_peered":"2023-10-09T04:33:18.355123+0000","last_unstale":"2023-10-09T04:33:43.006626+0000","last_undegraded":"2023-10-09T04:33:43.006626+0000","last_fullsized":"2023-10-09T04:33:43.006626+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:11.222818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891591+0000","last_change":"2023-10-09T04:33:37.880393+0000","last_active":"2023-10-09T04:33:42.891591+0000","last_peered":"2023-10-09T04:33:42.891591+0000","last_clean":"2023-10-09T04:33:42.891591+0000","last_became_active":"2023-10-09T04:33:37.879424+0000","last_became_peered":"2023-10-09T04:33:37.879424+0000","last_unstale":"2023-10-09T04:33:42.891591+0000","last_undegraded":"2023-10-09T04:33:42.891591+0000","last_fullsized":"2023-10-09T04:33:42.891591+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:43.370893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":1101,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893534+0000","last_change":"2023-10-09T04:32:11.324189+0000","last_active":"2023-10-09T04:33:42.893534+0000","last_peered":"2023-10-09T04:33:42.893534+0000","last_clean":"2023-10-09T04:33:42.893534+0000","last_became_active":"2023-10-09T04:32:11.323586+0000","last_became_peered":"2023-10-09T04:32:11.323586+0000","last_unstale":"2023-10-09T04:33:42.893534+0000","last_undegraded":"2023-10-09T04:33:42.893534+0000","last_fullsized":"2023-10-09T04:33:42.893534+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:11.107882+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":294,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895896+0000","last_change":"2023-10-09T04:32:24.896712+0000","last_active":"2023-10-09T04:33:42.895896+0000","last_peered":"2023-10-09T04:33:42.895896+0000","last_clean":"2023-10-09T04:33:42.895896+0000","last_became_active":"2023-10-09T04:32:24.896402+0000","last_became_peered":"2023-10-09T04:32:24.896402+0000","last_unstale":"2023-10-09T04:33:42.895896+0000","last_undegraded":"2023-10-09T04:33:42.895896+0000","last_fullsized":"2023-10-09T04:33:42.895896+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:52.911652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006435+0000","last_change":"2023-10-09T04:32:46.901434+0000","last_active":"2023-10-09T04:33:43.006435+0000","last_peered":"2023-10-09T04:33:43.006435+0000","last_clean":"2023-10-09T04:33:43.006435+0000","last_became_active":"2023-10-09T04:32:46.901097+0000","last_became_peered":"2023-10-09T04:32:46.901097+0000","last_unstale":"2023-10-09T04:33:43.006435+0000","last_undegraded":"2023-10-09T04:33:43.006435+0000","last_fullsized":"2023-10-09T04:33:43.006435+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:46.419780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":784,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.888023+0000","last_change":"2023-10-09T04:33:13.419941+0000","last_active":"2023-10-09T04:33:42.888023+0000","last_peered":"2023-10-09T04:33:42.888023+0000","last_clean":"2023-10-09T04:33:42.888023+0000","last_became_active":"2023-10-09T04:33:11.922864+0000","last_became_peered":"2023-10-09T04:33:11.922864+0000","last_unstale":"2023-10-09T04:33:42.888023+0000","last_undegraded":"2023-10-09T04:33:42.888023+0000","last_fullsized":"2023-10-09T04:33:42.888023+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:33:13.419844+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:33:13.419844+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:13.536384+0000","scrub_duration":0.0016237630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":137,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102446+0000","last_change":"2023-10-09T04:33:31.248390+0000","last_active":"2023-10-09T04:33:43.102446+0000","last_peered":"2023-10-09T04:33:43.102446+0000","last_clean":"2023-10-09T04:33:43.102446+0000","last_became_active":"2023-10-09T04:33:31.248053+0000","last_became_peered":"2023-10-09T04:33:31.248053+0000","last_unstale":"2023-10-09T04:33:43.102446+0000","last_undegraded":"2023-10-09T04:33:43.102446+0000","last_fullsized":"2023-10-09T04:33:43.102446+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:27.926582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":76,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006433+0000","last_change":"2023-10-09T04:32:46.907020+0000","last_active":"2023-10-09T04:33:43.006433+0000","last_peered":"2023-10-09T04:33:43.006433+0000","last_clean":"2023-10-09T04:33:43.006433+0000","last_became_active":"2023-10-09T04:32:46.906737+0000","last_became_peered":"2023-10-09T04:32:46.906737+0000","last_unstale":"2023-10-09T04:33:43.006433+0000","last_undegraded":"2023-10-09T04:33:43.006433+0000","last_fullsized":"2023-10-09T04:33:43.006433+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:42.668505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":750,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102537+0000","last_change":"2023-10-09T04:33:31.249124+0000","last_active":"2023-10-09T04:33:43.102537+0000","last_peered":"2023-10-09T04:33:43.102537+0000","last_clean":"2023-10-09T04:33:43.102537+0000","last_became_active":"2023-10-09T04:33:31.248816+0000","last_became_peered":"2023-10-09T04:33:31.248816+0000","last_unstale":"2023-10-09T04:33:43.102537+0000","last_undegraded":"2023-10-09T04:33:43.102537+0000","last_fullsized":"2023-10-09T04:33:43.102537+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:36.862471+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":748,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893783+0000","last_change":"2023-10-09T04:32:24.924362+0000","last_active":"2023-10-09T04:33:42.893783+0000","last_peered":"2023-10-09T04:33:42.893783+0000","last_clean":"2023-10-09T04:33:42.893783+0000","last_became_active":"2023-10-09T04:32:24.904276+0000","last_became_peered":"2023-10-09T04:32:24.904276+0000","last_unstale":"2023-10-09T04:33:42.893783+0000","last_undegraded":"2023-10-09T04:33:42.893783+0000","last_fullsized":"2023-10-09T04:33:42.893783+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:36.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":191,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101467+0000","last_change":"2023-10-09T04:33:06.299862+0000","last_active":"2023-10-09T04:33:43.101467+0000","last_peered":"2023-10-09T04:33:43.101467+0000","last_clean":"2023-10-09T04:33:43.101467+0000","last_became_active":"2023-10-09T04:33:06.299518+0000","last_became_peered":"2023-10-09T04:33:06.299518+0000","last_unstale":"2023-10-09T04:33:43.101467+0000","last_undegraded":"2023-10-09T04:33:43.101467+0000","last_fullsized":"2023-10-09T04:33:43.101467+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:53.622009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":825,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101740+0000","last_change":"2023-10-09T04:33:05.894578+0000","last_active":"2023-10-09T04:33:43.101740+0000","last_peered":"2023-10-09T04:33:43.101740+0000","last_clean":"2023-10-09T04:33:43.101740+0000","last_became_active":"2023-10-09T04:33:05.894248+0000","last_became_peered":"2023-10-09T04:33:05.894248+0000","last_unstale":"2023-10-09T04:33:43.101740+0000","last_undegraded":"2023-10-09T04:33:43.101740+0000","last_fullsized":"2023-10-09T04:33:43.101740+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:51.918596+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":257,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895670+0000","last_change":"2023-10-09T04:32:46.924180+0000","last_active":"2023-10-09T04:33:42.895670+0000","last_peered":"2023-10-09T04:33:42.895670+0000","last_clean":"2023-10-09T04:33:42.895670+0000","last_became_active":"2023-10-09T04:32:46.912929+0000","last_became_peered":"2023-10-09T04:32:46.912929+0000","last_unstale":"2023-10-09T04:33:42.895670+0000","last_undegraded":"2023-10-09T04:33:42.895670+0000","last_fullsized":"2023-10-09T04:33:42.895670+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:32.893089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":237,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895619+0000","last_change":"2023-10-09T04:32:46.914254+0000","last_active":"2023-10-09T04:33:42.895619+0000","last_peered":"2023-10-09T04:33:42.895619+0000","last_clean":"2023-10-09T04:33:42.895619+0000","last_became_active":"2023-10-09T04:32:46.913997+0000","last_became_peered":"2023-10-09T04:32:46.913997+0000","last_unstale":"2023-10-09T04:33:42.895619+0000","last_undegraded":"2023-10-09T04:33:42.895619+0000","last_fullsized":"2023-10-09T04:33:42.895619+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:37.049011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":744,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895792+0000","last_change":"2023-10-09T04:32:27.906425+0000","last_active":"2023-10-09T04:33:42.895792+0000","last_peered":"2023-10-09T04:33:42.895792+0000","last_clean":"2023-10-09T04:33:42.895792+0000","last_became_active":"2023-10-09T04:32:27.905962+0000","last_became_peered":"2023-10-09T04:32:27.905962+0000","last_unstale":"2023-10-09T04:33:42.895792+0000","last_undegraded":"2023-10-09T04:33:42.895792+0000","last_fullsized":"2023-10-09T04:33:42.895792+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:25.433020+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":685,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893221+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:33:42.893221+0000","last_peered":"2023-10-09T04:33:42.893221+0000","last_clean":"2023-10-09T04:33:42.893221+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:33:42.893221+0000","last_undegraded":"2023-10-09T04:33:42.893221+0000","last_fullsized":"2023-10-09T04:33:42.893221+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":410,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006851+0000","last_change":"2023-10-09T04:33:18.355460+0000","last_active":"2023-10-09T04:33:43.006851+0000","last_peered":"2023-10-09T04:33:43.006851+0000","last_clean":"2023-10-09T04:33:43.006851+0000","last_became_active":"2023-10-09T04:33:18.355181+0000","last_became_peered":"2023-10-09T04:33:18.355181+0000","last_unstale":"2023-10-09T04:33:43.006851+0000","last_undegraded":"2023-10-09T04:33:43.006851+0000","last_fullsized":"2023-10-09T04:33:43.006851+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:33.066128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":781,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102051+0000","last_change":"2023-10-09T04:33:31.249874+0000","last_active":"2023-10-09T04:33:43.102051+0000","last_peered":"2023-10-09T04:33:43.102051+0000","last_clean":"2023-10-09T04:33:43.102051+0000","last_became_active":"2023-10-09T04:33:31.248201+0000","last_became_peered":"2023-10-09T04:33:31.248201+0000","last_unstale":"2023-10-09T04:33:43.102051+0000","last_undegraded":"2023-10-09T04:33:43.102051+0000","last_fullsized":"2023-10-09T04:33:43.102051+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:30:46.778406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:30:46.778406+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:19.571151+0000","scrub_duration":0.001341243,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":267,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896019+0000","last_change":"2023-10-09T04:32:24.895666+0000","last_active":"2023-10-09T04:33:42.896019+0000","last_peered":"2023-10-09T04:33:42.896019+0000","last_clean":"2023-10-09T04:33:42.896019+0000","last_became_active":"2023-10-09T04:32:24.895214+0000","last_became_peered":"2023-10-09T04:32:24.895214+0000","last_unstale":"2023-10-09T04:33:42.896019+0000","last_undegraded":"2023-10-09T04:33:42.896019+0000","last_fullsized":"2023-10-09T04:33:42.896019+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:06.654301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":386,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102328+0000","last_change":"2023-10-09T04:33:31.250634+0000","last_active":"2023-10-09T04:33:43.102328+0000","last_peered":"2023-10-09T04:33:43.102328+0000","last_clean":"2023-10-09T04:33:43.102328+0000","last_became_active":"2023-10-09T04:33:31.250045+0000","last_became_peered":"2023-10-09T04:33:31.250045+0000","last_unstale":"2023-10-09T04:33:43.102328+0000","last_undegraded":"2023-10-09T04:33:43.102328+0000","last_fullsized":"2023-10-09T04:33:43.102328+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:59.436667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":546,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101627+0000","last_change":"2023-10-09T04:33:05.896882+0000","last_active":"2023-10-09T04:33:43.101627+0000","last_peered":"2023-10-09T04:33:43.101627+0000","last_clean":"2023-10-09T04:33:43.101627+0000","last_became_active":"2023-10-09T04:33:05.896613+0000","last_became_peered":"2023-10-09T04:33:05.896613+0000","last_unstale":"2023-10-09T04:33:43.101627+0000","last_undegraded":"2023-10-09T04:33:43.101627+0000","last_fullsized":"2023-10-09T04:33:43.101627+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:23.304877+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":204,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.901256+0000","last_change":"2023-10-09T04:33:43.901256+0000","last_active":"2023-10-09T04:33:43.901256+0000","last_peered":"2023-10-09T04:33:43.901256+0000","last_clean":"2023-10-09T04:33:43.901256+0000","last_became_active":"2023-10-09T04:33:43.900927+0000","last_became_peered":"2023-10-09T04:33:43.900927+0000","last_unstale":"2023-10-09T04:33:43.901256+0000","last_undegraded":"2023-10-09T04:33:43.901256+0000","last_fullsized":"2023-10-09T04:33:43.901256+0000","mapping_epoch":641,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:09.279606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":267,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895664+0000","last_change":"2023-10-09T04:32:24.897581+0000","last_active":"2023-10-09T04:33:42.895664+0000","last_peered":"2023-10-09T04:33:42.895664+0000","last_clean":"2023-10-09T04:33:42.895664+0000","last_became_active":"2023-10-09T04:32:24.897265+0000","last_became_peered":"2023-10-09T04:32:24.897265+0000","last_unstale":"2023-10-09T04:33:42.895664+0000","last_undegraded":"2023-10-09T04:33:42.895664+0000","last_fullsized":"2023-10-09T04:33:42.895664+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:46.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":545,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102113+0000","last_change":"2023-10-09T04:33:31.248609+0000","last_active":"2023-10-09T04:33:43.102113+0000","last_peered":"2023-10-09T04:33:43.102113+0000","last_clean":"2023-10-09T04:33:43.102113+0000","last_became_active":"2023-10-09T04:33:31.246775+0000","last_became_peered":"2023-10-09T04:33:31.246775+0000","last_unstale":"2023-10-09T04:33:43.102113+0000","last_undegraded":"2023-10-09T04:33:43.102113+0000","last_fullsized":"2023-10-09T04:33:43.102113+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:33:07.338110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:33:07.338110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:10.207892+0000","scrub_duration":0.011619229,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":343,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.890908+0000","last_change":"2023-10-09T04:33:37.878235+0000","last_active":"2023-10-09T04:33:42.890908+0000","last_peered":"2023-10-09T04:33:42.890908+0000","last_clean":"2023-10-09T04:33:42.890908+0000","last_became_active":"2023-10-09T04:33:37.877775+0000","last_became_peered":"2023-10-09T04:33:37.877775+0000","last_unstale":"2023-10-09T04:33:42.890908+0000","last_undegraded":"2023-10-09T04:33:42.890908+0000","last_fullsized":"2023-10-09T04:33:42.890908+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:37.189297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007190+0000","last_change":"2023-10-09T04:32:24.895968+0000","last_active":"2023-10-09T04:33:43.007190+0000","last_peered":"2023-10-09T04:33:43.007190+0000","last_clean":"2023-10-09T04:33:43.007190+0000","last_became_active":"2023-10-09T04:32:24.895621+0000","last_became_peered":"2023-10-09T04:32:24.895621+0000","last_unstale":"2023-10-09T04:33:43.007190+0000","last_undegraded":"2023-10-09T04:33:43.007190+0000","last_fullsized":"2023-10-09T04:33:43.007190+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:04.332236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":1084,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893580+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:33:42.893580+0000","last_peered":"2023-10-09T04:33:42.893580+0000","last_clean":"2023-10-09T04:33:42.893580+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:33:42.893580+0000","last_undegraded":"2023-10-09T04:33:42.893580+0000","last_fullsized":"2023-10-09T04:33:42.893580+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":686,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893876+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:33:42.893876+0000","last_peered":"2023-10-09T04:33:42.893876+0000","last_clean":"2023-10-09T04:33:42.893876+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:33:42.893876+0000","last_undegraded":"2023-10-09T04:33:42.893876+0000","last_fullsized":"2023-10-09T04:33:42.893876+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":458,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102422+0000","last_change":"2023-10-09T04:33:31.250105+0000","last_active":"2023-10-09T04:33:43.102422+0000","last_peered":"2023-10-09T04:33:43.102422+0000","last_clean":"2023-10-09T04:33:43.102422+0000","last_became_active":"2023-10-09T04:33:31.247987+0000","last_became_peered":"2023-10-09T04:33:31.247987+0000","last_unstale":"2023-10-09T04:33:43.102422+0000","last_undegraded":"2023-10-09T04:33:43.102422+0000","last_fullsized":"2023-10-09T04:33:43.102422+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:23.364398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":757,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007008+0000","last_change":"2023-10-09T04:32:49.906867+0000","last_active":"2023-10-09T04:33:43.007008+0000","last_peered":"2023-10-09T04:33:43.007008+0000","last_clean":"2023-10-09T04:33:43.007008+0000","last_became_active":"2023-10-09T04:32:49.906484+0000","last_became_peered":"2023-10-09T04:32:49.906484+0000","last_unstale":"2023-10-09T04:33:43.007008+0000","last_undegraded":"2023-10-09T04:33:43.007008+0000","last_fullsized":"2023-10-09T04:33:43.007008+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:32.775865+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2293760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":676,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893814+0000","last_change":"2023-10-09T04:30:52.027413+0000","last_active":"2023-10-09T04:33:42.893814+0000","last_peered":"2023-10-09T04:33:42.893814+0000","last_clean":"2023-10-09T04:33:42.893814+0000","last_became_active":"2023-10-09T04:30:51.982673+0000","last_became_peered":"2023-10-09T04:30:51.982673+0000","last_unstale":"2023-10-09T04:33:42.893814+0000","last_undegraded":"2023-10-09T04:33:42.893814+0000","last_fullsized":"2023-10-09T04:33:42.893814+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:22.580030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896167+0000","last_change":"2023-10-09T04:32:24.903965+0000","last_active":"2023-10-09T04:33:42.896167+0000","last_peered":"2023-10-09T04:33:42.896167+0000","last_clean":"2023-10-09T04:33:42.896167+0000","last_became_active":"2023-10-09T04:32:24.903426+0000","last_became_peered":"2023-10-09T04:32:24.903426+0000","last_unstale":"2023-10-09T04:33:42.896167+0000","last_undegraded":"2023-10-09T04:33:42.896167+0000","last_fullsized":"2023-10-09T04:33:42.896167+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:25.796771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":785,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891493+0000","last_change":"2023-10-09T04:33:37.881254+0000","last_active":"2023-10-09T04:33:42.891493+0000","last_peered":"2023-10-09T04:33:42.891493+0000","last_clean":"2023-10-09T04:33:42.891493+0000","last_became_active":"2023-10-09T04:33:37.880909+0000","last_became_peered":"2023-10-09T04:33:37.880909+0000","last_unstale":"2023-10-09T04:33:42.891493+0000","last_undegraded":"2023-10-09T04:33:42.891493+0000","last_fullsized":"2023-10-09T04:33:42.891493+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:13.893020+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":697,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893664+0000","last_change":"2023-10-09T04:32:46.925804+0000","last_active":"2023-10-09T04:33:42.893664+0000","last_peered":"2023-10-09T04:33:42.893664+0000","last_clean":"2023-10-09T04:33:42.893664+0000","last_became_active":"2023-10-09T04:32:46.908862+0000","last_became_peered":"2023-10-09T04:32:46.908862+0000","last_unstale":"2023-10-09T04:33:42.893664+0000","last_undegraded":"2023-10-09T04:33:42.893664+0000","last_fullsized":"2023-10-09T04:33:42.893664+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:05.607198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":340,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891191+0000","last_change":"2023-10-09T04:33:37.901902+0000","last_active":"2023-10-09T04:33:42.891191+0000","last_peered":"2023-10-09T04:33:42.891191+0000","last_clean":"2023-10-09T04:33:42.891191+0000","last_became_active":"2023-10-09T04:33:37.901578+0000","last_became_peered":"2023-10-09T04:33:37.901578+0000","last_unstale":"2023-10-09T04:33:42.891191+0000","last_undegraded":"2023-10-09T04:33:42.891191+0000","last_fullsized":"2023-10-09T04:33:42.891191+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:28.210734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":938,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895803+0000","last_change":"2023-10-09T04:32:27.911465+0000","last_active":"2023-10-09T04:33:42.895803+0000","last_peered":"2023-10-09T04:33:42.895803+0000","last_clean":"2023-10-09T04:33:42.895803+0000","last_became_active":"2023-10-09T04:32:27.911189+0000","last_became_peered":"2023-10-09T04:32:27.911189+0000","last_unstale":"2023-10-09T04:33:42.895803+0000","last_undegraded":"2023-10-09T04:33:42.895803+0000","last_fullsized":"2023-10-09T04:33:42.895803+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:23.798185+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102381+0000","last_change":"2023-10-09T04:33:31.250325+0000","last_active":"2023-10-09T04:33:43.102381+0000","last_peered":"2023-10-09T04:33:43.102381+0000","last_clean":"2023-10-09T04:33:43.102381+0000","last_became_active":"2023-10-09T04:33:31.248668+0000","last_became_peered":"2023-10-09T04:33:31.248668+0000","last_unstale":"2023-10-09T04:33:43.102381+0000","last_undegraded":"2023-10-09T04:33:43.102381+0000","last_fullsized":"2023-10-09T04:33:43.102381+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:38.097187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":267,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895712+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:33:42.895712+0000","last_peered":"2023-10-09T04:33:42.895712+0000","last_clean":"2023-10-09T04:33:42.895712+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:33:42.895712+0000","last_undegraded":"2023-10-09T04:33:42.895712+0000","last_fullsized":"2023-10-09T04:33:42.895712+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1165,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893017+0000","last_change":"2023-10-09T04:32:27.907423+0000","last_active":"2023-10-09T04:33:42.893017+0000","last_peered":"2023-10-09T04:33:42.893017+0000","last_clean":"2023-10-09T04:33:42.893017+0000","last_became_active":"2023-10-09T04:32:27.907058+0000","last_became_peered":"2023-10-09T04:32:27.907058+0000","last_unstale":"2023-10-09T04:33:42.893017+0000","last_undegraded":"2023-10-09T04:33:42.893017+0000","last_fullsized":"2023-10-09T04:33:42.893017+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:28.674711+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":272,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006702+0000","last_change":"2023-10-09T04:33:18.223152+0000","last_active":"2023-10-09T04:33:43.006702+0000","last_peered":"2023-10-09T04:33:43.006702+0000","last_clean":"2023-10-09T04:33:43.006702+0000","last_became_active":"2023-10-09T04:33:18.222815+0000","last_became_peered":"2023-10-09T04:33:18.222815+0000","last_unstale":"2023-10-09T04:33:43.006702+0000","last_undegraded":"2023-10-09T04:33:43.006702+0000","last_fullsized":"2023-10-09T04:33:43.006702+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:09.512377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":247,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895550+0000","last_change":"2023-10-09T04:32:46.912552+0000","last_active":"2023-10-09T04:33:42.895550+0000","last_peered":"2023-10-09T04:33:42.895550+0000","last_clean":"2023-10-09T04:33:42.895550+0000","last_became_active":"2023-10-09T04:32:46.912202+0000","last_became_peered":"2023-10-09T04:32:46.912202+0000","last_unstale":"2023-10-09T04:33:42.895550+0000","last_undegraded":"2023-10-09T04:33:42.895550+0000","last_fullsized":"2023-10-09T04:33:42.895550+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:26.746627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":833,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101912+0000","last_change":"2023-10-09T04:33:31.249644+0000","last_active":"2023-10-09T04:33:43.101912+0000","last_peered":"2023-10-09T04:33:43.101912+0000","last_clean":"2023-10-09T04:33:43.101912+0000","last_became_active":"2023-10-09T04:33:31.247737+0000","last_became_peered":"2023-10-09T04:33:31.247737+0000","last_unstale":"2023-10-09T04:33:43.101912+0000","last_undegraded":"2023-10-09T04:33:43.101912+0000","last_fullsized":"2023-10-09T04:33:43.101912+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:52.931370+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891056+0000","last_change":"2023-10-09T04:33:37.901844+0000","last_active":"2023-10-09T04:33:42.891056+0000","last_peered":"2023-10-09T04:33:42.891056+0000","last_clean":"2023-10-09T04:33:42.891056+0000","last_became_active":"2023-10-09T04:33:37.901518+0000","last_became_peered":"2023-10-09T04:33:37.901518+0000","last_unstale":"2023-10-09T04:33:42.891056+0000","last_undegraded":"2023-10-09T04:33:42.891056+0000","last_fullsized":"2023-10-09T04:33:42.891056+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:04.978549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":224,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895969+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:33:42.895969+0000","last_peered":"2023-10-09T04:33:42.895969+0000","last_clean":"2023-10-09T04:33:42.895969+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:33:42.895969+0000","last_undegraded":"2023-10-09T04:33:42.895969+0000","last_fullsized":"2023-10-09T04:33:42.895969+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":537,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101744+0000","last_change":"2023-10-09T04:33:05.892699+0000","last_active":"2023-10-09T04:33:43.101744+0000","last_peered":"2023-10-09T04:33:43.101744+0000","last_clean":"2023-10-09T04:33:43.101744+0000","last_became_active":"2023-10-09T04:33:05.892377+0000","last_became_peered":"2023-10-09T04:33:05.892377+0000","last_unstale":"2023-10-09T04:33:43.101744+0000","last_undegraded":"2023-10-09T04:33:43.101744+0000","last_fullsized":"2023-10-09T04:33:43.101744+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:20.040289+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896214+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:33:42.896214+0000","last_peered":"2023-10-09T04:33:42.896214+0000","last_clean":"2023-10-09T04:33:42.896214+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:33:42.896214+0000","last_undegraded":"2023-10-09T04:33:42.896214+0000","last_fullsized":"2023-10-09T04:33:42.896214+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":257,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896063+0000","last_change":"2023-10-09T04:32:46.913544+0000","last_active":"2023-10-09T04:33:42.896063+0000","last_peered":"2023-10-09T04:33:42.896063+0000","last_clean":"2023-10-09T04:33:42.896063+0000","last_became_active":"2023-10-09T04:32:46.912941+0000","last_became_peered":"2023-10-09T04:32:46.912941+0000","last_unstale":"2023-10-09T04:33:42.896063+0000","last_undegraded":"2023-10-09T04:33:42.896063+0000","last_fullsized":"2023-10-09T04:33:42.896063+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:02.930550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":790,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102003+0000","last_change":"2023-10-09T04:33:31.250999+0000","last_active":"2023-10-09T04:33:43.102003+0000","last_peered":"2023-10-09T04:33:43.102003+0000","last_clean":"2023-10-09T04:33:43.102003+0000","last_became_active":"2023-10-09T04:33:31.250203+0000","last_became_peered":"2023-10-09T04:33:31.250203+0000","last_unstale":"2023-10-09T04:33:43.102003+0000","last_undegraded":"2023-10-09T04:33:43.102003+0000","last_fullsized":"2023-10-09T04:33:43.102003+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:26.305117+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006724+0000","last_change":"2023-10-09T04:32:24.895007+0000","last_active":"2023-10-09T04:33:43.006724+0000","last_peered":"2023-10-09T04:33:43.006724+0000","last_clean":"2023-10-09T04:33:43.006724+0000","last_became_active":"2023-10-09T04:32:24.894679+0000","last_became_peered":"2023-10-09T04:32:24.894679+0000","last_unstale":"2023-10-09T04:33:43.006724+0000","last_undegraded":"2023-10-09T04:33:43.006724+0000","last_fullsized":"2023-10-09T04:33:43.006724+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:52.639216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":221,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.888279+0000","last_change":"2023-10-09T04:33:11.921041+0000","last_active":"2023-10-09T04:33:42.888279+0000","last_peered":"2023-10-09T04:33:42.888279+0000","last_clean":"2023-10-09T04:33:42.888279+0000","last_became_active":"2023-10-09T04:33:11.920740+0000","last_became_peered":"2023-10-09T04:33:11.920740+0000","last_unstale":"2023-10-09T04:33:42.888279+0000","last_undegraded":"2023-10-09T04:33:42.888279+0000","last_fullsized":"2023-10-09T04:33:42.888279+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:57.482646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":784,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891504+0000","last_change":"2023-10-09T04:33:37.880676+0000","last_active":"2023-10-09T04:33:42.891504+0000","last_peered":"2023-10-09T04:33:42.891504+0000","last_clean":"2023-10-09T04:33:42.891504+0000","last_became_active":"2023-10-09T04:33:37.880072+0000","last_became_peered":"2023-10-09T04:33:37.880072+0000","last_unstale":"2023-10-09T04:33:42.891504+0000","last_undegraded":"2023-10-09T04:33:42.891504+0000","last_fullsized":"2023-10-09T04:33:42.891504+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:33:24.202109+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:32:55.312916+0000","last_clean_scrub_stamp":"2023-10-09T04:33:24.202109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:13.979897+0000","scrub_duration":0.0015606909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"610'120","reported_seq":645,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101722+0000","last_change":"2023-10-09T04:33:05.892724+0000","last_active":"2023-10-09T04:33:43.101722+0000","last_peered":"2023-10-09T04:33:43.101722+0000","last_clean":"2023-10-09T04:33:43.101722+0000","last_became_active":"2023-10-09T04:33:05.892310+0000","last_became_peered":"2023-10-09T04:33:05.892310+0000","last_unstale":"2023-10-09T04:33:43.101722+0000","last_undegraded":"2023-10-09T04:33:43.101722+0000","last_fullsized":"2023-10-09T04:33:43.101722+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:48.391521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065711399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":557,"num_read_kb":2531,"num_write":123,"num_write_kb":1583,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":504,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102532+0000","last_change":"2023-10-09T04:33:31.250362+0000","last_active":"2023-10-09T04:33:43.102532+0000","last_peered":"2023-10-09T04:33:43.102532+0000","last_clean":"2023-10-09T04:33:43.102532+0000","last_became_active":"2023-10-09T04:33:31.249187+0000","last_became_peered":"2023-10-09T04:33:31.249187+0000","last_unstale":"2023-10-09T04:33:43.102532+0000","last_undegraded":"2023-10-09T04:33:43.102532+0000","last_fullsized":"2023-10-09T04:33:43.102532+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:23.409192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":432,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101501+0000","last_change":"2023-10-09T04:33:05.896838+0000","last_active":"2023-10-09T04:33:43.101501+0000","last_peered":"2023-10-09T04:33:43.101501+0000","last_clean":"2023-10-09T04:33:43.101501+0000","last_became_active":"2023-10-09T04:33:05.896519+0000","last_became_peered":"2023-10-09T04:33:05.896519+0000","last_unstale":"2023-10-09T04:33:43.101501+0000","last_undegraded":"2023-10-09T04:33:43.101501+0000","last_fullsized":"2023-10-09T04:33:43.101501+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:09:49.232498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":680,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102093+0000","last_change":"2023-10-09T04:33:31.246737+0000","last_active":"2023-10-09T04:33:43.102093+0000","last_peered":"2023-10-09T04:33:43.102093+0000","last_clean":"2023-10-09T04:33:43.102093+0000","last_became_active":"2023-10-09T04:33:31.246335+0000","last_became_peered":"2023-10-09T04:33:31.246335+0000","last_unstale":"2023-10-09T04:33:43.102093+0000","last_undegraded":"2023-10-09T04:33:43.102093+0000","last_fullsized":"2023-10-09T04:33:43.102093+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:02.380113+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"609'113","reported_seq":682,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102418+0000","last_change":"2023-10-09T04:33:31.250149+0000","last_active":"2023-10-09T04:33:43.102418+0000","last_peered":"2023-10-09T04:33:43.102418+0000","last_clean":"2023-10-09T04:33:43.102418+0000","last_became_active":"2023-10-09T04:33:31.247147+0000","last_became_peered":"2023-10-09T04:33:31.247147+0000","last_unstale":"2023-10-09T04:33:43.102418+0000","last_undegraded":"2023-10-09T04:33:43.102418+0000","last_fullsized":"2023-10-09T04:33:43.102418+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:53.540748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035804899999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":921,"num_read_kb":1093,"num_write":84,"num_write_kb":2221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895671+0000","last_change":"2023-10-09T04:32:46.915115+0000","last_active":"2023-10-09T04:33:42.895671+0000","last_peered":"2023-10-09T04:33:42.895671+0000","last_clean":"2023-10-09T04:33:42.895671+0000","last_became_active":"2023-10-09T04:32:46.914599+0000","last_became_peered":"2023-10-09T04:32:46.914599+0000","last_unstale":"2023-10-09T04:33:42.895671+0000","last_undegraded":"2023-10-09T04:33:42.895671+0000","last_fullsized":"2023-10-09T04:33:42.895671+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:07:34.079842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":114,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895485+0000","last_change":"2023-10-09T04:32:24.897206+0000","last_active":"2023-10-09T04:33:42.895485+0000","last_peered":"2023-10-09T04:33:42.895485+0000","last_clean":"2023-10-09T04:33:42.895485+0000","last_became_active":"2023-10-09T04:32:24.896906+0000","last_became_peered":"2023-10-09T04:32:24.896906+0000","last_unstale":"2023-10-09T04:33:42.895485+0000","last_undegraded":"2023-10-09T04:33:42.895485+0000","last_fullsized":"2023-10-09T04:33:42.895485+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.452604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":637,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.901374+0000","last_change":"2023-10-09T04:33:43.901374+0000","last_active":"2023-10-09T04:33:43.901374+0000","last_peered":"2023-10-09T04:33:43.901374+0000","last_clean":"2023-10-09T04:33:43.901374+0000","last_became_active":"2023-10-09T04:33:43.901068+0000","last_became_peered":"2023-10-09T04:33:43.901068+0000","last_unstale":"2023-10-09T04:33:43.901374+0000","last_undegraded":"2023-10-09T04:33:43.901374+0000","last_fullsized":"2023-10-09T04:33:43.901374+0000","mapping_epoch":641,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:08.333067+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"609'60","reported_seq":689,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.887881+0000","last_change":"2023-10-09T04:33:11.921468+0000","last_active":"2023-10-09T04:33:42.887881+0000","last_peered":"2023-10-09T04:33:42.887881+0000","last_clean":"2023-10-09T04:33:42.887881+0000","last_became_active":"2023-10-09T04:33:11.920922+0000","last_became_peered":"2023-10-09T04:33:11.920922+0000","last_unstale":"2023-10-09T04:33:42.887881+0000","last_undegraded":"2023-10-09T04:33:42.887881+0000","last_fullsized":"2023-10-09T04:33:42.887881+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":60,"log_dups_size":0,"ondisk_log_size":60,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:37:58.105236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052562699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":217,"num_read_kb":1147,"num_write":66,"num_write_kb":2092,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":16,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":541,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101349+0000","last_change":"2023-10-09T04:32:46.897368+0000","last_active":"2023-10-09T04:33:43.101349+0000","last_peered":"2023-10-09T04:33:43.101349+0000","last_clean":"2023-10-09T04:33:43.101349+0000","last_became_active":"2023-10-09T04:32:46.897021+0000","last_became_peered":"2023-10-09T04:32:46.897021+0000","last_unstale":"2023-10-09T04:33:43.101349+0000","last_undegraded":"2023-10-09T04:33:43.101349+0000","last_fullsized":"2023-10-09T04:33:43.101349+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:10.435389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":211,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101857+0000","last_change":"2023-10-09T04:33:05.892505+0000","last_active":"2023-10-09T04:33:43.101857+0000","last_peered":"2023-10-09T04:33:43.101857+0000","last_clean":"2023-10-09T04:33:43.101857+0000","last_became_active":"2023-10-09T04:33:05.892229+0000","last_became_peered":"2023-10-09T04:33:05.892229+0000","last_unstale":"2023-10-09T04:33:43.101857+0000","last_undegraded":"2023-10-09T04:33:43.101857+0000","last_fullsized":"2023-10-09T04:33:43.101857+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:54.364822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"610'264","reported_seq":953,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102330+0000","last_change":"2023-10-09T04:33:31.250569+0000","last_active":"2023-10-09T04:33:43.102330+0000","last_peered":"2023-10-09T04:33:43.102330+0000","last_clean":"2023-10-09T04:33:43.102330+0000","last_became_active":"2023-10-09T04:33:31.248970+0000","last_became_peered":"2023-10-09T04:33:31.248970+0000","last_unstale":"2023-10-09T04:33:43.102330+0000","last_undegraded":"2023-10-09T04:33:43.102330+0000","last_fullsized":"2023-10-09T04:33:43.102330+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:46.730123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001223699,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1597,"num_read_kb":1410,"num_write":240,"num_write_kb":158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":1136,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.892988+0000","last_change":"2023-10-09T04:32:49.912670+0000","last_active":"2023-10-09T04:33:42.892988+0000","last_peered":"2023-10-09T04:33:42.892988+0000","last_clean":"2023-10-09T04:33:42.892988+0000","last_became_active":"2023-10-09T04:32:49.912299+0000","last_became_peered":"2023-10-09T04:32:49.912299+0000","last_unstale":"2023-10-09T04:33:42.892988+0000","last_undegraded":"2023-10-09T04:33:42.892988+0000","last_fullsized":"2023-10-09T04:33:42.892988+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:50.176300+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":123,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006437+0000","last_change":"2023-10-09T04:32:46.901306+0000","last_active":"2023-10-09T04:33:43.006437+0000","last_peered":"2023-10-09T04:33:43.006437+0000","last_clean":"2023-10-09T04:33:43.006437+0000","last_became_active":"2023-10-09T04:32:46.901052+0000","last_became_peered":"2023-10-09T04:32:46.901052+0000","last_unstale":"2023-10-09T04:33:43.006437+0000","last_undegraded":"2023-10-09T04:33:43.006437+0000","last_fullsized":"2023-10-09T04:33:43.006437+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:41.288918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":204,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891298+0000","last_change":"2023-10-09T04:33:37.879982+0000","last_active":"2023-10-09T04:33:42.891298+0000","last_peered":"2023-10-09T04:33:42.891298+0000","last_clean":"2023-10-09T04:33:42.891298+0000","last_became_active":"2023-10-09T04:33:37.879660+0000","last_became_peered":"2023-10-09T04:33:37.879660+0000","last_unstale":"2023-10-09T04:33:42.891298+0000","last_undegraded":"2023-10-09T04:33:42.891298+0000","last_fullsized":"2023-10-09T04:33:42.891298+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:33.416521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":564,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101724+0000","last_change":"2023-10-09T04:33:05.898564+0000","last_active":"2023-10-09T04:33:43.101724+0000","last_peered":"2023-10-09T04:33:43.101724+0000","last_clean":"2023-10-09T04:33:43.101724+0000","last_became_active":"2023-10-09T04:33:05.898217+0000","last_became_peered":"2023-10-09T04:33:05.898217+0000","last_unstale":"2023-10-09T04:33:43.101724+0000","last_undegraded":"2023-10-09T04:33:43.101724+0000","last_fullsized":"2023-10-09T04:33:43.101724+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:35.314212+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"610'198","reported_seq":1550,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893012+0000","last_change":"2023-10-09T04:32:32.836552+0000","last_active":"2023-10-09T04:33:42.893012+0000","last_peered":"2023-10-09T04:33:42.893012+0000","last_clean":"2023-10-09T04:33:42.893012+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:33:42.893012+0000","last_undegraded":"2023-10-09T04:33:42.893012+0000","last_fullsized":"2023-10-09T04:33:42.893012+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":198,"log_dups_size":0,"ondisk_log_size":198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051653199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1649,"num_read_kb":1774,"num_write":137,"num_write_kb":233,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":313,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895813+0000","last_change":"2023-10-09T04:32:24.899713+0000","last_active":"2023-10-09T04:33:42.895813+0000","last_peered":"2023-10-09T04:33:42.895813+0000","last_clean":"2023-10-09T04:33:42.895813+0000","last_became_active":"2023-10-09T04:32:24.899312+0000","last_became_peered":"2023-10-09T04:32:24.899312+0000","last_unstale":"2023-10-09T04:33:42.895813+0000","last_undegraded":"2023-10-09T04:33:42.895813+0000","last_fullsized":"2023-10-09T04:33:42.895813+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:49.724726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":268,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.899401+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:33:43.899401+0000","last_peered":"2023-10-09T04:33:43.899401+0000","last_clean":"2023-10-09T04:33:43.899401+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:33:43.899401+0000","last_undegraded":"2023-10-09T04:33:43.899401+0000","last_fullsized":"2023-10-09T04:33:43.899401+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":555,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891505+0000","last_change":"2023-10-09T04:33:37.876943+0000","last_active":"2023-10-09T04:33:42.891505+0000","last_peered":"2023-10-09T04:33:42.891505+0000","last_clean":"2023-10-09T04:33:42.891505+0000","last_became_active":"2023-10-09T04:33:37.876607+0000","last_became_peered":"2023-10-09T04:33:37.876607+0000","last_unstale":"2023-10-09T04:33:42.891505+0000","last_undegraded":"2023-10-09T04:33:42.891505+0000","last_fullsized":"2023-10-09T04:33:42.891505+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:24:46.067430+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"609'65","reported_seq":869,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893714+0000","last_change":"2023-10-09T04:32:32.837194+0000","last_active":"2023-10-09T04:33:42.893714+0000","last_peered":"2023-10-09T04:33:42.893714+0000","last_clean":"2023-10-09T04:33:42.893714+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:33:42.893714+0000","last_undegraded":"2023-10-09T04:33:42.893714+0000","last_fullsized":"2023-10-09T04:33:42.893714+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042491300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":301,"num_read_kb":234,"num_write":65,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":394,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101867+0000","last_change":"2023-10-09T04:32:35.957013+0000","last_active":"2023-10-09T04:33:43.101867+0000","last_peered":"2023-10-09T04:33:43.101867+0000","last_clean":"2023-10-09T04:33:43.101867+0000","last_became_active":"2023-10-09T04:32:35.956726+0000","last_became_peered":"2023-10-09T04:32:35.956726+0000","last_unstale":"2023-10-09T04:33:43.101867+0000","last_undegraded":"2023-10-09T04:33:43.101867+0000","last_fullsized":"2023-10-09T04:33:43.101867+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:35.311809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":226,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.899378+0000","last_change":"2023-10-09T04:32:35.960473+0000","last_active":"2023-10-09T04:33:43.899378+0000","last_peered":"2023-10-09T04:33:43.899378+0000","last_clean":"2023-10-09T04:33:43.899378+0000","last_became_active":"2023-10-09T04:32:35.960255+0000","last_became_peered":"2023-10-09T04:32:35.960255+0000","last_unstale":"2023-10-09T04:33:43.899378+0000","last_undegraded":"2023-10-09T04:33:43.899378+0000","last_fullsized":"2023-10-09T04:33:43.899378+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:27.363502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":759,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895971+0000","last_change":"2023-10-09T04:32:49.911515+0000","last_active":"2023-10-09T04:33:42.895971+0000","last_peered":"2023-10-09T04:33:42.895971+0000","last_clean":"2023-10-09T04:33:42.895971+0000","last_became_active":"2023-10-09T04:32:49.911135+0000","last_became_peered":"2023-10-09T04:32:49.911135+0000","last_unstale":"2023-10-09T04:33:42.895971+0000","last_undegraded":"2023-10-09T04:33:42.895971+0000","last_fullsized":"2023-10-09T04:33:42.895971+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.520940+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":611,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007152+0000","last_change":"2023-10-09T04:33:18.223300+0000","last_active":"2023-10-09T04:33:43.007152+0000","last_peered":"2023-10-09T04:33:43.007152+0000","last_clean":"2023-10-09T04:33:43.007152+0000","last_became_active":"2023-10-09T04:33:18.223023+0000","last_became_peered":"2023-10-09T04:33:18.223023+0000","last_unstale":"2023-10-09T04:33:43.007152+0000","last_undegraded":"2023-10-09T04:33:43.007152+0000","last_fullsized":"2023-10-09T04:33:43.007152+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:44.099916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"610'100","reported_seq":530,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101882+0000","last_change":"2023-10-09T04:33:05.897728+0000","last_active":"2023-10-09T04:33:43.101882+0000","last_peered":"2023-10-09T04:33:43.101882+0000","last_clean":"2023-10-09T04:33:43.101882+0000","last_became_active":"2023-10-09T04:33:05.897410+0000","last_became_peered":"2023-10-09T04:33:05.897410+0000","last_unstale":"2023-10-09T04:33:43.101882+0000","last_undegraded":"2023-10-09T04:33:43.101882+0000","last_fullsized":"2023-10-09T04:33:43.101882+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:05.093600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049773700000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":433,"num_read_kb":514,"num_write":78,"num_write_kb":4235,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":10240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":541,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006536+0000","last_change":"2023-10-09T04:33:18.224403+0000","last_active":"2023-10-09T04:33:43.006536+0000","last_peered":"2023-10-09T04:33:43.006536+0000","last_clean":"2023-10-09T04:33:43.006536+0000","last_became_active":"2023-10-09T04:33:18.224079+0000","last_became_peered":"2023-10-09T04:33:18.224079+0000","last_unstale":"2023-10-09T04:33:43.006536+0000","last_undegraded":"2023-10-09T04:33:43.006536+0000","last_fullsized":"2023-10-09T04:33:43.006536+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:02:21.446044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":817,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006978+0000","last_change":"2023-10-09T04:32:27.906600+0000","last_active":"2023-10-09T04:33:43.006978+0000","last_peered":"2023-10-09T04:33:43.006978+0000","last_clean":"2023-10-09T04:33:43.006978+0000","last_became_active":"2023-10-09T04:32:27.906224+0000","last_became_peered":"2023-10-09T04:32:27.906224+0000","last_unstale":"2023-10-09T04:33:43.006978+0000","last_undegraded":"2023-10-09T04:33:43.006978+0000","last_fullsized":"2023-10-09T04:33:43.006978+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:31:59.104786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:31:59.104786+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:38.393804+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"609'56","reported_seq":50,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891653+0000","last_change":"2023-10-09T04:33:37.880966+0000","last_active":"2023-10-09T04:33:42.891653+0000","last_peered":"2023-10-09T04:33:42.891653+0000","last_clean":"2023-10-09T04:33:42.891653+0000","last_became_active":"2023-10-09T04:33:37.880325+0000","last_became_peered":"2023-10-09T04:33:37.880325+0000","last_unstale":"2023-10-09T04:33:42.891653+0000","last_undegraded":"2023-10-09T04:33:42.891653+0000","last_fullsized":"2023-10-09T04:33:42.891653+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":56,"log_dups_size":0,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:09.835733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0015777339999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":248,"num_read_kb":1374,"num_write":53,"num_write_kb":2138,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":280,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.899490+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:33:43.899490+0000","last_peered":"2023-10-09T04:33:43.899490+0000","last_clean":"2023-10-09T04:33:43.899490+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:33:43.899490+0000","last_undegraded":"2023-10-09T04:33:43.899490+0000","last_fullsized":"2023-10-09T04:33:43.899490+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896123+0000","last_change":"2023-10-09T04:32:46.912541+0000","last_active":"2023-10-09T04:33:42.896123+0000","last_peered":"2023-10-09T04:33:42.896123+0000","last_clean":"2023-10-09T04:33:42.896123+0000","last_became_active":"2023-10-09T04:32:46.912111+0000","last_became_peered":"2023-10-09T04:32:46.912111+0000","last_unstale":"2023-10-09T04:33:42.896123+0000","last_undegraded":"2023-10-09T04:33:42.896123+0000","last_fullsized":"2023-10-09T04:33:42.896123+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:14.456278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":702,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006574+0000","last_change":"2023-10-09T04:33:18.224514+0000","last_active":"2023-10-09T04:33:43.006574+0000","last_peered":"2023-10-09T04:33:43.006574+0000","last_clean":"2023-10-09T04:33:43.006574+0000","last_became_active":"2023-10-09T04:33:18.224032+0000","last_became_peered":"2023-10-09T04:33:18.224032+0000","last_unstale":"2023-10-09T04:33:43.006574+0000","last_undegraded":"2023-10-09T04:33:43.006574+0000","last_fullsized":"2023-10-09T04:33:43.006574+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:10.182910+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":276,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895952+0000","last_change":"2023-10-09T04:30:51.979500+0000","last_active":"2023-10-09T04:33:42.895952+0000","last_peered":"2023-10-09T04:33:42.895952+0000","last_clean":"2023-10-09T04:33:42.895952+0000","last_became_active":"2023-10-09T04:30:51.979149+0000","last_became_peered":"2023-10-09T04:30:51.979149+0000","last_unstale":"2023-10-09T04:33:42.895952+0000","last_undegraded":"2023-10-09T04:33:42.895952+0000","last_fullsized":"2023-10-09T04:33:42.895952+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:52.702260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":196,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:44.018840+0000","last_change":"2023-10-09T04:33:12.381121+0000","last_active":"2023-10-09T04:33:44.018840+0000","last_peered":"2023-10-09T04:33:44.018840+0000","last_clean":"2023-10-09T04:33:44.018840+0000","last_became_active":"2023-10-09T04:33:12.380752+0000","last_became_peered":"2023-10-09T04:33:12.380752+0000","last_unstale":"2023-10-09T04:33:44.018840+0000","last_undegraded":"2023-10-09T04:33:44.018840+0000","last_fullsized":"2023-10-09T04:33:44.018840+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:03.881486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":881,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007064+0000","last_change":"2023-10-09T04:33:18.221081+0000","last_active":"2023-10-09T04:33:43.007064+0000","last_peered":"2023-10-09T04:33:43.007064+0000","last_clean":"2023-10-09T04:33:43.007064+0000","last_became_active":"2023-10-09T04:33:18.220783+0000","last_became_peered":"2023-10-09T04:33:18.220783+0000","last_unstale":"2023-10-09T04:33:43.007064+0000","last_undegraded":"2023-10-09T04:33:43.007064+0000","last_fullsized":"2023-10-09T04:33:43.007064+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:24.537848+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":51,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102219+0000","last_change":"2023-10-09T04:33:31.243274+0000","last_active":"2023-10-09T04:33:43.102219+0000","last_peered":"2023-10-09T04:33:43.102219+0000","last_clean":"2023-10-09T04:33:43.102219+0000","last_became_active":"2023-10-09T04:33:31.242888+0000","last_became_peered":"2023-10-09T04:33:31.242888+0000","last_unstale":"2023-10-09T04:33:43.102219+0000","last_undegraded":"2023-10-09T04:33:43.102219+0000","last_fullsized":"2023-10-09T04:33:43.102219+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:40.681162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895547+0000","last_change":"2023-10-09T04:32:24.904229+0000","last_active":"2023-10-09T04:33:42.895547+0000","last_peered":"2023-10-09T04:33:42.895547+0000","last_clean":"2023-10-09T04:33:42.895547+0000","last_became_active":"2023-10-09T04:32:24.903889+0000","last_became_peered":"2023-10-09T04:32:24.903889+0000","last_unstale":"2023-10-09T04:33:42.895547+0000","last_undegraded":"2023-10-09T04:33:42.895547+0000","last_fullsized":"2023-10-09T04:33:42.895547+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:58.275008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":51,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102362+0000","last_change":"2023-10-09T04:33:31.251324+0000","last_active":"2023-10-09T04:33:43.102362+0000","last_peered":"2023-10-09T04:33:43.102362+0000","last_clean":"2023-10-09T04:33:43.102362+0000","last_became_active":"2023-10-09T04:33:31.251024+0000","last_became_peered":"2023-10-09T04:33:31.251024+0000","last_unstale":"2023-10-09T04:33:43.102362+0000","last_undegraded":"2023-10-09T04:33:43.102362+0000","last_fullsized":"2023-10-09T04:33:43.102362+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:04.659803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":164,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895349+0000","last_change":"2023-10-09T04:32:24.905833+0000","last_active":"2023-10-09T04:33:42.895349+0000","last_peered":"2023-10-09T04:33:42.895349+0000","last_clean":"2023-10-09T04:33:42.895349+0000","last_became_active":"2023-10-09T04:32:24.905447+0000","last_became_peered":"2023-10-09T04:32:24.905447+0000","last_unstale":"2023-10-09T04:33:42.895349+0000","last_undegraded":"2023-10-09T04:33:42.895349+0000","last_fullsized":"2023-10-09T04:33:42.895349+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:28.627681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":975,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893433+0000","last_change":"2023-10-09T04:32:49.908665+0000","last_active":"2023-10-09T04:33:42.893433+0000","last_peered":"2023-10-09T04:33:42.893433+0000","last_clean":"2023-10-09T04:33:42.893433+0000","last_became_active":"2023-10-09T04:32:49.908329+0000","last_became_peered":"2023-10-09T04:32:49.908329+0000","last_unstale":"2023-10-09T04:33:42.893433+0000","last_undegraded":"2023-10-09T04:33:42.893433+0000","last_fullsized":"2023-10-09T04:33:42.893433+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:34.358388+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":68,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007134+0000","last_change":"2023-10-09T04:33:18.220900+0000","last_active":"2023-10-09T04:33:43.007134+0000","last_peered":"2023-10-09T04:33:43.007134+0000","last_clean":"2023-10-09T04:33:43.007134+0000","last_became_active":"2023-10-09T04:33:18.220583+0000","last_became_peered":"2023-10-09T04:33:18.220583+0000","last_unstale":"2023-10-09T04:33:43.007134+0000","last_undegraded":"2023-10-09T04:33:43.007134+0000","last_fullsized":"2023-10-09T04:33:43.007134+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:26.503933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":494,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006804+0000","last_change":"2023-10-09T04:33:18.224318+0000","last_active":"2023-10-09T04:33:43.006804+0000","last_peered":"2023-10-09T04:33:43.006804+0000","last_clean":"2023-10-09T04:33:43.006804+0000","last_became_active":"2023-10-09T04:33:18.223688+0000","last_became_peered":"2023-10-09T04:33:18.223688+0000","last_unstale":"2023-10-09T04:33:43.006804+0000","last_undegraded":"2023-10-09T04:33:43.006804+0000","last_fullsized":"2023-10-09T04:33:43.006804+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:28.092920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":585,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101810+0000","last_change":"2023-10-09T04:33:05.897583+0000","last_active":"2023-10-09T04:33:43.101810+0000","last_peered":"2023-10-09T04:33:43.101810+0000","last_clean":"2023-10-09T04:33:43.101810+0000","last_became_active":"2023-10-09T04:33:05.897096+0000","last_became_peered":"2023-10-09T04:33:05.897096+0000","last_unstale":"2023-10-09T04:33:43.101810+0000","last_undegraded":"2023-10-09T04:33:43.101810+0000","last_fullsized":"2023-10-09T04:33:43.101810+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:36.724610+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":317,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895982+0000","last_change":"2023-10-09T04:32:07.865684+0000","last_active":"2023-10-09T04:33:42.895982+0000","last_peered":"2023-10-09T04:33:42.895982+0000","last_clean":"2023-10-09T04:33:42.895982+0000","last_became_active":"2023-10-09T04:32:07.849033+0000","last_became_peered":"2023-10-09T04:32:07.849033+0000","last_unstale":"2023-10-09T04:33:42.895982+0000","last_undegraded":"2023-10-09T04:33:42.895982+0000","last_fullsized":"2023-10-09T04:33:42.895982+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:08.096318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":257,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895277+0000","last_change":"2023-10-09T04:32:46.913746+0000","last_active":"2023-10-09T04:33:42.895277+0000","last_peered":"2023-10-09T04:33:42.895277+0000","last_clean":"2023-10-09T04:33:42.895277+0000","last_became_active":"2023-10-09T04:32:46.912066+0000","last_became_peered":"2023-10-09T04:32:46.912066+0000","last_unstale":"2023-10-09T04:33:42.895277+0000","last_undegraded":"2023-10-09T04:33:42.895277+0000","last_fullsized":"2023-10-09T04:33:42.895277+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:02.126877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":23344,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2293760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5923,"num_read_kb":10077,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10294,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201782,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910816,"kb_used_data":6808,"kb_used_omap":0,"kb_used_meta":904000,"kb_avail":92834336,"statfs":{"total":95995035648,"available":95062360064,"internally_reserved":0,"allocated":6971392,"data_stored":5291036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925696000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":7.7160000000000002,"15min":4.6150000000000002},"min":{"1min":0.36299999999999999,"5min":0.23899999999999999,"15min":0.21199999999999999},"max":{"1min":0.65600000000000003,"5min":218.697,"15min":218.697},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":7.6630000000000003,"15min":4.718},"min":{"1min":0.40300000000000002,"5min":0.255,"15min":0.221},"max":{"1min":0.85599999999999998,"5min":218.83099999999999,"15min":218.83099999999999},"last":0.44600000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":4.8609999999999998,"15min":5.7320000000000002},"min":{"1min":0.38300000000000001,"5min":0.26900000000000002,"15min":0.22600000000000001},"max":{"1min":0.99399999999999999,"5min":213.41800000000001,"15min":261.935},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":4.3010000000000002,"15min":5.4100000000000001},"min":{"1min":0.33400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.73399999999999999,"5min":213.352,"15min":262.07600000000002},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.589,"15min":2.4390000000000001},"min":{"1min":0.36399999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.70399999999999996,"5min":37.131999999999998,"15min":253.41200000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.6040000000000001,"15min":2.5840000000000001},"min":{"1min":0.443,"5min":0.33000000000000002,"15min":0.26800000000000002},"max":{"1min":0.68700000000000006,"5min":37.145000000000003,"15min":253.39400000000001},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 04:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":5.1790000000000003,"15min":3.702},"min":{"1min":0.314,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":1.6479999999999999,"5min":206.42500000000001,"15min":255.97300000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":5.0209999999999999,"15min":3.8079999999999998},"min":{"1min":0.49399999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":1.6619999999999999,"5min":206.44300000000001,"15min":256.06099999999998},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.73899999999999999,"15min":0.84799999999999998},"min":{"1min":0.40300000000000002,"5min":0.26800000000000002,"15min":0.24099999999999999},"max":{"1min":0.70999999999999996,"5min":9.7490000000000006,"15min":9.7490000000000006},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84499999999999997,"15min":1.04},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.251},"max":{"1min":0.77100000000000002,"5min":4.4160000000000004,"15min":12.566000000000001},"last":0.67600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.70499999999999996,"15min":1.2350000000000001},"min":{"1min":0.38,"5min":0.28899999999999998,"15min":0.254},"max":{"1min":0.86799999999999999,"5min":5.0620000000000003,"15min":17.882999999999999},"last":0.501},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.85599999999999998,"15min":1.111},"min":{"1min":0.46100000000000002,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":0.86299999999999999,"5min":7.1230000000000002,"15min":17.568999999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.78100000000000003,"15min":1.1799999999999999},"min":{"1min":0.48199999999999998,"5min":0.318,"15min":0.31},"max":{"1min":0.83199999999999996,"5min":6.5380000000000003,"15min":9.5069999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.69599999999999995,"5min":1,"15min":1.044},"min":{"1min":0.56699999999999995,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84399999999999997,"5min":9.8089999999999993,"15min":9.8089999999999993},"last":0.51900000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463418,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914576,"kb_used_data":10952,"kb_used_omap":0,"kb_used_meta":903616,"kb_avail":92830576,"statfs":{"total":95995035648,"available":95058509824,"internally_reserved":0,"allocated":11214848,"data_stored":9549882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925302784},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":3.0609999999999999,"15min":4.3129999999999997},"min":{"1min":0.33100000000000002,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":0.71799999999999997,"5min":134.34399999999999,"15min":162.524},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.497,"5min":3.0859999999999999,"15min":4.2880000000000003},"min":{"1min":0.36399999999999999,"5min":0.245,"15min":0.218},"max":{"1min":0.61799999999999999,"5min":130.87100000000001,"15min":161.73599999999999},"last":0.60699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.548,"15min":4.1020000000000003},"min":{"1min":0.377,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":0.90800000000000003,"5min":10.055,"15min":315.73599999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.6020000000000001,"15min":4.2329999999999997},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":0.75800000000000001,"5min":13.603999999999999,"15min":316.91500000000002},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.5860000000000001,"15min":2.5},"min":{"1min":0.39400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.78400000000000003,"5min":11.74,"15min":116.084},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.6279999999999999,"15min":2.5449999999999999},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.27100000000000002},"max":{"1min":0.82099999999999995,"5min":11.772,"15min":116.11199999999999},"last":0.71199999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":2.657,"15min":2.0529999999999999},"min":{"1min":0.34200000000000003,"5min":0.28799999999999998,"15min":0.25600000000000001},"max":{"1min":0.65600000000000003,"5min":127.44199999999999,"15min":127.44199999999999},"last":0.625},{"interface":"front","average":{"1min":0.61799999999999999,"5min":2.9929999999999999,"15min":2.1190000000000002},"min":{"1min":0.45300000000000001,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":1.052,"5min":127.51900000000001,"15min":127.51900000000001},"last":0.63400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.6339999999999999,"15min":1.6839999999999999},"min":{"1min":0.35899999999999999,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":0.752,"5min":90.221999999999994,"15min":161.27500000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.505,"5min":1.7869999999999999,"15min":1.794},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.248},"max":{"1min":0.80400000000000005,"5min":90.116,"15min":160.32599999999999},"last":0.44700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.89900000000000002,"15min":1.1399999999999999},"min":{"1min":0.33800000000000002,"5min":0.28999999999999998,"15min":0.26100000000000001},"max":{"1min":0.871,"5min":6.6340000000000003,"15min":74.043000000000006},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.042,"15min":1.1930000000000001},"min":{"1min":0.39000000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":0.92000000000000004,"5min":10.021000000000001,"15min":74.073999999999998},"last":0.55200000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.625,"15min":1.72},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.92900000000000005,"5min":93.212000000000003,"15min":161.39400000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.9299999999999999,"15min":1.802},"min":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.32900000000000001},"max":{"1min":0.94199999999999995,"5min":93.257000000000005,"15min":161.41200000000001},"last":0.65500000000000003}]}]},{"osd":1,"up_from":89,"seq":382252089727,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920752,"kb_used_data":11752,"kb_used_omap":9,"kb_used_meta":908982,"kb_avail":92824400,"statfs":{"total":95995035648,"available":95052185600,"internally_reserved":0,"allocated":12034048,"data_stored":9370420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":930798569},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":2.0169999999999999,"15min":3.4390000000000001},"min":{"1min":0.218,"5min":0.185,"15min":0.17699999999999999},"max":{"1min":0.53000000000000003,"5min":33.351999999999997,"15min":259.26799999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.38600000000000001,"5min":2.3450000000000002,"15min":3.6459999999999999},"min":{"1min":0.28699999999999998,"5min":0.187,"15min":0.187},"max":{"1min":0.53400000000000003,"5min":35.271000000000001,"15min":258.46899999999999},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:34:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":1.651,"15min":2.9039999999999999},"min":{"1min":0.248,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":0.48699999999999999,"5min":12.959,"15min":259.315},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.36099999999999999,"5min":1.446,"15min":3.0630000000000002},"min":{"1min":0.25700000000000001,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":0.53700000000000003,"5min":17.718,"15min":259.733},"last":0.30399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":4.0940000000000003,"15min":2.7000000000000002},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.22600000000000001},"max":{"1min":0.55500000000000005,"5min":156.261,"15min":156.261},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":4.1399999999999997,"15min":2.8069999999999999},"min":{"1min":0.32700000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55400000000000005,"5min":155.684,"15min":155.684},"last":0.379}]},{"osd":4,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.361,"15min":2.6459999999999999},"min":{"1min":0.41699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.82399999999999995,"5min":17.780000000000001,"15min":257.83300000000003},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.716,"15min":2.5409999999999999},"min":{"1min":0.40400000000000003,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":0.753,"5min":14.32,"15min":257.863},"last":0.625}]},{"osd":5,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.411,"15min":1.5820000000000001},"min":{"1min":0.432,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":0.85499999999999998,"5min":17.751999999999999,"15min":25.696999999999999},"last":0.5},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.7749999999999999,"15min":1.7470000000000001},"min":{"1min":0.41899999999999998,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":0.77700000000000002,"5min":14.356999999999999,"15min":21.893999999999998},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":2.2589999999999999,"15min":1.8799999999999999},"min":{"1min":0.46100000000000002,"5min":0.39100000000000001,"15min":0.31900000000000001},"max":{"1min":0.77200000000000002,"5min":50.814999999999998,"15min":50.814999999999998},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.8220000000000001,"15min":1.625},"min":{"1min":0.44500000000000001,"5min":0.34300000000000003,"15min":0.29999999999999999},"max":{"1min":0.80700000000000005,"5min":50.749000000000002,"15min":50.749000000000002},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":2.7730000000000001,"15min":3.8140000000000001},"min":{"1min":0.48499999999999999,"5min":0.36499999999999999,"15min":0.34799999999999998},"max":{"1min":0.81000000000000005,"5min":66.388000000000005,"15min":259.08499999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.66300000000000003,"5min":2.4660000000000002,"15min":3.5609999999999999},"min":{"1min":0.44600000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84499999999999997,"5min":66.221999999999994,"15min":259.13299999999998},"last":0.54300000000000004}]}]},{"osd":0,"up_from":84,"seq":360777253252,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889636,"kb_used_data":8156,"kb_used_omap":28,"kb_used_meta":881443,"kb_avail":92855516,"statfs":{"total":95995035648,"available":95084048384,"internally_reserved":0,"allocated":8351744,"data_stored":5522239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":902598480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:33:32 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":2.149,"15min":2.9380000000000002},"min":{"1min":0.21299999999999999,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":0.72899999999999998,"5min":36.654000000000003,"15min":209.215},"last":0.317},{"interface":"front","average":{"1min":0.38400000000000001,"5min":2.0619999999999998,"15min":2.754},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.78300000000000003,"5min":36.832999999999998,"15min":95.861000000000004},"last":0.32800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":1.4299999999999999,"15min":2.2480000000000002},"min":{"1min":0.248,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":0.504,"5min":12.036,"15min":88.088999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.36199999999999999,"5min":1.2330000000000001,"15min":2.238},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.20599999999999999},"max":{"1min":0.51800000000000002,"5min":9.9619999999999997,"15min":88.004999999999995},"last":0.251}]},{"osd":3,"last update":"Mon Oct 9 04:33:32 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":2.2629999999999999,"15min":2.165},"min":{"1min":0.27100000000000002,"5min":0.20499999999999999,"15min":0.192},"max":{"1min":0.83899999999999997,"5min":19.507999999999999,"15min":87.891999999999996},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":2.4079999999999999,"15min":2.3130000000000002},"min":{"1min":0.29599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.92800000000000005,"5min":19.741,"15min":88.147999999999996},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 04:33:15 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.3540000000000001,"15min":2.29},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68600000000000005,"5min":11.153,"15min":88.058000000000007},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.538,"15min":1.9870000000000001},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.24199999999999999},"max":{"1min":0.79500000000000004,"5min":9.8810000000000002,"15min":86.891000000000005},"last":0.70399999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:33:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":2.6789999999999998,"15min":2.6429999999999998},"min":{"1min":0.38400000000000001,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":0.85899999999999999,"5min":84.983000000000004,"15min":134.93000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":2.569,"15min":2.4729999999999999},"min":{"1min":0.32400000000000001,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":0.76200000000000001,"5min":81.691999999999993,"15min":135.04599999999999},"last":0.76300000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:33:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.671,"15min":1.6359999999999999},"min":{"1min":0.40999999999999998,"5min":0.35099999999999998,"15min":0.34699999999999998},"max":{"1min":0.79300000000000004,"5min":8.4920000000000009,"15min":63.468000000000004},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.4530000000000001,"15min":1.9470000000000001},"min":{"1min":0.42299999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.76400000000000001,"5min":11.128,"15min":88.180000000000007},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:33:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":3.2000000000000002,"15min":2.2440000000000002},"min":{"1min":0.36899999999999999,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":0.80900000000000005,"5min":78.307000000000002,"15min":78.307000000000002},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":3.1080000000000001,"15min":2.6139999999999999},"min":{"1min":0.45000000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78100000000000003,"5min":78.216999999999999,"15min":88.120000000000005},"last":0.53100000000000003}]}]},{"osd":2,"up_from":94,"seq":403726926201,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842684,"kb_used_data":7924,"kb_used_omap":81,"kb_used_meta":834670,"kb_avail":92902468,"statfs":{"total":95995035648,"available":95132127232,"internally_reserved":0,"allocated":8114176,"data_stored":5436959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":854702298},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":3.3159999999999998,"15min":2.8460000000000001},"min":{"1min":0.219,"5min":0.18099999999999999,"15min":0.17699999999999999},"max":{"1min":0.625,"5min":163.09800000000001,"15min":163.09800000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":3.4969999999999999,"15min":3.129},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.55400000000000005,"5min":163.11699999999999,"15min":163.11699999999999},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":1.9950000000000001,"15min":3.754},"min":{"1min":0.26600000000000001,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":0.56899999999999995,"5min":21.007999999999999,"15min":158.13399999999999},"last":0.31},{"interface":"front","average":{"1min":0.36899999999999999,"5min":1.8220000000000001,"15min":3.5819999999999999},"min":{"1min":0.26400000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.76100000000000001,"5min":20.797999999999998,"15min":161.12},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.9850000000000001,"15min":1.966},"min":{"1min":0.27700000000000002,"5min":0.22700000000000001,"15min":0.20200000000000001},"max":{"1min":0.71999999999999997,"5min":43.707999999999998,"15min":51.524999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":2.5310000000000001,"15min":2.4129999999999998},"min":{"1min":0.312,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":0.622,"5min":43.427999999999997,"15min":51.505000000000003},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.2410000000000001,"15min":2.4169999999999998},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.26500000000000001},"max":{"1min":0.79500000000000004,"5min":9.4149999999999991,"15min":196.57599999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.1020000000000001,"15min":1.802},"min":{"1min":0.40000000000000002,"5min":0.30099999999999999,"15min":0.26800000000000002},"max":{"1min":1.853,"5min":9.0289999999999999,"15min":201.81100000000001},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.349,"15min":2.5169999999999999},"min":{"1min":0.42999999999999999,"5min":0.34499999999999997,"15min":0.28299999999999997},"max":{"1min":0.90700000000000003,"5min":8.6699999999999999,"15min":190.06899999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.321,"15min":2.8650000000000002},"min":{"1min":0.45700000000000002,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":0.88500000000000001,"5min":9.8949999999999996,"15min":190.08500000000001},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.92100000000000004,"15min":1.25},"min":{"1min":0.44700000000000001,"5min":0.35399999999999998,"15min":0.32900000000000001},"max":{"1min":0.84899999999999998,"5min":9.0779999999999994,"15min":58.469999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.4419999999999999,"15min":1.833},"min":{"1min":0.47199999999999998,"5min":0.39800000000000002,"15min":0.32100000000000001},"max":{"1min":0.755,"5min":8.6989999999999998,"15min":58.067999999999998},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.2609999999999999,"15min":2.1509999999999998},"min":{"1min":0.46800000000000003,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":1.9199999999999999,"5min":9.1120000000000001,"15min":191.792},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.5760000000000001,"15min":2.8999999999999999},"min":{"1min":0.48299999999999998,"5min":0.42099999999999999,"15min":0.34499999999999997},"max":{"1min":1.9670000000000001,"5min":8.843,"15min":191.828},"last":0.53000000000000003}]}]},{"osd":3,"up_from":99,"seq":425201762675,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123228,"kb_used_data":8020,"kb_used_omap":7,"kb_used_meta":1115192,"kb_avail":92621924,"statfs":{"total":95995035648,"available":94844850176,"internally_reserved":0,"allocated":8212480,"data_stored":5508981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1141957224},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":3.3410000000000002,"15min":3.427},"min":{"1min":0.248,"5min":0.189,"15min":0.17399999999999999},"max":{"1min":0.51300000000000001,"5min":119.149,"15min":119.97799999999999},"last":0.372},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.1050000000000004,"15min":3.9790000000000001},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.185},"max":{"1min":0.49399999999999999,"5min":119.251,"15min":120.307},"last":0.35299999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":3.2410000000000001,"15min":3.1709999999999998},"min":{"1min":0.247,"5min":0.19,"15min":0.19},"max":{"1min":0.501,"5min":121.842,"15min":170.196},"last":0.307},{"interface":"front","average":{"1min":0.36199999999999999,"5min":2.7999999999999998,"15min":2.9929999999999999},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":0.54400000000000004,"5min":119.673,"15min":166.953},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":1.2090000000000001,"15min":1.405},"min":{"1min":0.25900000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.49399999999999999,"5min":10.215999999999999,"15min":43.152999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":2.3490000000000002,"15min":2.0680000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.223},"max":{"1min":0.52500000000000002,"5min":17.463000000000001,"15min":43.134},"last":0.26000000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.1680000000000001,"15min":2.9260000000000002},"min":{"1min":0.39300000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":0.97299999999999998,"5min":83.010000000000005,"15min":211.84700000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.706,"15min":2.786},"min":{"1min":0.38700000000000001,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":0.95499999999999996,"5min":82.915000000000006,"15min":211.17500000000001},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":2.6970000000000001,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.33600000000000002,"15min":0.28100000000000003},"max":{"1min":0.999,"5min":88.295000000000002,"15min":88.295000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.56899999999999995,"5min":2.1850000000000001,"15min":1.5740000000000001},"min":{"1min":0.372,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.94199999999999995,"5min":86.968999999999994,"15min":86.968999999999994},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.3029999999999999,"15min":1.4279999999999999},"min":{"1min":0.432,"5min":0.36599999999999999,"15min":0.32600000000000001},"max":{"1min":0.88400000000000001,"5min":21.529,"15min":64.378},"last":0.497},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.169,"15min":1.857},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.35999999999999999},"max":{"1min":0.91400000000000003,"5min":21.666,"15min":64.353999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":3.7749999999999999,"15min":3.2749999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":1.018,"5min":86.897000000000006,"15min":212.62700000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":3.363,"15min":3.1739999999999999},"min":{"1min":0.47599999999999998,"5min":0.36099999999999999,"15min":0.33300000000000002},"max":{"1min":0.90000000000000002,"5min":87.105999999999995,"15min":212.04599999999999},"last":0.622}]}]},{"osd":4,"up_from":31,"seq":133143986691,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920660,"kb_used_data":10636,"kb_used_omap":0,"kb_used_meta":910016,"kb_avail":92824492,"statfs":{"total":95995035648,"available":95052279808,"internally_reserved":0,"allocated":10891264,"data_stored":9222317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931856384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":2.4020000000000001,"15min":4.6870000000000003},"min":{"1min":0.245,"5min":0.20999999999999999,"15min":0.20799999999999999},"max":{"1min":0.72899999999999998,"5min":110.95,"15min":207.45699999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":2.1739999999999999,"15min":4.734},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.223},"max":{"1min":1.111,"5min":111.057,"15min":207.114},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":2.0289999999999999,"15min":3.476},"min":{"1min":0.33500000000000002,"5min":0.23699999999999999,"15min":0.23599999999999999},"max":{"1min":1.2430000000000001,"5min":57.161999999999999,"15min":135.25399999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.956,"15min":3.391},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.252},"max":{"1min":1.8520000000000001,"5min":55.222999999999999,"15min":134.44300000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:34:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.496,"15min":3.9689999999999999},"min":{"1min":0.34699999999999998,"5min":0.28799999999999998,"15min":0.23599999999999999},"max":{"1min":7.54,"5min":9.5809999999999995,"15min":203.61500000000001},"last":0.439},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.4670000000000001,"15min":3.9870000000000001},"min":{"1min":0.33100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.7610000000000001,"5min":9.2599999999999998,"15min":203.601},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 04:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":5.524,"15min":3.5019999999999998},"min":{"1min":0.35699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":6.2389999999999999,"5min":135.97800000000001,"15min":154.13200000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":5.2910000000000004,"15min":3.4470000000000001},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.79000000000000004,"5min":135.899,"15min":154.077},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:33:17 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":1.028,"15min":0.93999999999999995},"min":{"1min":0.313,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":0.82299999999999995,"5min":16.753,"15min":16.753},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.504,"5min":1.0349999999999999,"15min":1.1180000000000001},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":0.70699999999999996,"5min":17.792000000000002,"15min":17.792000000000002},"last":0.52500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.93100000000000005,"15min":1.248},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.26700000000000002},"max":{"1min":1.0820000000000001,"5min":9.4120000000000008,"15min":47.801000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.88600000000000001,"15min":1.1339999999999999},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":1.1399999999999999,"5min":9.1760000000000002,"15min":47.753},"last":0.67600000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0740000000000001,"15min":1.115},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0549999999999999,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.03,"15min":1.29},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.28999999999999998},"max":{"1min":1.7669999999999999,"5min":12.863,"15min":12.863},"last":0.64500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725055,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899340,"kb_used_data":6788,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":92845812,"statfs":{"total":95995035648,"available":95074111488,"internally_reserved":0,"allocated":6950912,"data_stored":5286406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.9670000000000001,"15min":3.6579999999999999},"min":{"1min":0.34300000000000003,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.79200000000000004,"5min":64.186000000000007,"15min":139.86799999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":3.2909999999999999,"15min":3.851},"min":{"1min":0.39900000000000002,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":0.66500000000000004,"5min":63.746000000000002,"15min":139.84899999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":3.3410000000000002,"5min":5.9029999999999996,"15min":5.1909999999999998},"min":{"1min":0.33900000000000002,"5min":0.255,"15min":0.22600000000000001},"max":{"1min":63.229999999999997,"5min":131.86199999999999,"15min":296.23099999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":3.3490000000000002,"5min":5.649,"15min":5.2480000000000002},"min":{"1min":0.35399999999999998,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":63.212000000000003,"5min":132.74600000000001,"15min":295.065},"last":0.624}]},{"osd":2,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.655,"15min":3.1669999999999998},"min":{"1min":0.379,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72199999999999998,"5min":64.335999999999999,"15min":168.779},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.776,"15min":3.282},"min":{"1min":0.41299999999999998,"5min":0.28899999999999998,"15min":0.25600000000000001},"max":{"1min":0.68500000000000005,"5min":65.811999999999998,"15min":168.03100000000001},"last":0.61399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":3.8359999999999999,"15min":2.6499999999999999},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.27800000000000002},"max":{"1min":0.73799999999999999,"5min":116.56,"15min":116.56},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":3.7330000000000001,"15min":2.5790000000000002},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":0.76500000000000001,"5min":116.527,"15min":116.527},"last":0.76200000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.69599999999999995,"15min":1.206},"min":{"1min":0.33000000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":0.77500000000000002,"5min":8.0180000000000007,"15min":95.668000000000006},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.67100000000000004,"15min":1.1830000000000001},"min":{"1min":0.36299999999999999,"5min":0.27900000000000003,"15min":0.26100000000000001},"max":{"1min":0.65500000000000003,"5min":5.4580000000000002,"15min":98.703999999999994},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":2.0619999999999998,"15min":1.2549999999999999},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":0.79700000000000004,"5min":145.054,"15min":145.054},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":2.1760000000000002,"15min":1.3300000000000001},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.313},"max":{"1min":0.68600000000000005,"5min":144.85900000000001,"15min":144.85900000000001},"last":0.57699999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.76400000000000001,"15min":0.94999999999999996},"min":{"1min":0.41899999999999998,"5min":0.32100000000000001,"15min":0.26900000000000002},"max":{"1min":0.82099999999999995,"5min":8.282,"15min":9.3800000000000008},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.70899999999999996,"15min":1.036},"min":{"1min":0.47999999999999998,"5min":0.35299999999999998,"15min":0.33600000000000002},"max":{"1min":0.82299999999999995,"5min":3.7280000000000002,"15min":11.522},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:34:18.048 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:34:18.394 INFO:teuthology.orchestra.run.smithi163.stderr:Inferring config /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:18 smithi181 ceph-mon[196979]: from='client.47909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:34:19.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:18 smithi181 ceph-mon[196979]: pgmap v1659: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:19.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:18 smithi163 ceph-mon[221067]: from='client.47909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:34:19.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:18 smithi163 ceph-mon[221067]: pgmap v1659: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:19.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:18 smithi163 ceph-mon[213841]: from='client.47909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:34:19.332 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:18 smithi163 ceph-mon[213841]: pgmap v1659: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:19.985 INFO:teuthology.orchestra.run.smithi163.stdout: 2023-10-09T04:34:19.987 INFO:teuthology.orchestra.run.smithi163.stderr:dumped all 2023-10-09T04:34:20.184 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:19 smithi163 ceph-mon[213841]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:20.184 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:19 smithi163 ceph-mon[221067]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:20.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:19 smithi181 ceph-mon[196979]: from='mgr.34104 172.21.15.163:0/1113246033' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:20.651 INFO:teuthology.orchestra.run.smithi163.stdout:{"pg_ready":true,"pg_map":{"version":1659,"stamp":"2023-10-09T04:34:18.162699+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5923,"num_read_kb":10077,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":170,"num_bytes_recovered":2327398,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7421692,"kb_used_data":71036,"kb_used_omap":126,"kb_used_meta":7350465,"kb_avail":742539524,"statfs":{"total":767960285184,"available":760360472576,"internally_reserved":0,"allocated":72740864,"data_stored":55188240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":129413,"internal_metadata":7526876795},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003255"},"pg_stats":[{"pgid":"1.18","version":"80'161","reported_seq":739,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102160+0000","last_change":"2023-10-09T04:33:31.249953+0000","last_active":"2023-10-09T04:33:43.102160+0000","last_peered":"2023-10-09T04:33:43.102160+0000","last_clean":"2023-10-09T04:33:43.102160+0000","last_became_active":"2023-10-09T04:33:31.248959+0000","last_became_peered":"2023-10-09T04:33:31.248959+0000","last_unstale":"2023-10-09T04:33:43.102160+0000","last_undegraded":"2023-10-09T04:33:43.102160+0000","last_fullsized":"2023-10-09T04:33:43.102160+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:32:30.709742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:32:30.709742+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:36.195749+0000","scrub_duration":0.001770293,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896242+0000","last_change":"2023-10-09T04:26:21.359329+0000","last_active":"2023-10-09T04:33:42.896242+0000","last_peered":"2023-10-09T04:33:42.896242+0000","last_clean":"2023-10-09T04:33:42.896242+0000","last_became_active":"2023-10-09T04:26:21.359035+0000","last_became_peered":"2023-10-09T04:26:21.359035+0000","last_unstale":"2023-10-09T04:33:42.896242+0000","last_undegraded":"2023-10-09T04:33:42.896242+0000","last_fullsized":"2023-10-09T04:33:42.896242+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:21.267920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":376,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007052+0000","last_change":"2023-10-09T04:33:18.355505+0000","last_active":"2023-10-09T04:33:43.007052+0000","last_peered":"2023-10-09T04:33:43.007052+0000","last_clean":"2023-10-09T04:33:43.007052+0000","last_became_active":"2023-10-09T04:33:18.355208+0000","last_became_peered":"2023-10-09T04:33:18.355208+0000","last_unstale":"2023-10-09T04:33:43.007052+0000","last_undegraded":"2023-10-09T04:33:43.007052+0000","last_fullsized":"2023-10-09T04:33:43.007052+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:00.226450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"80'161","reported_seq":609,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102275+0000","last_change":"2023-10-09T04:33:31.249327+0000","last_active":"2023-10-09T04:33:43.102275+0000","last_peered":"2023-10-09T04:33:43.102275+0000","last_clean":"2023-10-09T04:33:43.102275+0000","last_became_active":"2023-10-09T04:33:31.248273+0000","last_became_peered":"2023-10-09T04:33:31.248273+0000","last_unstale":"2023-10-09T04:33:43.102275+0000","last_undegraded":"2023-10-09T04:33:43.102275+0000","last_fullsized":"2023-10-09T04:33:43.102275+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:42.386026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:42.386026+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:16.054718+0000","scrub_duration":0.0081376120000000007,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":598,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101438+0000","last_change":"2023-10-09T04:33:05.892500+0000","last_active":"2023-10-09T04:33:43.101438+0000","last_peered":"2023-10-09T04:33:43.101438+0000","last_clean":"2023-10-09T04:33:43.101438+0000","last_became_active":"2023-10-09T04:33:05.892172+0000","last_became_peered":"2023-10-09T04:33:05.892172+0000","last_unstale":"2023-10-09T04:33:43.101438+0000","last_undegraded":"2023-10-09T04:33:43.101438+0000","last_fullsized":"2023-10-09T04:33:43.101438+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:13.971102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":268,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895702+0000","last_change":"2023-10-09T04:32:24.897913+0000","last_active":"2023-10-09T04:33:42.895702+0000","last_peered":"2023-10-09T04:33:42.895702+0000","last_clean":"2023-10-09T04:33:42.895702+0000","last_became_active":"2023-10-09T04:32:24.897593+0000","last_became_peered":"2023-10-09T04:32:24.897593+0000","last_unstale":"2023-10-09T04:33:42.895702+0000","last_undegraded":"2023-10-09T04:33:42.895702+0000","last_fullsized":"2023-10-09T04:33:42.895702+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:23.688984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"80'161","reported_seq":634,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896115+0000","last_change":"2023-10-09T04:27:06.351112+0000","last_active":"2023-10-09T04:33:42.896115+0000","last_peered":"2023-10-09T04:33:42.896115+0000","last_clean":"2023-10-09T04:33:42.896115+0000","last_became_active":"2023-10-09T04:24:58.731682+0000","last_became_peered":"2023-10-09T04:24:58.731682+0000","last_unstale":"2023-10-09T04:33:42.896115+0000","last_undegraded":"2023-10-09T04:33:42.896115+0000","last_fullsized":"2023-10-09T04:33:42.896115+0000","mapping_epoch":447,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":448,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:06.351040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:06.351040+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.705337+0000","scrub_duration":0.0018279259999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006626+0000","last_change":"2023-10-09T04:33:18.355441+0000","last_active":"2023-10-09T04:33:43.006626+0000","last_peered":"2023-10-09T04:33:43.006626+0000","last_clean":"2023-10-09T04:33:43.006626+0000","last_became_active":"2023-10-09T04:33:18.355123+0000","last_became_peered":"2023-10-09T04:33:18.355123+0000","last_unstale":"2023-10-09T04:33:43.006626+0000","last_undegraded":"2023-10-09T04:33:43.006626+0000","last_fullsized":"2023-10-09T04:33:43.006626+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:11.222818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891591+0000","last_change":"2023-10-09T04:33:37.880393+0000","last_active":"2023-10-09T04:33:42.891591+0000","last_peered":"2023-10-09T04:33:42.891591+0000","last_clean":"2023-10-09T04:33:42.891591+0000","last_became_active":"2023-10-09T04:33:37.879424+0000","last_became_peered":"2023-10-09T04:33:37.879424+0000","last_unstale":"2023-10-09T04:33:42.891591+0000","last_undegraded":"2023-10-09T04:33:42.891591+0000","last_fullsized":"2023-10-09T04:33:42.891591+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:40:43.370893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"80'161","reported_seq":1101,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893534+0000","last_change":"2023-10-09T04:32:11.324189+0000","last_active":"2023-10-09T04:33:42.893534+0000","last_peered":"2023-10-09T04:33:42.893534+0000","last_clean":"2023-10-09T04:33:42.893534+0000","last_became_active":"2023-10-09T04:32:11.323586+0000","last_became_peered":"2023-10-09T04:32:11.323586+0000","last_unstale":"2023-10-09T04:33:42.893534+0000","last_undegraded":"2023-10-09T04:33:42.893534+0000","last_fullsized":"2023-10-09T04:33:42.893534+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.379427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.379427+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:10:11.107882+0000","scrub_duration":0.0042972720000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":294,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895896+0000","last_change":"2023-10-09T04:32:24.896712+0000","last_active":"2023-10-09T04:33:42.895896+0000","last_peered":"2023-10-09T04:33:42.895896+0000","last_clean":"2023-10-09T04:33:42.895896+0000","last_became_active":"2023-10-09T04:32:24.896402+0000","last_became_peered":"2023-10-09T04:32:24.896402+0000","last_unstale":"2023-10-09T04:33:42.895896+0000","last_undegraded":"2023-10-09T04:33:42.895896+0000","last_fullsized":"2023-10-09T04:33:42.895896+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:52.911652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006435+0000","last_change":"2023-10-09T04:32:46.901434+0000","last_active":"2023-10-09T04:33:43.006435+0000","last_peered":"2023-10-09T04:33:43.006435+0000","last_clean":"2023-10-09T04:33:43.006435+0000","last_became_active":"2023-10-09T04:32:46.901097+0000","last_became_peered":"2023-10-09T04:32:46.901097+0000","last_unstale":"2023-10-09T04:33:43.006435+0000","last_undegraded":"2023-10-09T04:33:43.006435+0000","last_fullsized":"2023-10-09T04:33:43.006435+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:46.419780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"80'161","reported_seq":784,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.888023+0000","last_change":"2023-10-09T04:33:13.419941+0000","last_active":"2023-10-09T04:33:42.888023+0000","last_peered":"2023-10-09T04:33:42.888023+0000","last_clean":"2023-10-09T04:33:42.888023+0000","last_became_active":"2023-10-09T04:33:11.922864+0000","last_became_peered":"2023-10-09T04:33:11.922864+0000","last_unstale":"2023-10-09T04:33:42.888023+0000","last_undegraded":"2023-10-09T04:33:42.888023+0000","last_fullsized":"2023-10-09T04:33:42.888023+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:33:13.419844+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:33:13.419844+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:42:13.536384+0000","scrub_duration":0.0016237630000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":137,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102446+0000","last_change":"2023-10-09T04:33:31.248390+0000","last_active":"2023-10-09T04:33:43.102446+0000","last_peered":"2023-10-09T04:33:43.102446+0000","last_clean":"2023-10-09T04:33:43.102446+0000","last_became_active":"2023-10-09T04:33:31.248053+0000","last_became_peered":"2023-10-09T04:33:31.248053+0000","last_unstale":"2023-10-09T04:33:43.102446+0000","last_undegraded":"2023-10-09T04:33:43.102446+0000","last_fullsized":"2023-10-09T04:33:43.102446+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:27.926582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":76,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006433+0000","last_change":"2023-10-09T04:32:46.907020+0000","last_active":"2023-10-09T04:33:43.006433+0000","last_peered":"2023-10-09T04:33:43.006433+0000","last_clean":"2023-10-09T04:33:43.006433+0000","last_became_active":"2023-10-09T04:32:46.906737+0000","last_became_peered":"2023-10-09T04:32:46.906737+0000","last_unstale":"2023-10-09T04:33:43.006433+0000","last_undegraded":"2023-10-09T04:33:43.006433+0000","last_fullsized":"2023-10-09T04:33:43.006433+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:42.668505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"80'161","reported_seq":750,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102537+0000","last_change":"2023-10-09T04:33:31.249124+0000","last_active":"2023-10-09T04:33:43.102537+0000","last_peered":"2023-10-09T04:33:43.102537+0000","last_clean":"2023-10-09T04:33:43.102537+0000","last_became_active":"2023-10-09T04:33:31.248816+0000","last_became_peered":"2023-10-09T04:33:31.248816+0000","last_unstale":"2023-10-09T04:33:43.102537+0000","last_undegraded":"2023-10-09T04:33:43.102537+0000","last_fullsized":"2023-10-09T04:33:43.102537+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:46.322377+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:46.322377+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:36.862471+0000","scrub_duration":0.0014816619999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":748,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893783+0000","last_change":"2023-10-09T04:32:24.924362+0000","last_active":"2023-10-09T04:33:42.893783+0000","last_peered":"2023-10-09T04:33:42.893783+0000","last_clean":"2023-10-09T04:33:42.893783+0000","last_became_active":"2023-10-09T04:32:24.904276+0000","last_became_peered":"2023-10-09T04:32:24.904276+0000","last_unstale":"2023-10-09T04:33:42.893783+0000","last_undegraded":"2023-10-09T04:33:42.893783+0000","last_fullsized":"2023-10-09T04:33:42.893783+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:36.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":191,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101467+0000","last_change":"2023-10-09T04:33:06.299862+0000","last_active":"2023-10-09T04:33:43.101467+0000","last_peered":"2023-10-09T04:33:43.101467+0000","last_clean":"2023-10-09T04:33:43.101467+0000","last_became_active":"2023-10-09T04:33:06.299518+0000","last_became_peered":"2023-10-09T04:33:06.299518+0000","last_unstale":"2023-10-09T04:33:43.101467+0000","last_undegraded":"2023-10-09T04:33:43.101467+0000","last_fullsized":"2023-10-09T04:33:43.101467+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:53.622009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"80'161","reported_seq":825,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101740+0000","last_change":"2023-10-09T04:33:05.894578+0000","last_active":"2023-10-09T04:33:43.101740+0000","last_peered":"2023-10-09T04:33:43.101740+0000","last_clean":"2023-10-09T04:33:43.101740+0000","last_became_active":"2023-10-09T04:33:05.894248+0000","last_became_peered":"2023-10-09T04:33:05.894248+0000","last_unstale":"2023-10-09T04:33:43.101740+0000","last_undegraded":"2023-10-09T04:33:43.101740+0000","last_fullsized":"2023-10-09T04:33:43.101740+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:25.748410+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:25.748410+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:51.918596+0000","scrub_duration":0.0072797879999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"80'22","reported_seq":257,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895670+0000","last_change":"2023-10-09T04:32:46.924180+0000","last_active":"2023-10-09T04:33:42.895670+0000","last_peered":"2023-10-09T04:33:42.895670+0000","last_clean":"2023-10-09T04:33:42.895670+0000","last_became_active":"2023-10-09T04:32:46.912929+0000","last_became_peered":"2023-10-09T04:32:46.912929+0000","last_unstale":"2023-10-09T04:33:42.895670+0000","last_undegraded":"2023-10-09T04:33:42.895670+0000","last_fullsized":"2023-10-09T04:33:42.895670+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:32.893089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":237,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895619+0000","last_change":"2023-10-09T04:32:46.914254+0000","last_active":"2023-10-09T04:33:42.895619+0000","last_peered":"2023-10-09T04:33:42.895619+0000","last_clean":"2023-10-09T04:33:42.895619+0000","last_became_active":"2023-10-09T04:32:46.913997+0000","last_became_peered":"2023-10-09T04:32:46.913997+0000","last_unstale":"2023-10-09T04:33:42.895619+0000","last_undegraded":"2023-10-09T04:33:42.895619+0000","last_fullsized":"2023-10-09T04:33:42.895619+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:37.049011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"80'161","reported_seq":744,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895792+0000","last_change":"2023-10-09T04:32:27.906425+0000","last_active":"2023-10-09T04:33:42.895792+0000","last_peered":"2023-10-09T04:33:42.895792+0000","last_clean":"2023-10-09T04:33:42.895792+0000","last_became_active":"2023-10-09T04:32:27.905962+0000","last_became_peered":"2023-10-09T04:32:27.905962+0000","last_unstale":"2023-10-09T04:33:42.895792+0000","last_undegraded":"2023-10-09T04:33:42.895792+0000","last_fullsized":"2023-10-09T04:33:42.895792+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:02.906102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:02.906102+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:25.433020+0000","scrub_duration":0.104106807,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":685,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893221+0000","last_change":"2023-10-09T04:26:21.355104+0000","last_active":"2023-10-09T04:33:42.893221+0000","last_peered":"2023-10-09T04:33:42.893221+0000","last_clean":"2023-10-09T04:33:42.893221+0000","last_became_active":"2023-10-09T04:26:21.354692+0000","last_became_peered":"2023-10-09T04:26:21.354692+0000","last_unstale":"2023-10-09T04:33:42.893221+0000","last_undegraded":"2023-10-09T04:33:42.893221+0000","last_fullsized":"2023-10-09T04:33:42.893221+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:04:50.795594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":410,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006851+0000","last_change":"2023-10-09T04:33:18.355460+0000","last_active":"2023-10-09T04:33:43.006851+0000","last_peered":"2023-10-09T04:33:43.006851+0000","last_clean":"2023-10-09T04:33:43.006851+0000","last_became_active":"2023-10-09T04:33:18.355181+0000","last_became_peered":"2023-10-09T04:33:18.355181+0000","last_unstale":"2023-10-09T04:33:43.006851+0000","last_undegraded":"2023-10-09T04:33:43.006851+0000","last_fullsized":"2023-10-09T04:33:43.006851+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:33.066128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"80'161","reported_seq":781,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102051+0000","last_change":"2023-10-09T04:33:31.249874+0000","last_active":"2023-10-09T04:33:43.102051+0000","last_peered":"2023-10-09T04:33:43.102051+0000","last_clean":"2023-10-09T04:33:43.102051+0000","last_became_active":"2023-10-09T04:33:31.248201+0000","last_became_peered":"2023-10-09T04:33:31.248201+0000","last_unstale":"2023-10-09T04:33:43.102051+0000","last_undegraded":"2023-10-09T04:33:43.102051+0000","last_fullsized":"2023-10-09T04:33:43.102051+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:30:46.778406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:30:46.778406+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:19.571151+0000","scrub_duration":0.001341243,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":267,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896019+0000","last_change":"2023-10-09T04:32:24.895666+0000","last_active":"2023-10-09T04:33:42.896019+0000","last_peered":"2023-10-09T04:33:42.896019+0000","last_clean":"2023-10-09T04:33:42.896019+0000","last_became_active":"2023-10-09T04:32:24.895214+0000","last_became_peered":"2023-10-09T04:32:24.895214+0000","last_unstale":"2023-10-09T04:33:42.896019+0000","last_undegraded":"2023-10-09T04:33:42.896019+0000","last_fullsized":"2023-10-09T04:33:42.896019+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:06.654301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":386,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102328+0000","last_change":"2023-10-09T04:33:31.250634+0000","last_active":"2023-10-09T04:33:43.102328+0000","last_peered":"2023-10-09T04:33:43.102328+0000","last_clean":"2023-10-09T04:33:43.102328+0000","last_became_active":"2023-10-09T04:33:31.250045+0000","last_became_peered":"2023-10-09T04:33:31.250045+0000","last_unstale":"2023-10-09T04:33:43.102328+0000","last_undegraded":"2023-10-09T04:33:43.102328+0000","last_fullsized":"2023-10-09T04:33:43.102328+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:59.436667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"80'161","reported_seq":546,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101627+0000","last_change":"2023-10-09T04:33:05.896882+0000","last_active":"2023-10-09T04:33:43.101627+0000","last_peered":"2023-10-09T04:33:43.101627+0000","last_clean":"2023-10-09T04:33:43.101627+0000","last_became_active":"2023-10-09T04:33:05.896613+0000","last_became_peered":"2023-10-09T04:33:05.896613+0000","last_unstale":"2023-10-09T04:33:43.101627+0000","last_undegraded":"2023-10-09T04:33:43.101627+0000","last_fullsized":"2023-10-09T04:33:43.101627+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:19.682021+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:19.682021+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:23.304877+0000","scrub_duration":0.0036000120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":204,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.901256+0000","last_change":"2023-10-09T04:33:43.901256+0000","last_active":"2023-10-09T04:33:43.901256+0000","last_peered":"2023-10-09T04:33:43.901256+0000","last_clean":"2023-10-09T04:33:43.901256+0000","last_became_active":"2023-10-09T04:33:43.900927+0000","last_became_peered":"2023-10-09T04:33:43.900927+0000","last_unstale":"2023-10-09T04:33:43.901256+0000","last_undegraded":"2023-10-09T04:33:43.901256+0000","last_fullsized":"2023-10-09T04:33:43.901256+0000","mapping_epoch":641,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:09:09.279606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":267,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895664+0000","last_change":"2023-10-09T04:32:24.897581+0000","last_active":"2023-10-09T04:33:42.895664+0000","last_peered":"2023-10-09T04:33:42.895664+0000","last_clean":"2023-10-09T04:33:42.895664+0000","last_became_active":"2023-10-09T04:32:24.897265+0000","last_became_peered":"2023-10-09T04:32:24.897265+0000","last_unstale":"2023-10-09T04:33:42.895664+0000","last_undegraded":"2023-10-09T04:33:42.895664+0000","last_fullsized":"2023-10-09T04:33:42.895664+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:46.105426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"80'161","reported_seq":545,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102113+0000","last_change":"2023-10-09T04:33:31.248609+0000","last_active":"2023-10-09T04:33:43.102113+0000","last_peered":"2023-10-09T04:33:43.102113+0000","last_clean":"2023-10-09T04:33:43.102113+0000","last_became_active":"2023-10-09T04:33:31.246775+0000","last_became_peered":"2023-10-09T04:33:31.246775+0000","last_unstale":"2023-10-09T04:33:43.102113+0000","last_undegraded":"2023-10-09T04:33:43.102113+0000","last_fullsized":"2023-10-09T04:33:43.102113+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:33:07.338110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:33:07.338110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:44:10.207892+0000","scrub_duration":0.011619229,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":343,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.890908+0000","last_change":"2023-10-09T04:33:37.878235+0000","last_active":"2023-10-09T04:33:42.890908+0000","last_peered":"2023-10-09T04:33:42.890908+0000","last_clean":"2023-10-09T04:33:42.890908+0000","last_became_active":"2023-10-09T04:33:37.877775+0000","last_became_peered":"2023-10-09T04:33:37.877775+0000","last_unstale":"2023-10-09T04:33:42.890908+0000","last_undegraded":"2023-10-09T04:33:42.890908+0000","last_fullsized":"2023-10-09T04:33:42.890908+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:37.189297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007190+0000","last_change":"2023-10-09T04:32:24.895968+0000","last_active":"2023-10-09T04:33:43.007190+0000","last_peered":"2023-10-09T04:33:43.007190+0000","last_clean":"2023-10-09T04:33:43.007190+0000","last_became_active":"2023-10-09T04:32:24.895621+0000","last_became_peered":"2023-10-09T04:32:24.895621+0000","last_unstale":"2023-10-09T04:33:43.007190+0000","last_undegraded":"2023-10-09T04:33:43.007190+0000","last_fullsized":"2023-10-09T04:33:43.007190+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:04.332236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"80'161","reported_seq":1084,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893580+0000","last_change":"2023-10-09T04:25:03.598242+0000","last_active":"2023-10-09T04:33:42.893580+0000","last_peered":"2023-10-09T04:33:42.893580+0000","last_clean":"2023-10-09T04:33:42.893580+0000","last_became_active":"2023-10-09T04:25:03.597922+0000","last_became_peered":"2023-10-09T04:25:03.597922+0000","last_unstale":"2023-10-09T04:33:42.893580+0000","last_undegraded":"2023-10-09T04:33:42.893580+0000","last_fullsized":"2023-10-09T04:33:42.893580+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:14.471107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:14.471107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:09.415273+0000","scrub_duration":0.0053749089999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":686,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893876+0000","last_change":"2023-10-09T04:24:56.206981+0000","last_active":"2023-10-09T04:33:42.893876+0000","last_peered":"2023-10-09T04:33:42.893876+0000","last_clean":"2023-10-09T04:33:42.893876+0000","last_became_active":"2023-10-09T04:24:56.182759+0000","last_became_peered":"2023-10-09T04:24:56.182759+0000","last_unstale":"2023-10-09T04:33:42.893876+0000","last_undegraded":"2023-10-09T04:33:42.893876+0000","last_fullsized":"2023-10-09T04:33:42.893876+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:41.356258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":458,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102422+0000","last_change":"2023-10-09T04:33:31.250105+0000","last_active":"2023-10-09T04:33:43.102422+0000","last_peered":"2023-10-09T04:33:43.102422+0000","last_clean":"2023-10-09T04:33:43.102422+0000","last_became_active":"2023-10-09T04:33:31.247987+0000","last_became_peered":"2023-10-09T04:33:31.247987+0000","last_unstale":"2023-10-09T04:33:43.102422+0000","last_undegraded":"2023-10-09T04:33:43.102422+0000","last_fullsized":"2023-10-09T04:33:43.102422+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:23.364398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"80'161","reported_seq":757,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007008+0000","last_change":"2023-10-09T04:32:49.906867+0000","last_active":"2023-10-09T04:33:43.007008+0000","last_peered":"2023-10-09T04:33:43.007008+0000","last_clean":"2023-10-09T04:33:43.007008+0000","last_became_active":"2023-10-09T04:32:49.906484+0000","last_became_peered":"2023-10-09T04:32:49.906484+0000","last_unstale":"2023-10-09T04:33:43.007008+0000","last_undegraded":"2023-10-09T04:33:43.007008+0000","last_fullsized":"2023-10-09T04:33:43.007008+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:23.453189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:23.453189+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:32.775865+0000","scrub_duration":0.0058243009999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2293760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":676,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893814+0000","last_change":"2023-10-09T04:30:52.027413+0000","last_active":"2023-10-09T04:33:42.893814+0000","last_peered":"2023-10-09T04:33:42.893814+0000","last_clean":"2023-10-09T04:33:42.893814+0000","last_became_active":"2023-10-09T04:30:51.982673+0000","last_became_peered":"2023-10-09T04:30:51.982673+0000","last_unstale":"2023-10-09T04:33:42.893814+0000","last_undegraded":"2023-10-09T04:33:42.893814+0000","last_fullsized":"2023-10-09T04:33:42.893814+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:22.580030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896167+0000","last_change":"2023-10-09T04:32:24.903965+0000","last_active":"2023-10-09T04:33:42.896167+0000","last_peered":"2023-10-09T04:33:42.896167+0000","last_clean":"2023-10-09T04:33:42.896167+0000","last_became_active":"2023-10-09T04:32:24.903426+0000","last_became_peered":"2023-10-09T04:32:24.903426+0000","last_unstale":"2023-10-09T04:33:42.896167+0000","last_undegraded":"2023-10-09T04:33:42.896167+0000","last_fullsized":"2023-10-09T04:33:42.896167+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:25.796771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"80'161","reported_seq":785,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891493+0000","last_change":"2023-10-09T04:33:37.881254+0000","last_active":"2023-10-09T04:33:42.891493+0000","last_peered":"2023-10-09T04:33:42.891493+0000","last_clean":"2023-10-09T04:33:42.891493+0000","last_became_active":"2023-10-09T04:33:37.880909+0000","last_became_peered":"2023-10-09T04:33:37.880909+0000","last_unstale":"2023-10-09T04:33:42.891493+0000","last_undegraded":"2023-10-09T04:33:42.891493+0000","last_fullsized":"2023-10-09T04:33:42.891493+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:47.346773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:47.346773+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:13.893020+0000","scrub_duration":0.0097114200000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":697,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893664+0000","last_change":"2023-10-09T04:32:46.925804+0000","last_active":"2023-10-09T04:33:42.893664+0000","last_peered":"2023-10-09T04:33:42.893664+0000","last_clean":"2023-10-09T04:33:42.893664+0000","last_became_active":"2023-10-09T04:32:46.908862+0000","last_became_peered":"2023-10-09T04:32:46.908862+0000","last_unstale":"2023-10-09T04:33:42.893664+0000","last_undegraded":"2023-10-09T04:33:42.893664+0000","last_fullsized":"2023-10-09T04:33:42.893664+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:05.607198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":340,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891191+0000","last_change":"2023-10-09T04:33:37.901902+0000","last_active":"2023-10-09T04:33:42.891191+0000","last_peered":"2023-10-09T04:33:42.891191+0000","last_clean":"2023-10-09T04:33:42.891191+0000","last_became_active":"2023-10-09T04:33:37.901578+0000","last_became_peered":"2023-10-09T04:33:37.901578+0000","last_unstale":"2023-10-09T04:33:42.891191+0000","last_undegraded":"2023-10-09T04:33:42.891191+0000","last_fullsized":"2023-10-09T04:33:42.891191+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:28.210734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"80'161","reported_seq":938,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895803+0000","last_change":"2023-10-09T04:32:27.911465+0000","last_active":"2023-10-09T04:33:42.895803+0000","last_peered":"2023-10-09T04:33:42.895803+0000","last_clean":"2023-10-09T04:33:42.895803+0000","last_became_active":"2023-10-09T04:32:27.911189+0000","last_became_peered":"2023-10-09T04:32:27.911189+0000","last_unstale":"2023-10-09T04:33:42.895803+0000","last_undegraded":"2023-10-09T04:33:42.895803+0000","last_fullsized":"2023-10-09T04:33:42.895803+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:04.271706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:04.271706+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:23.798185+0000","scrub_duration":0.001449044,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102381+0000","last_change":"2023-10-09T04:33:31.250325+0000","last_active":"2023-10-09T04:33:43.102381+0000","last_peered":"2023-10-09T04:33:43.102381+0000","last_clean":"2023-10-09T04:33:43.102381+0000","last_became_active":"2023-10-09T04:33:31.248668+0000","last_became_peered":"2023-10-09T04:33:31.248668+0000","last_unstale":"2023-10-09T04:33:43.102381+0000","last_undegraded":"2023-10-09T04:33:43.102381+0000","last_fullsized":"2023-10-09T04:33:43.102381+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:38.097187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":267,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895712+0000","last_change":"2023-10-09T04:24:56.170072+0000","last_active":"2023-10-09T04:33:42.895712+0000","last_peered":"2023-10-09T04:33:42.895712+0000","last_clean":"2023-10-09T04:33:42.895712+0000","last_became_active":"2023-10-09T04:24:56.169728+0000","last_became_peered":"2023-10-09T04:24:56.169728+0000","last_unstale":"2023-10-09T04:33:42.895712+0000","last_undegraded":"2023-10-09T04:33:42.895712+0000","last_fullsized":"2023-10-09T04:33:42.895712+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:22.952602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"80'161","reported_seq":1165,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893017+0000","last_change":"2023-10-09T04:32:27.907423+0000","last_active":"2023-10-09T04:33:42.893017+0000","last_peered":"2023-10-09T04:33:42.893017+0000","last_clean":"2023-10-09T04:33:42.893017+0000","last_became_active":"2023-10-09T04:32:27.907058+0000","last_became_peered":"2023-10-09T04:32:27.907058+0000","last_unstale":"2023-10-09T04:33:42.893017+0000","last_undegraded":"2023-10-09T04:33:42.893017+0000","last_fullsized":"2023-10-09T04:33:42.893017+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:15.469241+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:19:15.469241+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:28.674711+0000","scrub_duration":0.0014461070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":272,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006702+0000","last_change":"2023-10-09T04:33:18.223152+0000","last_active":"2023-10-09T04:33:43.006702+0000","last_peered":"2023-10-09T04:33:43.006702+0000","last_clean":"2023-10-09T04:33:43.006702+0000","last_became_active":"2023-10-09T04:33:18.222815+0000","last_became_peered":"2023-10-09T04:33:18.222815+0000","last_unstale":"2023-10-09T04:33:43.006702+0000","last_undegraded":"2023-10-09T04:33:43.006702+0000","last_fullsized":"2023-10-09T04:33:43.006702+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:09.512377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":247,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895550+0000","last_change":"2023-10-09T04:32:46.912552+0000","last_active":"2023-10-09T04:33:42.895550+0000","last_peered":"2023-10-09T04:33:42.895550+0000","last_clean":"2023-10-09T04:33:42.895550+0000","last_became_active":"2023-10-09T04:32:46.912202+0000","last_became_peered":"2023-10-09T04:32:46.912202+0000","last_unstale":"2023-10-09T04:33:42.895550+0000","last_undegraded":"2023-10-09T04:33:42.895550+0000","last_fullsized":"2023-10-09T04:33:42.895550+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:26.746627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"80'161","reported_seq":833,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101912+0000","last_change":"2023-10-09T04:33:31.249644+0000","last_active":"2023-10-09T04:33:43.101912+0000","last_peered":"2023-10-09T04:33:43.101912+0000","last_clean":"2023-10-09T04:33:43.101912+0000","last_became_active":"2023-10-09T04:33:31.247737+0000","last_became_peered":"2023-10-09T04:33:31.247737+0000","last_unstale":"2023-10-09T04:33:43.101912+0000","last_undegraded":"2023-10-09T04:33:43.101912+0000","last_fullsized":"2023-10-09T04:33:43.101912+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:21.356633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:21.356633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:52.931370+0000","scrub_duration":0.0060821690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891056+0000","last_change":"2023-10-09T04:33:37.901844+0000","last_active":"2023-10-09T04:33:42.891056+0000","last_peered":"2023-10-09T04:33:42.891056+0000","last_clean":"2023-10-09T04:33:42.891056+0000","last_became_active":"2023-10-09T04:33:37.901518+0000","last_became_peered":"2023-10-09T04:33:37.901518+0000","last_unstale":"2023-10-09T04:33:42.891056+0000","last_undegraded":"2023-10-09T04:33:42.891056+0000","last_fullsized":"2023-10-09T04:33:42.891056+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:04.978549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":224,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895969+0000","last_change":"2023-10-09T04:26:21.365541+0000","last_active":"2023-10-09T04:33:42.895969+0000","last_peered":"2023-10-09T04:33:42.895969+0000","last_clean":"2023-10-09T04:33:42.895969+0000","last_became_active":"2023-10-09T04:26:21.364915+0000","last_became_peered":"2023-10-09T04:26:21.364915+0000","last_unstale":"2023-10-09T04:33:42.895969+0000","last_undegraded":"2023-10-09T04:33:42.895969+0000","last_fullsized":"2023-10-09T04:33:42.895969+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.567176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"80'161","reported_seq":537,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101744+0000","last_change":"2023-10-09T04:33:05.892699+0000","last_active":"2023-10-09T04:33:43.101744+0000","last_peered":"2023-10-09T04:33:43.101744+0000","last_clean":"2023-10-09T04:33:43.101744+0000","last_became_active":"2023-10-09T04:33:05.892377+0000","last_became_peered":"2023-10-09T04:33:05.892377+0000","last_unstale":"2023-10-09T04:33:43.101744+0000","last_undegraded":"2023-10-09T04:33:43.101744+0000","last_fullsized":"2023-10-09T04:33:43.101744+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":282,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:10.752296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:10.752296+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:20.040289+0000","scrub_duration":0.012087095000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896214+0000","last_change":"2023-10-09T04:26:21.353292+0000","last_active":"2023-10-09T04:33:42.896214+0000","last_peered":"2023-10-09T04:33:42.896214+0000","last_clean":"2023-10-09T04:33:42.896214+0000","last_became_active":"2023-10-09T04:26:21.352983+0000","last_became_peered":"2023-10-09T04:26:21.352983+0000","last_unstale":"2023-10-09T04:33:42.896214+0000","last_undegraded":"2023-10-09T04:33:42.896214+0000","last_fullsized":"2023-10-09T04:33:42.896214+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:30:35.777129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":257,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896063+0000","last_change":"2023-10-09T04:32:46.913544+0000","last_active":"2023-10-09T04:33:42.896063+0000","last_peered":"2023-10-09T04:33:42.896063+0000","last_clean":"2023-10-09T04:33:42.896063+0000","last_became_active":"2023-10-09T04:32:46.912941+0000","last_became_peered":"2023-10-09T04:32:46.912941+0000","last_unstale":"2023-10-09T04:33:42.896063+0000","last_undegraded":"2023-10-09T04:33:42.896063+0000","last_fullsized":"2023-10-09T04:33:42.896063+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:02.930550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"80'161","reported_seq":790,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102003+0000","last_change":"2023-10-09T04:33:31.250999+0000","last_active":"2023-10-09T04:33:43.102003+0000","last_peered":"2023-10-09T04:33:43.102003+0000","last_clean":"2023-10-09T04:33:43.102003+0000","last_became_active":"2023-10-09T04:33:31.250203+0000","last_became_peered":"2023-10-09T04:33:31.250203+0000","last_unstale":"2023-10-09T04:33:43.102003+0000","last_undegraded":"2023-10-09T04:33:43.102003+0000","last_fullsized":"2023-10-09T04:33:43.102003+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:19:25.277798+0000","last_clean_scrub_stamp":"2023-10-09T04:19:25.277798+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:32:26.305117+0000","scrub_duration":0.014879692999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006724+0000","last_change":"2023-10-09T04:32:24.895007+0000","last_active":"2023-10-09T04:33:43.006724+0000","last_peered":"2023-10-09T04:33:43.006724+0000","last_clean":"2023-10-09T04:33:43.006724+0000","last_became_active":"2023-10-09T04:32:24.894679+0000","last_became_peered":"2023-10-09T04:32:24.894679+0000","last_unstale":"2023-10-09T04:33:43.006724+0000","last_undegraded":"2023-10-09T04:33:43.006724+0000","last_fullsized":"2023-10-09T04:33:43.006724+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:52.639216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":221,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.888279+0000","last_change":"2023-10-09T04:33:11.921041+0000","last_active":"2023-10-09T04:33:42.888279+0000","last_peered":"2023-10-09T04:33:42.888279+0000","last_clean":"2023-10-09T04:33:42.888279+0000","last_became_active":"2023-10-09T04:33:11.920740+0000","last_became_peered":"2023-10-09T04:33:11.920740+0000","last_unstale":"2023-10-09T04:33:42.888279+0000","last_undegraded":"2023-10-09T04:33:42.888279+0000","last_fullsized":"2023-10-09T04:33:42.888279+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:17:57.482646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"80'161","reported_seq":784,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891504+0000","last_change":"2023-10-09T04:33:37.880676+0000","last_active":"2023-10-09T04:33:42.891504+0000","last_peered":"2023-10-09T04:33:42.891504+0000","last_clean":"2023-10-09T04:33:42.891504+0000","last_became_active":"2023-10-09T04:33:37.880072+0000","last_became_peered":"2023-10-09T04:33:37.880072+0000","last_unstale":"2023-10-09T04:33:42.891504+0000","last_undegraded":"2023-10-09T04:33:42.891504+0000","last_fullsized":"2023-10-09T04:33:42.891504+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:33:24.202109+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:32:55.312916+0000","last_clean_scrub_stamp":"2023-10-09T04:33:24.202109+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:13.979897+0000","scrub_duration":0.0015606909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"610'120","reported_seq":645,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101722+0000","last_change":"2023-10-09T04:33:05.892724+0000","last_active":"2023-10-09T04:33:43.101722+0000","last_peered":"2023-10-09T04:33:43.101722+0000","last_clean":"2023-10-09T04:33:43.101722+0000","last_became_active":"2023-10-09T04:33:05.892310+0000","last_became_peered":"2023-10-09T04:33:05.892310+0000","last_unstale":"2023-10-09T04:33:43.101722+0000","last_undegraded":"2023-10-09T04:33:43.101722+0000","last_fullsized":"2023-10-09T04:33:43.101722+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":120,"log_dups_size":0,"ondisk_log_size":120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:48.391521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065711399999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":557,"num_read_kb":2531,"num_write":123,"num_write_kb":1583,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":504,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102532+0000","last_change":"2023-10-09T04:33:31.250362+0000","last_active":"2023-10-09T04:33:43.102532+0000","last_peered":"2023-10-09T04:33:43.102532+0000","last_clean":"2023-10-09T04:33:43.102532+0000","last_became_active":"2023-10-09T04:33:31.249187+0000","last_became_peered":"2023-10-09T04:33:31.249187+0000","last_unstale":"2023-10-09T04:33:43.102532+0000","last_undegraded":"2023-10-09T04:33:43.102532+0000","last_fullsized":"2023-10-09T04:33:43.102532+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:23.409192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":432,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101501+0000","last_change":"2023-10-09T04:33:05.896838+0000","last_active":"2023-10-09T04:33:43.101501+0000","last_peered":"2023-10-09T04:33:43.101501+0000","last_clean":"2023-10-09T04:33:43.101501+0000","last_became_active":"2023-10-09T04:33:05.896519+0000","last_became_peered":"2023-10-09T04:33:05.896519+0000","last_unstale":"2023-10-09T04:33:43.101501+0000","last_undegraded":"2023-10-09T04:33:43.101501+0000","last_fullsized":"2023-10-09T04:33:43.101501+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:09:49.232498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"80'161","reported_seq":680,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102093+0000","last_change":"2023-10-09T04:33:31.246737+0000","last_active":"2023-10-09T04:33:43.102093+0000","last_peered":"2023-10-09T04:33:43.102093+0000","last_clean":"2023-10-09T04:33:43.102093+0000","last_became_active":"2023-10-09T04:33:31.246335+0000","last_became_peered":"2023-10-09T04:33:31.246335+0000","last_unstale":"2023-10-09T04:33:43.102093+0000","last_undegraded":"2023-10-09T04:33:43.102093+0000","last_fullsized":"2023-10-09T04:33:43.102093+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:21:44.346526+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:21:44.346526+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:02.380113+0000","scrub_duration":0.0055215940000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"609'113","reported_seq":682,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102418+0000","last_change":"2023-10-09T04:33:31.250149+0000","last_active":"2023-10-09T04:33:43.102418+0000","last_peered":"2023-10-09T04:33:43.102418+0000","last_clean":"2023-10-09T04:33:43.102418+0000","last_became_active":"2023-10-09T04:33:31.247147+0000","last_became_peered":"2023-10-09T04:33:31.247147+0000","last_unstale":"2023-10-09T04:33:43.102418+0000","last_undegraded":"2023-10-09T04:33:43.102418+0000","last_fullsized":"2023-10-09T04:33:43.102418+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:53.540748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035804899999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":921,"num_read_kb":1093,"num_write":84,"num_write_kb":2221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895671+0000","last_change":"2023-10-09T04:32:46.915115+0000","last_active":"2023-10-09T04:33:42.895671+0000","last_peered":"2023-10-09T04:33:42.895671+0000","last_clean":"2023-10-09T04:33:42.895671+0000","last_became_active":"2023-10-09T04:32:46.914599+0000","last_became_peered":"2023-10-09T04:32:46.914599+0000","last_unstale":"2023-10-09T04:33:42.895671+0000","last_undegraded":"2023-10-09T04:33:42.895671+0000","last_fullsized":"2023-10-09T04:33:42.895671+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:07:34.079842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":114,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895485+0000","last_change":"2023-10-09T04:32:24.897206+0000","last_active":"2023-10-09T04:33:42.895485+0000","last_peered":"2023-10-09T04:33:42.895485+0000","last_clean":"2023-10-09T04:33:42.895485+0000","last_became_active":"2023-10-09T04:32:24.896906+0000","last_became_peered":"2023-10-09T04:32:24.896906+0000","last_unstale":"2023-10-09T04:33:42.895485+0000","last_undegraded":"2023-10-09T04:33:42.895485+0000","last_fullsized":"2023-10-09T04:33:42.895485+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:47.452604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"80'161","reported_seq":637,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.901374+0000","last_change":"2023-10-09T04:33:43.901374+0000","last_active":"2023-10-09T04:33:43.901374+0000","last_peered":"2023-10-09T04:33:43.901374+0000","last_clean":"2023-10-09T04:33:43.901374+0000","last_became_active":"2023-10-09T04:33:43.901068+0000","last_became_peered":"2023-10-09T04:33:43.901068+0000","last_unstale":"2023-10-09T04:33:43.901374+0000","last_undegraded":"2023-10-09T04:33:43.901374+0000","last_fullsized":"2023-10-09T04:33:43.901374+0000","mapping_epoch":641,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":4,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:24.663779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:24.663779+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:08.333067+0000","scrub_duration":0.010964524,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"609'60","reported_seq":689,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.887881+0000","last_change":"2023-10-09T04:33:11.921468+0000","last_active":"2023-10-09T04:33:42.887881+0000","last_peered":"2023-10-09T04:33:42.887881+0000","last_clean":"2023-10-09T04:33:42.887881+0000","last_became_active":"2023-10-09T04:33:11.920922+0000","last_became_peered":"2023-10-09T04:33:11.920922+0000","last_unstale":"2023-10-09T04:33:42.887881+0000","last_undegraded":"2023-10-09T04:33:42.887881+0000","last_fullsized":"2023-10-09T04:33:42.887881+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":60,"log_dups_size":0,"ondisk_log_size":60,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:37:58.105236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052562699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":217,"num_read_kb":1147,"num_write":66,"num_write_kb":2092,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":16,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":541,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101349+0000","last_change":"2023-10-09T04:32:46.897368+0000","last_active":"2023-10-09T04:33:43.101349+0000","last_peered":"2023-10-09T04:33:43.101349+0000","last_clean":"2023-10-09T04:33:43.101349+0000","last_became_active":"2023-10-09T04:32:46.897021+0000","last_became_peered":"2023-10-09T04:32:46.897021+0000","last_unstale":"2023-10-09T04:33:43.101349+0000","last_undegraded":"2023-10-09T04:33:43.101349+0000","last_fullsized":"2023-10-09T04:33:43.101349+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:10.435389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":211,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101857+0000","last_change":"2023-10-09T04:33:05.892505+0000","last_active":"2023-10-09T04:33:43.101857+0000","last_peered":"2023-10-09T04:33:43.101857+0000","last_clean":"2023-10-09T04:33:43.101857+0000","last_became_active":"2023-10-09T04:33:05.892229+0000","last_became_peered":"2023-10-09T04:33:05.892229+0000","last_unstale":"2023-10-09T04:33:43.101857+0000","last_undegraded":"2023-10-09T04:33:43.101857+0000","last_fullsized":"2023-10-09T04:33:43.101857+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:54.364822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"610'264","reported_seq":953,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102330+0000","last_change":"2023-10-09T04:33:31.250569+0000","last_active":"2023-10-09T04:33:43.102330+0000","last_peered":"2023-10-09T04:33:43.102330+0000","last_clean":"2023-10-09T04:33:43.102330+0000","last_became_active":"2023-10-09T04:33:31.248970+0000","last_became_peered":"2023-10-09T04:33:31.248970+0000","last_unstale":"2023-10-09T04:33:43.102330+0000","last_undegraded":"2023-10-09T04:33:43.102330+0000","last_fullsized":"2023-10-09T04:33:43.102330+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:46.730123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001223699,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1597,"num_read_kb":1410,"num_write":240,"num_write_kb":158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"80'161","reported_seq":1136,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.892988+0000","last_change":"2023-10-09T04:32:49.912670+0000","last_active":"2023-10-09T04:33:42.892988+0000","last_peered":"2023-10-09T04:33:42.892988+0000","last_clean":"2023-10-09T04:33:42.892988+0000","last_became_active":"2023-10-09T04:32:49.912299+0000","last_became_peered":"2023-10-09T04:32:49.912299+0000","last_unstale":"2023-10-09T04:33:42.892988+0000","last_undegraded":"2023-10-09T04:33:42.892988+0000","last_fullsized":"2023-10-09T04:33:42.892988+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":0,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:17:11.306129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:17:11.306129+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:50.176300+0000","scrub_duration":0.0098035020000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":123,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006437+0000","last_change":"2023-10-09T04:32:46.901306+0000","last_active":"2023-10-09T04:33:43.006437+0000","last_peered":"2023-10-09T04:33:43.006437+0000","last_clean":"2023-10-09T04:33:43.006437+0000","last_became_active":"2023-10-09T04:32:46.901052+0000","last_became_peered":"2023-10-09T04:32:46.901052+0000","last_unstale":"2023-10-09T04:33:43.006437+0000","last_undegraded":"2023-10-09T04:33:43.006437+0000","last_fullsized":"2023-10-09T04:33:43.006437+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:41.288918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":204,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891298+0000","last_change":"2023-10-09T04:33:37.879982+0000","last_active":"2023-10-09T04:33:42.891298+0000","last_peered":"2023-10-09T04:33:42.891298+0000","last_clean":"2023-10-09T04:33:42.891298+0000","last_became_active":"2023-10-09T04:33:37.879660+0000","last_became_peered":"2023-10-09T04:33:37.879660+0000","last_unstale":"2023-10-09T04:33:42.891298+0000","last_undegraded":"2023-10-09T04:33:42.891298+0000","last_fullsized":"2023-10-09T04:33:42.891298+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:33.416521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"80'161","reported_seq":564,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101724+0000","last_change":"2023-10-09T04:33:05.898564+0000","last_active":"2023-10-09T04:33:43.101724+0000","last_peered":"2023-10-09T04:33:43.101724+0000","last_clean":"2023-10-09T04:33:43.101724+0000","last_became_active":"2023-10-09T04:33:05.898217+0000","last_became_peered":"2023-10-09T04:33:05.898217+0000","last_unstale":"2023-10-09T04:33:43.101724+0000","last_undegraded":"2023-10-09T04:33:43.101724+0000","last_fullsized":"2023-10-09T04:33:43.101724+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.969363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.969363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:35.314212+0000","scrub_duration":0.003156795,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"610'198","reported_seq":1550,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893012+0000","last_change":"2023-10-09T04:32:32.836552+0000","last_active":"2023-10-09T04:33:42.893012+0000","last_peered":"2023-10-09T04:33:42.893012+0000","last_clean":"2023-10-09T04:33:42.893012+0000","last_became_active":"2023-10-09T04:23:06.347801+0000","last_became_peered":"2023-10-09T04:23:06.347801+0000","last_unstale":"2023-10-09T04:33:42.893012+0000","last_undegraded":"2023-10-09T04:33:42.893012+0000","last_fullsized":"2023-10-09T04:33:42.893012+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":198,"log_dups_size":0,"ondisk_log_size":198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:07:17.720354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051653199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1649,"num_read_kb":1774,"num_write":137,"num_write_kb":233,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":313,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895813+0000","last_change":"2023-10-09T04:32:24.899713+0000","last_active":"2023-10-09T04:33:42.895813+0000","last_peered":"2023-10-09T04:33:42.895813+0000","last_clean":"2023-10-09T04:33:42.895813+0000","last_became_active":"2023-10-09T04:32:24.899312+0000","last_became_peered":"2023-10-09T04:32:24.899312+0000","last_unstale":"2023-10-09T04:33:42.895813+0000","last_undegraded":"2023-10-09T04:33:42.895813+0000","last_fullsized":"2023-10-09T04:33:42.895813+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:33:49.724726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":268,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.899401+0000","last_change":"2023-10-09T04:26:21.349028+0000","last_active":"2023-10-09T04:33:43.899401+0000","last_peered":"2023-10-09T04:33:43.899401+0000","last_clean":"2023-10-09T04:33:43.899401+0000","last_became_active":"2023-10-09T04:26:21.348635+0000","last_became_peered":"2023-10-09T04:26:21.348635+0000","last_unstale":"2023-10-09T04:33:43.899401+0000","last_undegraded":"2023-10-09T04:33:43.899401+0000","last_fullsized":"2023-10-09T04:33:43.899401+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:43.276520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"80'161","reported_seq":555,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891505+0000","last_change":"2023-10-09T04:33:37.876943+0000","last_active":"2023-10-09T04:33:42.891505+0000","last_peered":"2023-10-09T04:33:42.891505+0000","last_clean":"2023-10-09T04:33:42.891505+0000","last_became_active":"2023-10-09T04:33:37.876607+0000","last_became_peered":"2023-10-09T04:33:37.876607+0000","last_unstale":"2023-10-09T04:33:42.891505+0000","last_undegraded":"2023-10-09T04:33:42.891505+0000","last_fullsized":"2023-10-09T04:33:42.891505+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:18:27.648935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:18:27.648935+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:24:46.067430+0000","scrub_duration":0.0023188589999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"609'65","reported_seq":869,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893714+0000","last_change":"2023-10-09T04:32:32.837194+0000","last_active":"2023-10-09T04:33:42.893714+0000","last_peered":"2023-10-09T04:33:42.893714+0000","last_clean":"2023-10-09T04:33:42.893714+0000","last_became_active":"2023-10-09T04:24:56.184041+0000","last_became_peered":"2023-10-09T04:24:56.184041+0000","last_unstale":"2023-10-09T04:33:42.893714+0000","last_undegraded":"2023-10-09T04:33:42.893714+0000","last_fullsized":"2023-10-09T04:33:42.893714+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:54.992517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042491300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":301,"num_read_kb":234,"num_write":65,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":394,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101867+0000","last_change":"2023-10-09T04:32:35.957013+0000","last_active":"2023-10-09T04:33:43.101867+0000","last_peered":"2023-10-09T04:33:43.101867+0000","last_clean":"2023-10-09T04:33:43.101867+0000","last_became_active":"2023-10-09T04:32:35.956726+0000","last_became_peered":"2023-10-09T04:32:35.956726+0000","last_unstale":"2023-10-09T04:33:43.101867+0000","last_undegraded":"2023-10-09T04:33:43.101867+0000","last_fullsized":"2023-10-09T04:33:43.101867+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:35.311809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":226,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.899378+0000","last_change":"2023-10-09T04:32:35.960473+0000","last_active":"2023-10-09T04:33:43.899378+0000","last_peered":"2023-10-09T04:33:43.899378+0000","last_clean":"2023-10-09T04:33:43.899378+0000","last_became_active":"2023-10-09T04:32:35.960255+0000","last_became_peered":"2023-10-09T04:32:35.960255+0000","last_unstale":"2023-10-09T04:33:43.899378+0000","last_undegraded":"2023-10-09T04:33:43.899378+0000","last_fullsized":"2023-10-09T04:33:43.899378+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:27.363502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"80'161","reported_seq":759,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895971+0000","last_change":"2023-10-09T04:32:49.911515+0000","last_active":"2023-10-09T04:33:42.895971+0000","last_peered":"2023-10-09T04:33:42.895971+0000","last_clean":"2023-10-09T04:33:42.895971+0000","last_became_active":"2023-10-09T04:32:49.911135+0000","last_became_peered":"2023-10-09T04:32:49.911135+0000","last_unstale":"2023-10-09T04:33:42.895971+0000","last_undegraded":"2023-10-09T04:33:42.895971+0000","last_fullsized":"2023-10-09T04:33:42.895971+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:10.344462+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:10.344462+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:48.520940+0000","scrub_duration":0.001275394,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":611,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007152+0000","last_change":"2023-10-09T04:33:18.223300+0000","last_active":"2023-10-09T04:33:43.007152+0000","last_peered":"2023-10-09T04:33:43.007152+0000","last_clean":"2023-10-09T04:33:43.007152+0000","last_became_active":"2023-10-09T04:33:18.223023+0000","last_became_peered":"2023-10-09T04:33:18.223023+0000","last_unstale":"2023-10-09T04:33:43.007152+0000","last_undegraded":"2023-10-09T04:33:43.007152+0000","last_fullsized":"2023-10-09T04:33:43.007152+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:44.099916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"610'100","reported_seq":530,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101882+0000","last_change":"2023-10-09T04:33:05.897728+0000","last_active":"2023-10-09T04:33:43.101882+0000","last_peered":"2023-10-09T04:33:43.101882+0000","last_clean":"2023-10-09T04:33:43.101882+0000","last_became_active":"2023-10-09T04:33:05.897410+0000","last_became_peered":"2023-10-09T04:33:05.897410+0000","last_unstale":"2023-10-09T04:33:43.101882+0000","last_undegraded":"2023-10-09T04:33:43.101882+0000","last_fullsized":"2023-10-09T04:33:43.101882+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:15:05.093600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049773700000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":433,"num_read_kb":514,"num_write":78,"num_write_kb":4235,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":10240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":541,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006536+0000","last_change":"2023-10-09T04:33:18.224403+0000","last_active":"2023-10-09T04:33:43.006536+0000","last_peered":"2023-10-09T04:33:43.006536+0000","last_clean":"2023-10-09T04:33:43.006536+0000","last_became_active":"2023-10-09T04:33:18.224079+0000","last_became_peered":"2023-10-09T04:33:18.224079+0000","last_unstale":"2023-10-09T04:33:43.006536+0000","last_undegraded":"2023-10-09T04:33:43.006536+0000","last_fullsized":"2023-10-09T04:33:43.006536+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:02:21.446044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"80'161","reported_seq":817,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006978+0000","last_change":"2023-10-09T04:32:27.906600+0000","last_active":"2023-10-09T04:33:43.006978+0000","last_peered":"2023-10-09T04:33:43.006978+0000","last_clean":"2023-10-09T04:33:43.006978+0000","last_became_active":"2023-10-09T04:32:27.906224+0000","last_became_peered":"2023-10-09T04:32:27.906224+0000","last_unstale":"2023-10-09T04:33:43.006978+0000","last_undegraded":"2023-10-09T04:33:43.006978+0000","last_fullsized":"2023-10-09T04:33:43.006978+0000","mapping_epoch":601,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":3,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:31:59.104786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:31:59.104786+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:38.393804+0000","scrub_duration":0.003247907,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"609'56","reported_seq":50,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.891653+0000","last_change":"2023-10-09T04:33:37.880966+0000","last_active":"2023-10-09T04:33:42.891653+0000","last_peered":"2023-10-09T04:33:42.891653+0000","last_clean":"2023-10-09T04:33:42.891653+0000","last_became_active":"2023-10-09T04:33:37.880325+0000","last_became_peered":"2023-10-09T04:33:37.880325+0000","last_unstale":"2023-10-09T04:33:42.891653+0000","last_undegraded":"2023-10-09T04:33:42.891653+0000","last_fullsized":"2023-10-09T04:33:42.891653+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:53:19.096249+0000","last_clean_scrub_stamp":"2023-10-09T03:53:19.096249+0000","objects_scrubbed":0,"log_size":56,"log_dups_size":0,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:09.835733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0015777339999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":248,"num_read_kb":1374,"num_write":53,"num_write_kb":2138,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":280,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.899490+0000","last_change":"2023-10-09T04:24:56.204209+0000","last_active":"2023-10-09T04:33:43.899490+0000","last_peered":"2023-10-09T04:33:43.899490+0000","last_clean":"2023-10-09T04:33:43.899490+0000","last_became_active":"2023-10-09T04:24:56.168981+0000","last_became_peered":"2023-10-09T04:24:56.168981+0000","last_unstale":"2023-10-09T04:33:43.899490+0000","last_undegraded":"2023-10-09T04:33:43.899490+0000","last_fullsized":"2023-10-09T04:33:43.899490+0000","mapping_epoch":444,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:31.369948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.896123+0000","last_change":"2023-10-09T04:32:46.912541+0000","last_active":"2023-10-09T04:33:42.896123+0000","last_peered":"2023-10-09T04:33:42.896123+0000","last_clean":"2023-10-09T04:33:42.896123+0000","last_became_active":"2023-10-09T04:32:46.912111+0000","last_became_peered":"2023-10-09T04:32:46.912111+0000","last_unstale":"2023-10-09T04:33:42.896123+0000","last_undegraded":"2023-10-09T04:33:42.896123+0000","last_fullsized":"2023-10-09T04:33:42.896123+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:14.456278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"80'161","reported_seq":702,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006574+0000","last_change":"2023-10-09T04:33:18.224514+0000","last_active":"2023-10-09T04:33:43.006574+0000","last_peered":"2023-10-09T04:33:43.006574+0000","last_clean":"2023-10-09T04:33:43.006574+0000","last_became_active":"2023-10-09T04:33:18.224032+0000","last_became_peered":"2023-10-09T04:33:18.224032+0000","last_unstale":"2023-10-09T04:33:43.006574+0000","last_undegraded":"2023-10-09T04:33:43.006574+0000","last_fullsized":"2023-10-09T04:33:43.006574+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:03.845515+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:27:03.845515+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:10.182910+0000","scrub_duration":0.001436709,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":276,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895952+0000","last_change":"2023-10-09T04:30:51.979500+0000","last_active":"2023-10-09T04:33:42.895952+0000","last_peered":"2023-10-09T04:33:42.895952+0000","last_clean":"2023-10-09T04:33:42.895952+0000","last_became_active":"2023-10-09T04:30:51.979149+0000","last_became_peered":"2023-10-09T04:30:51.979149+0000","last_unstale":"2023-10-09T04:33:42.895952+0000","last_undegraded":"2023-10-09T04:33:42.895952+0000","last_fullsized":"2023-10-09T04:33:42.895952+0000","mapping_epoch":557,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:52.702260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":196,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-10-09T04:33:44.018840+0000","last_change":"2023-10-09T04:33:12.381121+0000","last_active":"2023-10-09T04:33:44.018840+0000","last_peered":"2023-10-09T04:33:44.018840+0000","last_clean":"2023-10-09T04:33:44.018840+0000","last_became_active":"2023-10-09T04:33:12.380752+0000","last_became_peered":"2023-10-09T04:33:12.380752+0000","last_unstale":"2023-10-09T04:33:44.018840+0000","last_undegraded":"2023-10-09T04:33:44.018840+0000","last_fullsized":"2023-10-09T04:33:44.018840+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:03.881486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"80'161","reported_seq":881,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007064+0000","last_change":"2023-10-09T04:33:18.221081+0000","last_active":"2023-10-09T04:33:43.007064+0000","last_peered":"2023-10-09T04:33:43.007064+0000","last_clean":"2023-10-09T04:33:43.007064+0000","last_became_active":"2023-10-09T04:33:18.220783+0000","last_became_peered":"2023-10-09T04:33:18.220783+0000","last_unstale":"2023-10-09T04:33:43.007064+0000","last_undegraded":"2023-10-09T04:33:43.007064+0000","last_fullsized":"2023-10-09T04:33:43.007064+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:22:18.236436+0000","last_clean_scrub_stamp":"2023-10-09T04:22:18.236436+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:24.537848+0000","scrub_duration":0.003113462,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":51,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102219+0000","last_change":"2023-10-09T04:33:31.243274+0000","last_active":"2023-10-09T04:33:43.102219+0000","last_peered":"2023-10-09T04:33:43.102219+0000","last_clean":"2023-10-09T04:33:43.102219+0000","last_became_active":"2023-10-09T04:33:31.242888+0000","last_became_peered":"2023-10-09T04:33:31.242888+0000","last_unstale":"2023-10-09T04:33:43.102219+0000","last_undegraded":"2023-10-09T04:33:43.102219+0000","last_fullsized":"2023-10-09T04:33:43.102219+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:40.681162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895547+0000","last_change":"2023-10-09T04:32:24.904229+0000","last_active":"2023-10-09T04:33:42.895547+0000","last_peered":"2023-10-09T04:33:42.895547+0000","last_clean":"2023-10-09T04:33:42.895547+0000","last_became_active":"2023-10-09T04:32:24.903889+0000","last_became_peered":"2023-10-09T04:32:24.903889+0000","last_unstale":"2023-10-09T04:33:42.895547+0000","last_undegraded":"2023-10-09T04:33:42.895547+0000","last_fullsized":"2023-10-09T04:33:42.895547+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:58.275008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":51,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.102362+0000","last_change":"2023-10-09T04:33:31.251324+0000","last_active":"2023-10-09T04:33:43.102362+0000","last_peered":"2023-10-09T04:33:43.102362+0000","last_clean":"2023-10-09T04:33:43.102362+0000","last_became_active":"2023-10-09T04:33:31.251024+0000","last_became_peered":"2023-10-09T04:33:31.251024+0000","last_unstale":"2023-10-09T04:33:43.102362+0000","last_undegraded":"2023-10-09T04:33:43.102362+0000","last_fullsized":"2023-10-09T04:33:43.102362+0000","mapping_epoch":635,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":636,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:04.659803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":164,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895349+0000","last_change":"2023-10-09T04:32:24.905833+0000","last_active":"2023-10-09T04:33:42.895349+0000","last_peered":"2023-10-09T04:33:42.895349+0000","last_clean":"2023-10-09T04:33:42.895349+0000","last_became_active":"2023-10-09T04:32:24.905447+0000","last_became_peered":"2023-10-09T04:32:24.905447+0000","last_unstale":"2023-10-09T04:33:42.895349+0000","last_undegraded":"2023-10-09T04:33:42.895349+0000","last_fullsized":"2023-10-09T04:33:42.895349+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:37:28.627681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"80'161","reported_seq":975,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.893433+0000","last_change":"2023-10-09T04:32:49.908665+0000","last_active":"2023-10-09T04:33:42.893433+0000","last_peered":"2023-10-09T04:33:42.893433+0000","last_clean":"2023-10-09T04:33:42.893433+0000","last_became_active":"2023-10-09T04:32:49.908329+0000","last_became_peered":"2023-10-09T04:32:49.908329+0000","last_unstale":"2023-10-09T04:33:42.893433+0000","last_undegraded":"2023-10-09T04:33:42.893433+0000","last_fullsized":"2023-10-09T04:33:42.893433+0000","mapping_epoch":617,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:27:00.788186+0000","last_deep_scrub":"80'161","last_deep_scrub_stamp":"2023-10-09T04:21:55.959478+0000","last_clean_scrub_stamp":"2023-10-09T04:27:00.788186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:34.358388+0000","scrub_duration":0.0048035049999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":68,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.007134+0000","last_change":"2023-10-09T04:33:18.220900+0000","last_active":"2023-10-09T04:33:43.007134+0000","last_peered":"2023-10-09T04:33:43.007134+0000","last_clean":"2023-10-09T04:33:43.007134+0000","last_became_active":"2023-10-09T04:33:18.220583+0000","last_became_peered":"2023-10-09T04:33:18.220583+0000","last_unstale":"2023-10-09T04:33:43.007134+0000","last_undegraded":"2023-10-09T04:33:43.007134+0000","last_fullsized":"2023-10-09T04:33:43.007134+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:15:26.503933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":494,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.006804+0000","last_change":"2023-10-09T04:33:18.224318+0000","last_active":"2023-10-09T04:33:43.006804+0000","last_peered":"2023-10-09T04:33:43.006804+0000","last_clean":"2023-10-09T04:33:43.006804+0000","last_became_active":"2023-10-09T04:33:18.223688+0000","last_became_peered":"2023-10-09T04:33:18.223688+0000","last_unstale":"2023-10-09T04:33:43.006804+0000","last_undegraded":"2023-10-09T04:33:43.006804+0000","last_fullsized":"2023-10-09T04:33:43.006804+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:28.092920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"80'161","reported_seq":585,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:43.101810+0000","last_change":"2023-10-09T04:33:05.897583+0000","last_active":"2023-10-09T04:33:43.101810+0000","last_peered":"2023-10-09T04:33:43.101810+0000","last_clean":"2023-10-09T04:33:43.101810+0000","last_became_active":"2023-10-09T04:33:05.897096+0000","last_became_peered":"2023-10-09T04:33:05.897096+0000","last_unstale":"2023-10-09T04:33:43.101810+0000","last_undegraded":"2023-10-09T04:33:43.101810+0000","last_fullsized":"2023-10-09T04:33:43.101810+0000","mapping_epoch":624,"log_start":"0'0","ondisk_log_start":"0'0","created":292,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":5,"last_scrub":"80'161","last_scrub_stamp":"2023-10-09T04:22:16.013472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:50:59.589675+0000","last_clean_scrub_stamp":"2023-10-09T04:22:16.013472+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:36.724610+0000","scrub_duration":0.001705481,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":317,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895982+0000","last_change":"2023-10-09T04:32:07.865684+0000","last_active":"2023-10-09T04:33:42.895982+0000","last_peered":"2023-10-09T04:33:42.895982+0000","last_clean":"2023-10-09T04:33:42.895982+0000","last_became_active":"2023-10-09T04:32:07.849033+0000","last_became_peered":"2023-10-09T04:32:07.849033+0000","last_unstale":"2023-10-09T04:33:42.895982+0000","last_undegraded":"2023-10-09T04:33:42.895982+0000","last_fullsized":"2023-10-09T04:33:42.895982+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:12.181737+0000","last_clean_scrub_stamp":"2023-10-09T03:54:12.181737+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:08.096318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":257,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-10-09T04:33:42.895277+0000","last_change":"2023-10-09T04:32:46.913746+0000","last_active":"2023-10-09T04:33:42.895277+0000","last_peered":"2023-10-09T04:33:42.895277+0000","last_clean":"2023-10-09T04:33:42.895277+0000","last_became_active":"2023-10-09T04:32:46.912066+0000","last_became_peered":"2023-10-09T04:32:46.912066+0000","last_unstale":"2023-10-09T04:33:42.895277+0000","last_undegraded":"2023-10-09T04:33:42.895277+0000","last_fullsized":"2023-10-09T04:33:42.895277+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T03:54:13.193812+0000","last_clean_scrub_stamp":"2023-10-09T03:54:13.193812+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:02.126877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":23344,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2293760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5923,"num_read_kb":10077,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10294,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201782,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910816,"kb_used_data":6808,"kb_used_omap":0,"kb_used_meta":904000,"kb_avail":92834336,"statfs":{"total":95995035648,"available":95062360064,"internally_reserved":0,"allocated":6971392,"data_stored":5291036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925696000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":7.7160000000000002,"15min":4.6150000000000002},"min":{"1min":0.36299999999999999,"5min":0.23899999999999999,"15min":0.21199999999999999},"max":{"1min":0.65600000000000003,"5min":218.697,"15min":218.697},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":7.6630000000000003,"15min":4.718},"min":{"1min":0.40300000000000002,"5min":0.255,"15min":0.221},"max":{"1min":0.85599999999999998,"5min":218.83099999999999,"15min":218.83099999999999},"last":0.44600000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":4.8609999999999998,"15min":5.7320000000000002},"min":{"1min":0.38300000000000001,"5min":0.26900000000000002,"15min":0.22600000000000001},"max":{"1min":0.99399999999999999,"5min":213.41800000000001,"15min":261.935},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":4.3010000000000002,"15min":5.4100000000000001},"min":{"1min":0.33400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.73399999999999999,"5min":213.352,"15min":262.07600000000002},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.589,"15min":2.4390000000000001},"min":{"1min":0.36399999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.70399999999999996,"5min":37.131999999999998,"15min":253.41200000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.6040000000000001,"15min":2.5840000000000001},"min":{"1min":0.443,"5min":0.33000000000000002,"15min":0.26800000000000002},"max":{"1min":0.68700000000000006,"5min":37.145000000000003,"15min":253.39400000000001},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 04:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":5.1790000000000003,"15min":3.702},"min":{"1min":0.314,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":1.6479999999999999,"5min":206.42500000000001,"15min":255.97300000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":5.0209999999999999,"15min":3.8079999999999998},"min":{"1min":0.49399999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":1.6619999999999999,"5min":206.44300000000001,"15min":256.06099999999998},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.73899999999999999,"15min":0.84799999999999998},"min":{"1min":0.40300000000000002,"5min":0.26800000000000002,"15min":0.24099999999999999},"max":{"1min":0.70999999999999996,"5min":9.7490000000000006,"15min":9.7490000000000006},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84499999999999997,"15min":1.04},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.251},"max":{"1min":0.77100000000000002,"5min":4.4160000000000004,"15min":12.566000000000001},"last":0.67600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.70499999999999996,"15min":1.2350000000000001},"min":{"1min":0.38,"5min":0.28899999999999998,"15min":0.254},"max":{"1min":0.86799999999999999,"5min":5.0620000000000003,"15min":17.882999999999999},"last":0.501},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.85599999999999998,"15min":1.111},"min":{"1min":0.46100000000000002,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":0.86299999999999999,"5min":7.1230000000000002,"15min":17.568999999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.78100000000000003,"15min":1.1799999999999999},"min":{"1min":0.48199999999999998,"5min":0.318,"15min":0.31},"max":{"1min":0.83199999999999996,"5min":6.5380000000000003,"15min":9.5069999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.69599999999999995,"5min":1,"15min":1.044},"min":{"1min":0.56699999999999995,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84399999999999997,"5min":9.8089999999999993,"15min":9.8089999999999993},"last":0.51900000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463418,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914576,"kb_used_data":10952,"kb_used_omap":0,"kb_used_meta":903616,"kb_avail":92830576,"statfs":{"total":95995035648,"available":95058509824,"internally_reserved":0,"allocated":11214848,"data_stored":9549882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925302784},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":3.0609999999999999,"15min":4.3129999999999997},"min":{"1min":0.33100000000000002,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":0.71799999999999997,"5min":134.34399999999999,"15min":162.524},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.497,"5min":3.0859999999999999,"15min":4.2880000000000003},"min":{"1min":0.36399999999999999,"5min":0.245,"15min":0.218},"max":{"1min":0.61799999999999999,"5min":130.87100000000001,"15min":161.73599999999999},"last":0.60699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.548,"15min":4.1020000000000003},"min":{"1min":0.377,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":0.90800000000000003,"5min":10.055,"15min":315.73599999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.6020000000000001,"15min":4.2329999999999997},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":0.75800000000000001,"5min":13.603999999999999,"15min":316.91500000000002},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.5860000000000001,"15min":2.5},"min":{"1min":0.39400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.78400000000000003,"5min":11.74,"15min":116.084},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.6279999999999999,"15min":2.5449999999999999},"min":{"1min":0.41699999999999998,"5min":0.32800000000000001,"15min":0.27100000000000002},"max":{"1min":0.82099999999999995,"5min":11.772,"15min":116.11199999999999},"last":0.71199999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":2.657,"15min":2.0529999999999999},"min":{"1min":0.34200000000000003,"5min":0.28799999999999998,"15min":0.25600000000000001},"max":{"1min":0.65600000000000003,"5min":127.44199999999999,"15min":127.44199999999999},"last":0.625},{"interface":"front","average":{"1min":0.61799999999999999,"5min":2.9929999999999999,"15min":2.1190000000000002},"min":{"1min":0.45300000000000001,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":1.052,"5min":127.51900000000001,"15min":127.51900000000001},"last":0.63400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.6339999999999999,"15min":1.6839999999999999},"min":{"1min":0.35899999999999999,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":0.752,"5min":90.221999999999994,"15min":161.27500000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.505,"5min":1.7869999999999999,"15min":1.794},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.248},"max":{"1min":0.80400000000000005,"5min":90.116,"15min":160.32599999999999},"last":0.44700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.89900000000000002,"15min":1.1399999999999999},"min":{"1min":0.33800000000000002,"5min":0.28999999999999998,"15min":0.26100000000000001},"max":{"1min":0.871,"5min":6.6340000000000003,"15min":74.043000000000006},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.042,"15min":1.1930000000000001},"min":{"1min":0.39000000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":0.92000000000000004,"5min":10.021000000000001,"15min":74.073999999999998},"last":0.55200000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:33:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.625,"15min":1.72},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.92900000000000005,"5min":93.212000000000003,"15min":161.39400000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.9299999999999999,"15min":1.802},"min":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.32900000000000001},"max":{"1min":0.94199999999999995,"5min":93.257000000000005,"15min":161.41200000000001},"last":0.65500000000000003}]}]},{"osd":1,"up_from":89,"seq":382252089728,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920752,"kb_used_data":11752,"kb_used_omap":9,"kb_used_meta":908982,"kb_avail":92824400,"statfs":{"total":95995035648,"available":95052185600,"internally_reserved":0,"allocated":12034048,"data_stored":9370420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9239,"internal_metadata":930798569},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":2.0169999999999999,"15min":3.4390000000000001},"min":{"1min":0.218,"5min":0.185,"15min":0.17699999999999999},"max":{"1min":0.53000000000000003,"5min":33.351999999999997,"15min":259.26799999999997},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.38600000000000001,"5min":2.3450000000000002,"15min":3.6459999999999999},"min":{"1min":0.28699999999999998,"5min":0.187,"15min":0.187},"max":{"1min":0.53400000000000003,"5min":35.271000000000001,"15min":258.46899999999999},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:34:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":1.651,"15min":2.9039999999999999},"min":{"1min":0.248,"5min":0.20899999999999999,"15min":0.19800000000000001},"max":{"1min":0.48699999999999999,"5min":12.959,"15min":259.315},"last":0.249},{"interface":"front","average":{"1min":0.36099999999999999,"5min":1.446,"15min":3.0630000000000002},"min":{"1min":0.25700000000000001,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":0.53700000000000003,"5min":17.718,"15min":259.733},"last":0.26200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":4.0940000000000003,"15min":2.7000000000000002},"min":{"1min":0.28999999999999998,"5min":0.23300000000000001,"15min":0.22600000000000001},"max":{"1min":0.55500000000000005,"5min":156.261,"15min":156.261},"last":0.313},{"interface":"front","average":{"1min":0.42099999999999999,"5min":4.1399999999999997,"15min":2.8069999999999999},"min":{"1min":0.32700000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55400000000000005,"5min":155.684,"15min":155.684},"last":0.29899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":1.361,"15min":2.6459999999999999},"min":{"1min":0.41699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.82399999999999995,"5min":17.780000000000001,"15min":257.83300000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.716,"15min":2.5409999999999999},"min":{"1min":0.40400000000000003,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":0.753,"5min":14.32,"15min":257.863},"last":0.41499999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.411,"15min":1.5820000000000001},"min":{"1min":0.432,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":0.85499999999999998,"5min":17.751999999999999,"15min":25.696999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.7749999999999999,"15min":1.7470000000000001},"min":{"1min":0.41899999999999998,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":0.77700000000000002,"5min":14.356999999999999,"15min":21.893999999999998},"last":0.47399999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":2.2589999999999999,"15min":1.8799999999999999},"min":{"1min":0.46100000000000002,"5min":0.39100000000000001,"15min":0.31900000000000001},"max":{"1min":0.77200000000000002,"5min":50.814999999999998,"15min":50.814999999999998},"last":0.502},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.8220000000000001,"15min":1.625},"min":{"1min":0.44500000000000001,"5min":0.34300000000000003,"15min":0.29999999999999999},"max":{"1min":0.80700000000000005,"5min":50.749000000000002,"15min":50.749000000000002},"last":0.40899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:33:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":2.7730000000000001,"15min":3.8140000000000001},"min":{"1min":0.48499999999999999,"5min":0.36499999999999999,"15min":0.34799999999999998},"max":{"1min":0.81000000000000005,"5min":66.388000000000005,"15min":259.08499999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":2.4660000000000002,"15min":3.5609999999999999},"min":{"1min":0.44600000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84499999999999997,"5min":66.221999999999994,"15min":259.13299999999998},"last":0.46300000000000002}]}]},{"osd":0,"up_from":84,"seq":360777253253,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889636,"kb_used_data":8156,"kb_used_omap":28,"kb_used_meta":881443,"kb_avail":92855516,"statfs":{"total":95995035648,"available":95084048384,"internally_reserved":0,"allocated":8351744,"data_stored":5522239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28848,"internal_metadata":902598480},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:33:32 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":2.149,"15min":2.9380000000000002},"min":{"1min":0.21299999999999999,"5min":0.189,"15min":0.16800000000000001},"max":{"1min":0.72899999999999998,"5min":36.654000000000003,"15min":209.215},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":2.0619999999999998,"15min":2.754},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.78300000000000003,"5min":36.832999999999998,"15min":95.861000000000004},"last":0.24199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":1.4299999999999999,"15min":2.2480000000000002},"min":{"1min":0.248,"5min":0.217,"15min":0.19600000000000001},"max":{"1min":0.504,"5min":12.036,"15min":88.088999999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.36199999999999999,"5min":1.2330000000000001,"15min":2.238},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.20599999999999999},"max":{"1min":0.51800000000000002,"5min":9.9619999999999997,"15min":88.004999999999995},"last":0.312}]},{"osd":3,"last update":"Mon Oct 9 04:33:32 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":2.2629999999999999,"15min":2.165},"min":{"1min":0.27100000000000002,"5min":0.20499999999999999,"15min":0.192},"max":{"1min":0.83899999999999997,"5min":19.507999999999999,"15min":87.891999999999996},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":2.4079999999999999,"15min":2.3130000000000002},"min":{"1min":0.29599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.92800000000000005,"5min":19.741,"15min":88.147999999999996},"last":0.33000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:34:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.016,"15min":2.0590000000000002},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.94299999999999995,"5min":7.9770000000000003,"15min":88.058000000000007},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0760000000000001,"15min":1.9450000000000001},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.24199999999999999},"max":{"1min":0.871,"5min":9.8439999999999994,"15min":86.891000000000005},"last":0.52700000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:34:15 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.236,"15min":2.4209999999999998},"min":{"1min":0.39800000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":0.88300000000000001,"5min":13.198,"15min":134.93000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.304,"15min":2.2509999999999999},"min":{"1min":0.48799999999999999,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":0.99399999999999999,"5min":8.5399999999999991,"15min":135.04599999999999},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:34:15 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.27,"15min":1.4079999999999999},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.34699999999999998},"max":{"1min":0.876,"5min":8.4920000000000009,"15min":8.4920000000000009},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.1559999999999999,"15min":1.7190000000000001},"min":{"1min":0.42199999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79600000000000004,"5min":11.052,"15min":88.180000000000007},"last":0.54300000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:34:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.857,"15min":2.0139999999999998},"min":{"1min":0.44,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":0.96099999999999997,"5min":64.295000000000002,"15min":78.307000000000002},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.786,"15min":2.371},"min":{"1min":0.39700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.84899999999999998,"5min":64.319000000000003,"15min":88.120000000000005},"last":0.505}]}]},{"osd":2,"up_from":94,"seq":403726926201,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842684,"kb_used_data":7924,"kb_used_omap":81,"kb_used_meta":834670,"kb_avail":92902468,"statfs":{"total":95995035648,"available":95132127232,"internally_reserved":0,"allocated":8114176,"data_stored":5436959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":83750,"internal_metadata":854702298},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":3.3159999999999998,"15min":2.8460000000000001},"min":{"1min":0.219,"5min":0.18099999999999999,"15min":0.17699999999999999},"max":{"1min":0.625,"5min":163.09800000000001,"15min":163.09800000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":3.4969999999999999,"15min":3.129},"min":{"1min":0.23999999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.55400000000000005,"5min":163.11699999999999,"15min":163.11699999999999},"last":0.36799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":1.9950000000000001,"15min":3.754},"min":{"1min":0.26600000000000001,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":0.56899999999999995,"5min":21.007999999999999,"15min":158.13399999999999},"last":0.31},{"interface":"front","average":{"1min":0.36899999999999999,"5min":1.8220000000000001,"15min":3.5819999999999999},"min":{"1min":0.26400000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.76100000000000001,"5min":20.797999999999998,"15min":161.12},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.9850000000000001,"15min":1.966},"min":{"1min":0.27700000000000002,"5min":0.22700000000000001,"15min":0.20200000000000001},"max":{"1min":0.71999999999999997,"5min":43.707999999999998,"15min":51.524999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":2.5310000000000001,"15min":2.4129999999999998},"min":{"1min":0.312,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":0.622,"5min":43.427999999999997,"15min":51.505000000000003},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.2410000000000001,"15min":2.4169999999999998},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.26500000000000001},"max":{"1min":0.79500000000000004,"5min":9.4149999999999991,"15min":196.57599999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.1020000000000001,"15min":1.802},"min":{"1min":0.40000000000000002,"5min":0.30099999999999999,"15min":0.26800000000000002},"max":{"1min":1.853,"5min":9.0289999999999999,"15min":201.81100000000001},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.349,"15min":2.5169999999999999},"min":{"1min":0.42999999999999999,"5min":0.34499999999999997,"15min":0.28299999999999997},"max":{"1min":0.90700000000000003,"5min":8.6699999999999999,"15min":190.06899999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.321,"15min":2.8650000000000002},"min":{"1min":0.45700000000000002,"5min":0.34200000000000003,"15min":0.30399999999999999},"max":{"1min":0.88500000000000001,"5min":9.8949999999999996,"15min":190.08500000000001},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.92100000000000004,"15min":1.25},"min":{"1min":0.44700000000000001,"5min":0.35399999999999998,"15min":0.32900000000000001},"max":{"1min":0.84899999999999998,"5min":9.0779999999999994,"15min":58.469999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.4419999999999999,"15min":1.833},"min":{"1min":0.47199999999999998,"5min":0.39800000000000002,"15min":0.32100000000000001},"max":{"1min":0.755,"5min":8.6989999999999998,"15min":58.067999999999998},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":1.2609999999999999,"15min":2.1509999999999998},"min":{"1min":0.46800000000000003,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":1.9199999999999999,"5min":9.1120000000000001,"15min":191.792},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.5760000000000001,"15min":2.8999999999999999},"min":{"1min":0.48299999999999998,"5min":0.42099999999999999,"15min":0.34499999999999997},"max":{"1min":1.9670000000000001,"5min":8.843,"15min":191.828},"last":0.53000000000000003}]}]},{"osd":3,"up_from":99,"seq":425201762675,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123228,"kb_used_data":8020,"kb_used_omap":7,"kb_used_meta":1115192,"kb_avail":92621924,"statfs":{"total":95995035648,"available":94844850176,"internally_reserved":0,"allocated":8212480,"data_stored":5508981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7576,"internal_metadata":1141957224},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":3.3410000000000002,"15min":3.427},"min":{"1min":0.248,"5min":0.189,"15min":0.17399999999999999},"max":{"1min":0.51300000000000001,"5min":119.149,"15min":119.97799999999999},"last":0.372},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.1050000000000004,"15min":3.9790000000000001},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.185},"max":{"1min":0.49399999999999999,"5min":119.251,"15min":120.307},"last":0.35299999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":3.2410000000000001,"15min":3.1709999999999998},"min":{"1min":0.247,"5min":0.19,"15min":0.19},"max":{"1min":0.501,"5min":121.842,"15min":170.196},"last":0.307},{"interface":"front","average":{"1min":0.36199999999999999,"5min":2.7999999999999998,"15min":2.9929999999999999},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":0.54400000000000004,"5min":119.673,"15min":166.953},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":1.2090000000000001,"15min":1.405},"min":{"1min":0.25900000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.49399999999999999,"5min":10.215999999999999,"15min":43.152999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":2.3490000000000002,"15min":2.0680000000000001},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.223},"max":{"1min":0.52500000000000002,"5min":17.463000000000001,"15min":43.134},"last":0.26000000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":3.1680000000000001,"15min":2.9260000000000002},"min":{"1min":0.39300000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":0.97299999999999998,"5min":83.010000000000005,"15min":211.84700000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.706,"15min":2.786},"min":{"1min":0.38700000000000001,"5min":0.308,"15min":0.29099999999999998},"max":{"1min":0.95499999999999996,"5min":82.915000000000006,"15min":211.17500000000001},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":2.6970000000000001,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.33600000000000002,"15min":0.28100000000000003},"max":{"1min":0.999,"5min":88.295000000000002,"15min":88.295000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.56899999999999995,"5min":2.1850000000000001,"15min":1.5740000000000001},"min":{"1min":0.372,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.94199999999999995,"5min":86.968999999999994,"15min":86.968999999999994},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.3029999999999999,"15min":1.4279999999999999},"min":{"1min":0.432,"5min":0.36599999999999999,"15min":0.32600000000000001},"max":{"1min":0.88400000000000001,"5min":21.529,"15min":64.378},"last":0.497},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.169,"15min":1.857},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.35999999999999999},"max":{"1min":0.91400000000000003,"5min":21.666,"15min":64.353999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":3.7749999999999999,"15min":3.2749999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":1.018,"5min":86.897000000000006,"15min":212.62700000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":3.363,"15min":3.1739999999999999},"min":{"1min":0.47599999999999998,"5min":0.36099999999999999,"15min":0.33300000000000002},"max":{"1min":0.90000000000000002,"5min":87.105999999999995,"15min":212.04599999999999},"last":0.622}]}]},{"osd":4,"up_from":31,"seq":133143986691,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920660,"kb_used_data":10636,"kb_used_omap":0,"kb_used_meta":910016,"kb_avail":92824492,"statfs":{"total":95995035648,"available":95052279808,"internally_reserved":0,"allocated":10891264,"data_stored":9222317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":931856384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":2.4020000000000001,"15min":4.6870000000000003},"min":{"1min":0.245,"5min":0.20999999999999999,"15min":0.20799999999999999},"max":{"1min":0.72899999999999998,"5min":110.95,"15min":207.45699999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":2.1739999999999999,"15min":4.734},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.223},"max":{"1min":1.111,"5min":111.057,"15min":207.114},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":2.0289999999999999,"15min":3.476},"min":{"1min":0.33500000000000002,"5min":0.23699999999999999,"15min":0.23599999999999999},"max":{"1min":1.2430000000000001,"5min":57.161999999999999,"15min":135.25399999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.956,"15min":3.391},"min":{"1min":0.309,"5min":0.26100000000000001,"15min":0.252},"max":{"1min":1.8520000000000001,"5min":55.222999999999999,"15min":134.44300000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:34:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.496,"15min":3.9689999999999999},"min":{"1min":0.34699999999999998,"5min":0.28799999999999998,"15min":0.23599999999999999},"max":{"1min":7.54,"5min":9.5809999999999995,"15min":203.61500000000001},"last":0.439},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.4670000000000001,"15min":3.9870000000000001},"min":{"1min":0.33100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.7610000000000001,"5min":9.2599999999999998,"15min":203.601},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 04:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":5.524,"15min":3.5019999999999998},"min":{"1min":0.35699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":6.2389999999999999,"5min":135.97800000000001,"15min":154.13200000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":5.2910000000000004,"15min":3.4470000000000001},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.79000000000000004,"5min":135.899,"15min":154.077},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:33:17 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":1.028,"15min":0.93999999999999995},"min":{"1min":0.313,"5min":0.248,"15min":0.23300000000000001},"max":{"1min":0.82299999999999995,"5min":16.753,"15min":16.753},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.504,"5min":1.0349999999999999,"15min":1.1180000000000001},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":0.70699999999999996,"5min":17.792000000000002,"15min":17.792000000000002},"last":0.52500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.93100000000000005,"15min":1.248},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.26700000000000002},"max":{"1min":1.0820000000000001,"5min":9.4120000000000008,"15min":47.801000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.88600000000000001,"15min":1.1339999999999999},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":1.1399999999999999,"5min":9.1760000000000002,"15min":47.753},"last":0.67600000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0740000000000001,"15min":1.115},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0549999999999999,"5min":12.587999999999999,"15min":12.587999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.03,"15min":1.29},"min":{"1min":0.42099999999999999,"5min":0.32300000000000001,"15min":0.28999999999999998},"max":{"1min":1.7669999999999999,"5min":12.863,"15min":12.863},"last":0.64500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725055,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899340,"kb_used_data":6788,"kb_used_omap":0,"kb_used_meta":892544,"kb_avail":92845812,"statfs":{"total":95995035648,"available":95074111488,"internally_reserved":0,"allocated":6950912,"data_stored":5286406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913965056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.9670000000000001,"15min":3.6579999999999999},"min":{"1min":0.34300000000000003,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.79200000000000004,"5min":64.186000000000007,"15min":139.86799999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":3.2909999999999999,"15min":3.851},"min":{"1min":0.39900000000000002,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":0.66500000000000004,"5min":63.746000000000002,"15min":139.84899999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":3.3410000000000002,"5min":5.9029999999999996,"15min":5.1909999999999998},"min":{"1min":0.33900000000000002,"5min":0.255,"15min":0.22600000000000001},"max":{"1min":63.229999999999997,"5min":131.86199999999999,"15min":296.23099999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":3.3490000000000002,"5min":5.649,"15min":5.2480000000000002},"min":{"1min":0.35399999999999998,"5min":0.26400000000000001,"15min":0.26200000000000001},"max":{"1min":63.212000000000003,"5min":132.74600000000001,"15min":295.065},"last":0.624}]},{"osd":2,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.655,"15min":3.1669999999999998},"min":{"1min":0.379,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.72199999999999998,"5min":64.335999999999999,"15min":168.779},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.776,"15min":3.282},"min":{"1min":0.41299999999999998,"5min":0.28899999999999998,"15min":0.25600000000000001},"max":{"1min":0.68500000000000005,"5min":65.811999999999998,"15min":168.03100000000001},"last":0.61399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":3.8359999999999999,"15min":2.6499999999999999},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.27800000000000002},"max":{"1min":0.73799999999999999,"5min":116.56,"15min":116.56},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":3.7330000000000001,"15min":2.5790000000000002},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":0.76500000000000001,"5min":116.527,"15min":116.527},"last":0.76200000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.69599999999999995,"15min":1.206},"min":{"1min":0.33000000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":0.77500000000000002,"5min":8.0180000000000007,"15min":95.668000000000006},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.67100000000000004,"15min":1.1830000000000001},"min":{"1min":0.36299999999999999,"5min":0.27900000000000003,"15min":0.26100000000000001},"max":{"1min":0.65500000000000003,"5min":5.4580000000000002,"15min":98.703999999999994},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":2.0619999999999998,"15min":1.2549999999999999},"min":{"1min":0.32700000000000001,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":0.79700000000000004,"5min":145.054,"15min":145.054},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":2.1760000000000002,"15min":1.3300000000000001},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.313},"max":{"1min":0.68600000000000005,"5min":144.85900000000001,"15min":144.85900000000001},"last":0.57699999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.76400000000000001,"15min":0.94999999999999996},"min":{"1min":0.41899999999999998,"5min":0.32100000000000001,"15min":0.26900000000000002},"max":{"1min":0.82099999999999995,"5min":8.282,"15min":9.3800000000000008},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.70899999999999996,"15min":1.036},"min":{"1min":0.47999999999999998,"5min":0.35299999999999998,"15min":0.33600000000000002},"max":{"1min":0.82299999999999995,"5min":3.7280000000000002,"15min":11.522},"last":0.65800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4575,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:34:20.658 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:34:20.658 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T04:34:20.726 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=8b296e7722894848ab98c7300e7c3af4 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T04:34:20.728 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T04:34:20.739 INFO:tasks.cephadm:Teardown begin 2023-10-09T04:34:20.740 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:34:20.799 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:34:20.864 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T04:34:20.865 DEBUG:teuthology.orchestra.run.smithi163:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T04:34:20.882 DEBUG:teuthology.orchestra.run.smithi181:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T04:34:20.901 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T04:34:20.901 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T04:34:20.901 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T04:34:20.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:20 smithi163 ceph-mon[221067]: from='client.47915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:34:20.946 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:20 smithi163 ceph-mon[221067]: pgmap v1660: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:20.947 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:20 smithi163 ceph-mon[213841]: from='client.47915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:34:20.947 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:20 smithi163 ceph-mon[213841]: pgmap v1660: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:20.947 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:20 smithi163 conmon[200567]: ::ffff:172.21.15.181 - - [09/Oct/2023:04:34:20] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2023-10-09T04:34:21.202 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:20 smithi163 systemd[1]: Stopping Ceph mon.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:20 smithi181 ceph-mon[196979]: from='client.47915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:34:21.269 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:20 smithi181 ceph-mon[196979]: pgmap v1660: 103 pgs: 103 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:34:21.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[213817]: 2023-10-09T04:34:21.200+0000 7fe170dc6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:21.502 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[213817]: 2023-10-09T04:34:21.200+0000 7fe170dc6700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T04:34:21.503 INFO:journalctl@ceph.mon.a.smithi163.stdout:Oct 09 04:34:21 smithi163 bash[702501]: 8efafb4ec97da0aa3b40923217a63a1d11f35fa64d38f55f3d998b443979ba74 2023-10-09T04:34:21.503 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[200567]: [09/Oct/2023:04:34:21] ENGINE Bus STOPPING 2023-10-09T04:34:21.503 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[200567]: [09/Oct/2023:04:34:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:34:21.503 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[200567]: [09/Oct/2023:04:34:21] ENGINE Bus STOPPED 2023-10-09T04:34:21.503 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[200567]: [09/Oct/2023:04:34:21] ENGINE Bus STARTING 2023-10-09T04:34:21.503 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[200567]: [09/Oct/2023:04:34:21] ENGINE Serving on http://:::9283 2023-10-09T04:34:21.504 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:21 smithi163 conmon[200567]: [09/Oct/2023:04:34:21] ENGINE Bus STARTED 2023-10-09T04:34:22.052 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T04:34:22.149 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:22.149 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T04:34:22.149 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T04:34:22.149 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T04:34:22.332 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:22 smithi163 systemd[1]: Stopping Ceph mon.c for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:22.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[221043]: 2023-10-09T04:34:22.448+0000 7fd50d316700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:22.744 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[221043]: 2023-10-09T04:34:22.448+0000 7fd50d316700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T04:34:22.744 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[200567]: [09/Oct/2023:04:34:22] ENGINE Bus STOPPING 2023-10-09T04:34:22.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[200567]: [09/Oct/2023:04:34:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:34:22.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[200567]: [09/Oct/2023:04:34:22] ENGINE Bus STOPPED 2023-10-09T04:34:22.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[200567]: [09/Oct/2023:04:34:22] ENGINE Bus STARTING 2023-10-09T04:34:22.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[200567]: [09/Oct/2023:04:34:22] ENGINE Serving on http://:::9283 2023-10-09T04:34:22.745 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:22 smithi163 conmon[200567]: [09/Oct/2023:04:34:22] ENGINE Bus STARTED 2023-10-09T04:34:23.081 INFO:journalctl@ceph.mon.c.smithi163.stdout:Oct 09 04:34:22 smithi163 bash[702811]: 0f3bd992ec5ce3e2a852faed7b3c37a692e3be22c53134201d86305665919d86 2023-10-09T04:34:23.319 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T04:34:23.359 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:23.360 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T04:34:23.360 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T04:34:23.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T04:34:23.649 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:23 smithi181 systemd[1]: Stopping Ceph mon.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:24.018 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:23 smithi181 conmon[196955]: 2023-10-09T04:34:23.647+0000 7f6c24010700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:24.019 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:23 smithi181 conmon[196955]: 2023-10-09T04:34:23.647+0000 7f6c24010700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T04:34:24.519 INFO:journalctl@ceph.mon.b.smithi181.stdout:Oct 09 04:34:24 smithi181 bash[419064]: 48065a0413d3bf0b4e7c3019910ed706495daee9c8f9fa3aacd623bd748de8bd 2023-10-09T04:34:24.607 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T04:34:24.654 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:24.654 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T04:34:24.655 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T04:34:24.655 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T04:34:24.955 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:24 smithi163 systemd[1]: Stopping Ceph mgr.y for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:24.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:24 smithi163 conmon[200567]: 2023-10-09T04:34:24.925+0000 7f40c251f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:24.956 INFO:journalctl@ceph.mgr.y.smithi163.stdout:Oct 09 04:34:24 smithi163 conmon[200567]: 2023-10-09T04:34:24.925+0000 7f40c251f700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T04:34:25.768 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T04:34:25.806 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:25.806 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T04:34:25.806 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T04:34:25.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T04:34:25.837 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:34:25 smithi181 conmon[180297]: ts=2023-10-09T04:34:25.565Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:34:25.837 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:34:25 smithi181 conmon[180297]: ts=2023-10-09T04:34:25.567Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:34:25.837 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:34:25 smithi181 conmon[180297]: ts=2023-10-09T04:34:25.567Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:34:25.837 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:34:25 smithi181 conmon[180297]: ts=2023-10-09T04:34:25.569Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:34:26.197 INFO:journalctl@ceph.mgr.x.smithi181.stdout:Oct 09 04:34:25 smithi181 systemd[1]: Stopping Ceph mgr.x for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:26.773 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T04:34:26.851 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:26.851 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T04:34:26.851 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T04:34:26.851 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T04:34:27.150 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:34:26 smithi163 systemd[1]: Stopping Ceph osd.0 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:27.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:34:27 smithi163 conmon[237889]: 2023-10-09T04:34:27.149+0000 7fa7a2481700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:27.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:34:27 smithi163 conmon[237889]: 2023-10-09T04:34:27.149+0000 7fa7a2481700 -1 osd.0 642 *** Got signal Terminated *** 2023-10-09T04:34:27.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:34:27 smithi163 conmon[237889]: 2023-10-09T04:34:27.149+0000 7fa7a2481700 -1 osd.0 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:34:33.582 INFO:journalctl@ceph.osd.0.smithi163.stdout:Oct 09 04:34:33 smithi163 bash[703420]: 663e3d75cc4756b9055330f64b604d7837cbb1f244e2e1aa10fb6e49206a88ef 2023-10-09T04:34:35.078 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T04:34:35.122 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:35.123 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T04:34:35.123 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T04:34:35.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T04:34:35.332 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:34:35 smithi163 systemd[1]: Stopping Ceph osd.1 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:35.832 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:34:35 smithi163 conmon[246279]: 2023-10-09T04:34:35.422+0000 7f4293dd2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:35.833 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:34:35 smithi163 conmon[246279]: 2023-10-09T04:34:35.422+0000 7f4293dd2700 -1 osd.1 642 *** Got signal Terminated *** 2023-10-09T04:34:35.833 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:34:35 smithi163 conmon[246279]: 2023-10-09T04:34:35.422+0000 7f4293dd2700 -1 osd.1 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:34:41.332 INFO:journalctl@ceph.osd.1.smithi163.stdout:Oct 09 04:34:41 smithi163 bash[703996]: e0fb1e6b8d2362088024bfe435f4cb76ac239c43926590d37e136557d9f5a2a8 2023-10-09T04:34:42.905 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T04:34:42.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:42.946 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T04:34:42.946 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T04:34:42.946 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T04:34:43.236 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:34:43 smithi163 systemd[1]: Stopping Ceph osd.2 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:43.582 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:34:43 smithi163 conmon[254030]: 2023-10-09T04:34:43.234+0000 7f054b461700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:43.582 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:34:43 smithi163 conmon[254030]: 2023-10-09T04:34:43.234+0000 7f054b461700 -1 osd.2 642 *** Got signal Terminated *** 2023-10-09T04:34:43.582 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:34:43 smithi163 conmon[254030]: 2023-10-09T04:34:43.234+0000 7f054b461700 -1 osd.2 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:34:49.832 INFO:journalctl@ceph.osd.2.smithi163.stdout:Oct 09 04:34:49 smithi163 bash[704583]: e242e555fe1b87e6b4e8c92fd76f92a5376ac32458cfe90c8a0c093af4d95aa5 2023-10-09T04:34:51.003 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T04:34:51.045 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:51.045 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T04:34:51.045 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T04:34:51.045 DEBUG:teuthology.orchestra.run.smithi163:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T04:34:51.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:51 smithi163 systemd[1]: Stopping Ceph osd.3 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:51.832 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:51 smithi163 conmon[261591]: 2023-10-09T04:34:51.342+0000 7fe5272f8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:51.832 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:51 smithi163 conmon[261591]: 2023-10-09T04:34:51.342+0000 7fe5272f8700 -1 osd.3 642 *** Got signal Terminated *** 2023-10-09T04:34:51.832 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:51 smithi163 conmon[261591]: 2023-10-09T04:34:51.342+0000 7fe5272f8700 -1 osd.3 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:34:53.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:52 smithi163 conmon[261591]: 2023-10-09T04:34:52.952+0000 7fe5230ed700 -1 osd.3 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:28.840946+0000 front 2023-10-09T04:34:28.840919+0000 (oldest deadline 2023-10-09T04:34:52.941002+0000) 2023-10-09T04:34:54.269 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:53 smithi181 conmon[112997]: 2023-10-09T04:34:53.879+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:34:54.270 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:53 smithi181 conmon[132062]: 2023-10-09T04:34:53.768+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:34:54.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:53 smithi163 conmon[261591]: 2023-10-09T04:34:53.948+0000 7fe5230ed700 -1 osd.3 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:28.840946+0000 front 2023-10-09T04:34:28.840919+0000 (oldest deadline 2023-10-09T04:34:52.941002+0000) 2023-10-09T04:34:55.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:54 smithi181 conmon[132062]: 2023-10-09T04:34:54.741+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:34:55.019 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:54 smithi181 conmon[112997]: 2023-10-09T04:34:54.882+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:34:55.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:54 smithi163 conmon[261591]: 2023-10-09T04:34:54.945+0000 7fe5230ed700 -1 osd.3 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:28.840946+0000 front 2023-10-09T04:34:28.840919+0000 (oldest deadline 2023-10-09T04:34:52.941002+0000) 2023-10-09T04:34:55.740 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:34:55 smithi181 conmon[141718]: 2023-10-09T04:34:55.483+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:34:56.018 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:55 smithi181 conmon[112997]: 2023-10-09T04:34:55.864+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:34:56.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:55 smithi181 conmon[132062]: 2023-10-09T04:34:55.739+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:34:56.332 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:55 smithi163 conmon[261591]: 2023-10-09T04:34:55.936+0000 7fe5230ed700 -1 osd.3 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:28.840946+0000 front 2023-10-09T04:34:28.840919+0000 (oldest deadline 2023-10-09T04:34:52.941002+0000) 2023-10-09T04:34:56.707 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:34:56 smithi181 conmon[141718]: 2023-10-09T04:34:56.436+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:34:57.018 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:56 smithi181 conmon[112997]: 2023-10-09T04:34:56.867+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:34:57.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:56 smithi181 conmon[132062]: 2023-10-09T04:34:56.706+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:34:57.582 INFO:journalctl@ceph.osd.3.smithi163.stdout:Oct 09 04:34:57 smithi163 bash[705163]: 859a75049c4733676092e751e2cccc1b34cdec12b6706bdf13225dea4431a4f5 2023-10-09T04:34:57.691 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:34:57 smithi181 conmon[141718]: 2023-10-09T04:34:57.439+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:34:58.019 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:57 smithi181 conmon[112997]: 2023-10-09T04:34:57.856+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:34:58.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:57 smithi181 conmon[132062]: 2023-10-09T04:34:57.690+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:34:58.020 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:34:57 smithi181 conmon[122624]: 2023-10-09T04:34:57.715+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:34:58.761 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:58 smithi181 conmon[132062]: 2023-10-09T04:34:58.671+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:34:58.761 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:34:58 smithi181 conmon[141718]: 2023-10-09T04:34:58.461+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:34:58.895 DEBUG:teuthology.orchestra.run.smithi163:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T04:34:58.938 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:34:58.938 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T04:34:58.938 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T04:34:58.938 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T04:34:59.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:34:58 smithi181 conmon[122624]: 2023-10-09T04:34:58.760+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:34:59.019 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:58 smithi181 conmon[112997]: 2023-10-09T04:34:58.829+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:34:59.019 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:58 smithi181 systemd[1]: Stopping Ceph osd.4 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:59.519 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[141718]: 2023-10-09T04:34:59.464+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:34:59.519 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[112997]: 2023-10-09T04:34:59.223+0000 7f474735e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:59.519 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[112997]: 2023-10-09T04:34:59.223+0000 7f474735e700 -1 osd.4 642 *** Got signal Terminated *** 2023-10-09T04:34:59.519 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[112997]: 2023-10-09T04:34:59.223+0000 7f474735e700 -1 osd.4 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:35:00.019 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[112997]: 2023-10-09T04:34:59.844+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:35:00.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[132062]: 2023-10-09T04:34:59.710+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:00.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:34:59 smithi181 conmon[122624]: 2023-10-09T04:34:59.747+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:00.712 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:00 smithi181 conmon[141718]: 2023-10-09T04:35:00.429+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:01.018 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:35:00 smithi181 conmon[112997]: 2023-10-09T04:35:00.840+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:35:01.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:00 smithi181 conmon[122624]: 2023-10-09T04:35:00.738+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:01.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:00 smithi181 conmon[132062]: 2023-10-09T04:35:00.711+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:01.768 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:01 smithi181 conmon[122624]: 2023-10-09T04:35:01.695+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:01.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:01 smithi181 conmon[132062]: 2023-10-09T04:35:01.698+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:01.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:01 smithi181 conmon[141718]: 2023-10-09T04:35:01.471+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:02.269 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:35:01 smithi181 conmon[112997]: 2023-10-09T04:35:01.872+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:35:02.768 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:02 smithi181 conmon[122624]: 2023-10-09T04:35:02.723+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:02.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:02 smithi181 conmon[132062]: 2023-10-09T04:35:02.741+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:02.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:02 smithi181 conmon[141718]: 2023-10-09T04:35:02.496+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:03.269 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:35:02 smithi181 conmon[112997]: 2023-10-09T04:35:02.840+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:35:03.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:03 smithi181 conmon[132062]: 2023-10-09T04:35:03.724+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:03.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:03 smithi181 conmon[141718]: 2023-10-09T04:35:03.517+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:04.271 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:35:03 smithi181 conmon[112997]: 2023-10-09T04:35:03.816+0000 7f474495e700 -1 osd.4 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:27.325703+0000 front 2023-10-09T04:34:27.325861+0000 (oldest deadline 2023-10-09T04:34:53.126351+0000) 2023-10-09T04:35:04.284 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:03 smithi181 conmon[122624]: 2023-10-09T04:35:03.768+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:05.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:04 smithi181 conmon[122624]: 2023-10-09T04:35:04.784+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:05.019 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:04 smithi181 conmon[141718]: 2023-10-09T04:35:04.546+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:05.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:04 smithi181 conmon[132062]: 2023-10-09T04:35:04.757+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:05.020 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:04 smithi181 conmon[132062]: 2023-10-09T04:35:04.757+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:05.514 INFO:journalctl@ceph.osd.4.smithi181.stdout:Oct 09 04:35:05 smithi181 bash[419670]: b4d4d04d2a3e659a06efbd280a581653eeeae516aba6e77035a018d5dbf3ba55 2023-10-09T04:35:05.514 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:05 smithi181 conmon[141718]: 2023-10-09T04:35:05.513+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:06.230 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:05 smithi181 conmon[122624]: 2023-10-09T04:35:05.791+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:06.230 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:05 smithi181 conmon[122624]: 2023-10-09T04:35:05.791+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:06.230 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:05 smithi181 conmon[132062]: 2023-10-09T04:35:05.803+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:06.230 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:05 smithi181 conmon[132062]: 2023-10-09T04:35:05.803+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:06.810 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:06 smithi181 conmon[141718]: 2023-10-09T04:35:06.535+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:06.811 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:06 smithi181 conmon[132062]: 2023-10-09T04:35:06.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:06.811 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:06 smithi181 conmon[132062]: 2023-10-09T04:35:06.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:06.897 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T04:35:06.939 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:35:06.939 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T04:35:06.939 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T04:35:06.939 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T04:35:07.240 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:06 smithi181 conmon[122624]: 2023-10-09T04:35:06.810+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:07.240 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:06 smithi181 conmon[122624]: 2023-10-09T04:35:06.810+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:07.241 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:07 smithi181 systemd[1]: Stopping Ceph osd.5 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:07.241 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[122624]: 2023-10-09T04:35:07.239+0000 7fc826304700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:35:07.518 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[122624]: 2023-10-09T04:35:07.239+0000 7fc826304700 -1 osd.5 642 *** Got signal Terminated *** 2023-10-09T04:35:07.519 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[122624]: 2023-10-09T04:35:07.239+0000 7fc826304700 -1 osd.5 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:35:08.018 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[122624]: 2023-10-09T04:35:07.776+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:08.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[122624]: 2023-10-09T04:35:07.776+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:08.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[132062]: 2023-10-09T04:35:07.706+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:08.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[132062]: 2023-10-09T04:35:07.706+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:08.020 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[141718]: 2023-10-09T04:35:07.554+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:08.020 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:07 smithi181 conmon[141718]: 2023-10-09T04:35:07.554+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:08.768 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:08 smithi181 conmon[132062]: 2023-10-09T04:35:08.725+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:08.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:08 smithi181 conmon[132062]: 2023-10-09T04:35:08.725+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:08.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:08 smithi181 conmon[141718]: 2023-10-09T04:35:08.507+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:08.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:08 smithi181 conmon[141718]: 2023-10-09T04:35:08.507+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:09.268 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:08 smithi181 conmon[122624]: 2023-10-09T04:35:08.782+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:09.269 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:08 smithi181 conmon[122624]: 2023-10-09T04:35:08.782+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:09.755 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[132062]: 2023-10-09T04:35:09.694+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:09.755 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[132062]: 2023-10-09T04:35:09.694+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:09.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[141718]: 2023-10-09T04:35:09.463+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:09.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[141718]: 2023-10-09T04:35:09.463+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:09.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[141718]: 2023-10-09T04:35:09.463+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:10.018 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[122624]: 2023-10-09T04:35:09.755+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:10.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:09 smithi181 conmon[122624]: 2023-10-09T04:35:09.755+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:10.751 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[132062]: 2023-10-09T04:35:10.707+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:10.751 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[132062]: 2023-10-09T04:35:10.707+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:10.751 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[132062]: 2023-10-09T04:35:10.707+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:10.751 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[141718]: 2023-10-09T04:35:10.490+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:10.752 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[141718]: 2023-10-09T04:35:10.490+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:10.752 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[141718]: 2023-10-09T04:35:10.490+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:11.018 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[122624]: 2023-10-09T04:35:10.750+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:11.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[122624]: 2023-10-09T04:35:10.750+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:11.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:10 smithi181 conmon[122624]: 2023-10-09T04:35:10.750+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.957623+0000 front 2023-10-09T04:34:46.957532+0000 (oldest deadline 2023-10-09T04:35:10.457463+0000) 2023-10-09T04:35:11.732 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[132062]: 2023-10-09T04:35:11.721+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:11.733 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[132062]: 2023-10-09T04:35:11.721+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:11.733 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[132062]: 2023-10-09T04:35:11.721+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:11.733 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[141718]: 2023-10-09T04:35:11.480+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:11.733 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[141718]: 2023-10-09T04:35:11.480+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:11.733 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[141718]: 2023-10-09T04:35:11.480+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:12.018 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[122624]: 2023-10-09T04:35:11.732+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:31.155542+0000 front 2023-10-09T04:34:31.155443+0000 (oldest deadline 2023-10-09T04:34:57.055439+0000) 2023-10-09T04:35:12.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[122624]: 2023-10-09T04:35:11.732+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:39.956691+0000 front 2023-10-09T04:34:39.956634+0000 (oldest deadline 2023-10-09T04:35:05.256590+0000) 2023-10-09T04:35:12.019 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:11 smithi181 conmon[122624]: 2023-10-09T04:35:11.732+0000 7fc824105700 -1 osd.5 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.957623+0000 front 2023-10-09T04:34:46.957532+0000 (oldest deadline 2023-10-09T04:35:10.457463+0000) 2023-10-09T04:35:12.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:12 smithi181 conmon[132062]: 2023-10-09T04:35:12.712+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:12.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:12 smithi181 conmon[132062]: 2023-10-09T04:35:12.712+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:12.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:12 smithi181 conmon[132062]: 2023-10-09T04:35:12.712+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:12.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:12 smithi181 conmon[141718]: 2023-10-09T04:35:12.512+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:12.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:12 smithi181 conmon[141718]: 2023-10-09T04:35:12.512+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:12.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:12 smithi181 conmon[141718]: 2023-10-09T04:35:12.512+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:13.268 INFO:journalctl@ceph.osd.5.smithi181.stdout:Oct 09 04:35:12 smithi181 bash[420255]: 4ed298059fdfe493d88fcee4c523ffad12866781445056b4e3119dbcebc14c02 2023-10-09T04:35:13.768 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:13 smithi181 conmon[132062]: 2023-10-09T04:35:13.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:13.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:13 smithi181 conmon[132062]: 2023-10-09T04:35:13.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:13.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:13 smithi181 conmon[132062]: 2023-10-09T04:35:13.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:13.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:13 smithi181 conmon[141718]: 2023-10-09T04:35:13.511+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:13.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:13 smithi181 conmon[141718]: 2023-10-09T04:35:13.511+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:13.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:13 smithi181 conmon[141718]: 2023-10-09T04:35:13.511+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:14.687 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T04:35:14.728 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:35:14.728 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T04:35:14.728 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T04:35:14.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T04:35:14.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:14 smithi181 conmon[141718]: 2023-10-09T04:35:14.478+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:14.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:14 smithi181 conmon[141718]: 2023-10-09T04:35:14.478+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:14.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:14 smithi181 conmon[141718]: 2023-10-09T04:35:14.478+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:15.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:14 smithi181 conmon[132062]: 2023-10-09T04:35:14.798+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:15.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:14 smithi181 conmon[132062]: 2023-10-09T04:35:14.798+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:15.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:14 smithi181 conmon[132062]: 2023-10-09T04:35:14.798+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:15.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:14 smithi181 systemd[1]: Stopping Ceph osd.6 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:15.270 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[132062]: 2023-10-09T04:35:15.021+0000 7fbdeb742700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:35:15.270 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[132062]: 2023-10-09T04:35:15.021+0000 7fbdeb742700 -1 osd.6 642 *** Got signal Terminated *** 2023-10-09T04:35:15.270 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[132062]: 2023-10-09T04:35:15.021+0000 7fbdeb742700 -1 osd.6 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:35:15.758 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[141718]: 2023-10-09T04:35:15.456+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:15.758 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[141718]: 2023-10-09T04:35:15.456+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:15.758 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[141718]: 2023-10-09T04:35:15.456+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:16.018 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[132062]: 2023-10-09T04:35:15.757+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:16.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[132062]: 2023-10-09T04:35:15.757+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:16.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:15 smithi181 conmon[132062]: 2023-10-09T04:35:15.757+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:16.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:16 smithi181 conmon[132062]: 2023-10-09T04:35:16.737+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:16.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:16 smithi181 conmon[132062]: 2023-10-09T04:35:16.737+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:16.769 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:16 smithi181 conmon[132062]: 2023-10-09T04:35:16.737+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:16.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:16 smithi181 conmon[141718]: 2023-10-09T04:35:16.500+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:16.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:16 smithi181 conmon[141718]: 2023-10-09T04:35:16.500+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:16.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:16 smithi181 conmon[141718]: 2023-10-09T04:35:16.500+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:17.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[141718]: 2023-10-09T04:35:17.513+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:17.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[141718]: 2023-10-09T04:35:17.513+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:17.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[141718]: 2023-10-09T04:35:17.513+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:18.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[132062]: 2023-10-09T04:35:17.778+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:18.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[132062]: 2023-10-09T04:35:17.778+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:18.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[132062]: 2023-10-09T04:35:17.778+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:18.269 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:17 smithi181 conmon[132062]: 2023-10-09T04:35:17.778+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:54.607306+0000 front 2023-10-09T04:34:54.607354+0000 (oldest deadline 2023-10-09T04:35:16.907234+0000) 2023-10-09T04:35:19.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[132062]: 2023-10-09T04:35:18.790+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:19.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[132062]: 2023-10-09T04:35:18.790+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:19.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[132062]: 2023-10-09T04:35:18.790+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:19.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[132062]: 2023-10-09T04:35:18.790+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:54.607306+0000 front 2023-10-09T04:34:54.607354+0000 (oldest deadline 2023-10-09T04:35:16.907234+0000) 2023-10-09T04:35:19.020 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[141718]: 2023-10-09T04:35:18.542+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:19.020 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[141718]: 2023-10-09T04:35:18.542+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:19.020 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:18 smithi181 conmon[141718]: 2023-10-09T04:35:18.542+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:19.755 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[141718]: 2023-10-09T04:35:19.504+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:19.755 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[141718]: 2023-10-09T04:35:19.504+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:19.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[141718]: 2023-10-09T04:35:19.504+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:20.018 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[132062]: 2023-10-09T04:35:19.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.604534+0000 front 2023-10-09T04:34:30.604623+0000 (oldest deadline 2023-10-09T04:34:53.504423+0000) 2023-10-09T04:35:20.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[132062]: 2023-10-09T04:35:19.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:38.805480+0000 front 2023-10-09T04:34:38.805376+0000 (oldest deadline 2023-10-09T04:35:04.105316+0000) 2023-10-09T04:35:20.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[132062]: 2023-10-09T04:35:19.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:47.606377+0000 front 2023-10-09T04:34:47.606352+0000 (oldest deadline 2023-10-09T04:35:10.506217+0000) 2023-10-09T04:35:20.019 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:19 smithi181 conmon[132062]: 2023-10-09T04:35:19.755+0000 7fbde6d3e700 -1 osd.6 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:54.607306+0000 front 2023-10-09T04:34:54.607354+0000 (oldest deadline 2023-10-09T04:35:16.907234+0000) 2023-10-09T04:35:20.768 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:20 smithi181 conmon[141718]: 2023-10-09T04:35:20.484+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:20.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:20 smithi181 conmon[141718]: 2023-10-09T04:35:20.484+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:20.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:20 smithi181 conmon[141718]: 2023-10-09T04:35:20.484+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:20.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:20 smithi181 conmon[141718]: 2023-10-09T04:35:20.484+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:21.444 INFO:journalctl@ceph.osd.6.smithi181.stdout:Oct 09 04:35:21 smithi181 bash[420840]: 4d54b928e978225e668448fe1448963b372d1144ec0c6a9f3c3f5faeab2e2954 2023-10-09T04:35:21.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:21 smithi181 conmon[141718]: 2023-10-09T04:35:21.444+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:21.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:21 smithi181 conmon[141718]: 2023-10-09T04:35:21.444+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:21.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:21 smithi181 conmon[141718]: 2023-10-09T04:35:21.444+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:21.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:21 smithi181 conmon[141718]: 2023-10-09T04:35:21.444+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:22.504 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T04:35:22.543 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:35:22.544 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T04:35:22.544 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T04:35:22.544 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T04:35:22.768 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.421+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:22.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.421+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:22.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.421+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:22.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.421+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:22.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 systemd[1]: Stopping Ceph osd.7 for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:23.268 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.816+0000 7f1554053700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:35:23.269 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.816+0000 7f1554053700 -1 osd.7 642 *** Got signal Terminated *** 2023-10-09T04:35:23.269 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:22 smithi181 conmon[141718]: 2023-10-09T04:35:22.816+0000 7f1554053700 -1 osd.7 642 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:35:23.768 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:23 smithi181 conmon[141718]: 2023-10-09T04:35:23.384+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:23.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:23 smithi181 conmon[141718]: 2023-10-09T04:35:23.384+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:23.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:23 smithi181 conmon[141718]: 2023-10-09T04:35:23.384+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:23.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:23 smithi181 conmon[141718]: 2023-10-09T04:35:23.384+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:24.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:24 smithi181 conmon[141718]: 2023-10-09T04:35:24.430+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:24.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:24 smithi181 conmon[141718]: 2023-10-09T04:35:24.430+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:24.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:24 smithi181 conmon[141718]: 2023-10-09T04:35:24.430+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:24.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:24 smithi181 conmon[141718]: 2023-10-09T04:35:24.430+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:25.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[141718]: 2023-10-09T04:35:25.388+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:25.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[141718]: 2023-10-09T04:35:25.388+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:25.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[141718]: 2023-10-09T04:35:25.388+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:25.770 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[141718]: 2023-10-09T04:35:25.388+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:25.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[180297]: ts=2023-10-09T04:35:25.565Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:35:25.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[180297]: ts=2023-10-09T04:35:25.568Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:35:25.771 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[180297]: ts=2023-10-09T04:35:25.568Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:35:25.771 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:25 smithi181 conmon[180297]: ts=2023-10-09T04:35:25.569Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.163:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.163:8765: connect: connection refused" 2023-10-09T04:35:26.768 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:26 smithi181 conmon[141718]: 2023-10-09T04:35:26.343+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:26.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:26 smithi181 conmon[141718]: 2023-10-09T04:35:26.343+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:26.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:26 smithi181 conmon[141718]: 2023-10-09T04:35:26.343+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:26.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:26 smithi181 conmon[141718]: 2023-10-09T04:35:26.343+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:26.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:26 smithi181 conmon[141718]: 2023-10-09T04:35:26.343+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2023-10-09T04:35:01.424045+0000 front 2023-10-09T04:35:01.424097+0000 (oldest deadline 2023-10-09T04:35:25.524299+0000) 2023-10-09T04:35:27.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:27 smithi181 conmon[141718]: 2023-10-09T04:35:27.391+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6806 osd.0 since back 2023-10-09T04:34:30.218445+0000 front 2023-10-09T04:34:30.218403+0000 (oldest deadline 2023-10-09T04:34:54.819022+0000) 2023-10-09T04:35:27.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:27 smithi181 conmon[141718]: 2023-10-09T04:35:27.391+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6814 osd.1 since back 2023-10-09T04:34:34.819443+0000 front 2023-10-09T04:34:34.819480+0000 (oldest deadline 2023-10-09T04:35:06.620068+0000) 2023-10-09T04:35:27.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:27 smithi181 conmon[141718]: 2023-10-09T04:35:27.391+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6822 osd.2 since back 2023-10-09T04:34:46.620615+0000 front 2023-10-09T04:34:46.620640+0000 (oldest deadline 2023-10-09T04:35:09.321464+0000) 2023-10-09T04:35:27.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:27 smithi181 conmon[141718]: 2023-10-09T04:35:27.391+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.163:6830 osd.3 since back 2023-10-09T04:34:52.123019+0000 front 2023-10-09T04:34:52.123048+0000 (oldest deadline 2023-10-09T04:35:19.723284+0000) 2023-10-09T04:35:27.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:27 smithi181 conmon[141718]: 2023-10-09T04:35:27.391+0000 7f154f64f700 -1 osd.7 642 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2023-10-09T04:35:01.424045+0000 front 2023-10-09T04:35:01.424097+0000 (oldest deadline 2023-10-09T04:35:25.524299+0000) 2023-10-09T04:35:28.769 INFO:journalctl@ceph.osd.7.smithi181.stdout:Oct 09 04:35:28 smithi181 bash[421418]: 6ca0d6954129b5266a72c8497b344602141ca73adaf4094ec53d87ebf6530141 2023-10-09T04:35:30.120 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T04:35:30.159 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:35:30.159 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T04:35:30.159 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T04:35:30.159 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-8628d772-6656-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T04:35:30.445 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 systemd[1]: Stopping Ceph prometheus.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:30.768 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:35:30.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:35:30.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:35:30.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:35:30.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:35:30.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:35:30.769 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:35:30.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.444Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:35:30.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.445Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:35:30.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.446Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:35:30.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.446Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:35:30.770 INFO:journalctl@ceph.prometheus.a.smithi181.stdout:Oct 09 04:35:30 smithi181 conmon[180297]: ts=2023-10-09T04:35:30.446Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:35:31.081 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8628d772-6656-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T04:35:31.117 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:35:31.117 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T04:35:31.117 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T04:35:32.515 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 04:35:32 smithi163 systemd[1]: Stopping Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:32.832 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 04:35:32 smithi163 conmon[198775]: ts=2023-10-09T04:35:32.514Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:35:33.415 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 04:35:33 smithi163 bash[706043]: b04e9e96f8bb09f7e5ecd897d9b220334af258e994ff6eb448298e21e8041138 2023-10-09T04:35:33.415 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 04:35:33 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:35:33.416 INFO:journalctl@ceph.alertmanager.a.smithi163.stdout:Oct 09 04:35:33 smithi163 systemd[1]: Stopped Ceph alertmanager.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:35:33.764 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 04:35:33 smithi163 systemd[1]: Stopping Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:34.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 04:35:34 smithi163 bash[706322]: 381d36940ced53d4c603b982f93ce59bccc0ee879e1367b2347a7f345de936e7 2023-10-09T04:35:34.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 04:35:34 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:35:34.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 04:35:34 smithi163 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T04:35:34.332 INFO:journalctl@ceph.node-exporter.a.smithi163.stdout:Oct 09 04:35:34 smithi163 systemd[1]: Stopped Ceph node-exporter.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:35:40.595 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T04:35:42.019 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:35:41 smithi181 systemd[1]: Stopping Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:42.690 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 04:35:42 smithi181 systemd[1]: Stopping Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7... 2023-10-09T04:35:42.690 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:35:42 smithi181 bash[422599]: dd057a75b8e1e522a020cbc70ec6a2f4fedc8fdc69b3c2793ef7faa3eb90b162 2023-10-09T04:35:42.691 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:35:42 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:35:42.691 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:35:42 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T04:35:42.691 INFO:journalctl@ceph.grafana.a.smithi181.stdout:Oct 09 04:35:42 smithi181 systemd[1]: Stopped Ceph grafana.a for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:35:43.618 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 04:35:43 smithi181 bash[422877]: 79b2ee0a484da009b46c07c10df30eb4c800df626e4a7c230f73af7e9557bc18 2023-10-09T04:35:43.619 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 04:35:43 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:35:43.619 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 04:35:43 smithi181 systemd[1]: ceph-8628d772-6656-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T04:35:43.619 INFO:journalctl@ceph.node-exporter.b.smithi181.stdout:Oct 09 04:35:43 smithi181 systemd[1]: Stopped Ceph node-exporter.b for 8628d772-6656-11ee-8db5-212e2dc638e7. 2023-10-09T04:35:48.935 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:35:48.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:35:49.003 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T04:35:49.005 DEBUG:teuthology.misc:Transferring archived files from smithi163:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/remote/smithi163/crash 2023-10-09T04:35:49.006 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T04:35:49.063 INFO:teuthology.orchestra.run.smithi163.stderr:tar: /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T04:35:49.063 INFO:teuthology.orchestra.run.smithi163.stderr:tar: Error is not recoverable: exiting now 2023-10-09T04:35:49.065 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/remote/smithi181/crash 2023-10-09T04:35:49.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T04:35:49.122 INFO:teuthology.orchestra.run.smithi181.stderr:tar: /var/lib/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T04:35:49.122 INFO:teuthology.orchestra.run.smithi181.stderr:tar: Error is not recoverable: exiting now 2023-10-09T04:35:49.123 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T04:35:49.123 DEBUG:teuthology.orchestra.run.smithi163:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T04:35:49.152 INFO:teuthology.orchestra.run.smithi163.stderr:grep: /var/log/ceph/8628d772-6656-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T04:35:49.154 INFO:tasks.cephadm:Compressing logs... 2023-10-09T04:35:49.154 DEBUG:teuthology.orchestra.run.smithi163:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T04:35:49.197 DEBUG:teuthology.orchestra.run.smithi181:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T04:35:49.224 INFO:teuthology.orchestra.run.smithi163.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T04:35:49.239 INFO:teuthology.orchestra.run.smithi181.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T04:44:32.079 INFO:tasks.cephadm:Archiving logs... 2023-10-09T04:44:32.081 DEBUG:teuthology.misc:Transferring archived files from smithi163:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/remote/smithi163/log 2023-10-09T04:44:32.082 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T04:45:42.730 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/remote/smithi181/log 2023-10-09T04:45:42.731 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T04:46:30.402 INFO:tasks.cephadm:Removing cluster... 2023-10-09T04:46:30.403 DEBUG:teuthology.orchestra.run.smithi163:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 --force 2023-10-09T04:46:31.512 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8628d772-6656-11ee-8db5-212e2dc638e7 --force 2023-10-09T04:46:32.543 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T04:46:32.646 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T04:46:32.668 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T04:46:32.706 INFO:tasks.cephadm:Teardown complete 2023-10-09T04:46:32.707 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T04:46:32.717 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T04:46:32.717 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T04:46:32.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T04:46:32.842 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T04:46:32.852 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T04:46:32.852 DEBUG:teuthology.orchestra.run.smithi163:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T04:46:32.855 DEBUG:teuthology.orchestra.run.smithi181:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T04:46:32.871 INFO:teuthology.orchestra.run.smithi181.stderr:bash: ntpq: command not found 2023-10-09T04:46:32.872 INFO:teuthology.orchestra.run.smithi163.stderr:bash: ntpq: command not found 2023-10-09T04:46:32.885 INFO:teuthology.orchestra.run.smithi163.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T04:46:32.885 INFO:teuthology.orchestra.run.smithi163.stdout:=============================================================================== 2023-10-09T04:46:32.885 INFO:teuthology.orchestra.run.smithi163.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 54 -479us[ -479us] +/- 86ms 2023-10-09T04:46:32.885 INFO:teuthology.orchestra.run.smithi163.stdout:^* hv02.front.sepia.ceph.com 3 6 377 56 +48us[ +38us] +/- 48ms 2023-10-09T04:46:32.885 INFO:teuthology.orchestra.run.smithi163.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 +334us[ +334us] +/- 87ms 2023-10-09T04:46:32.886 INFO:teuthology.orchestra.run.smithi163.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T04:46:32.895 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T04:46:32.895 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2023-10-09T04:46:32.895 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 56 -480us[ -461us] +/- 86ms 2023-10-09T04:46:32.895 INFO:teuthology.orchestra.run.smithi181.stdout:^* hv02.front.sepia.ceph.com 3 6 377 55 +103us[ +122us] +/- 48ms 2023-10-09T04:46:32.896 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 +337us[ +337us] +/- 87ms 2023-10-09T04:46:32.896 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T04:46:32.896 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T04:46:32.906 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T04:46:32.907 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T04:46:32.916 DEBUG:teuthology.orchestra.run.smithi163:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T04:46:33.034 DEBUG:teuthology.orchestra.run.smithi181:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T04:46:33.131 DEBUG:teuthology.orchestra.run.smithi163:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T04:46:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.165 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T04:46:33.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:33.196 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T04:46:33.207 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T04:46:33.250 INFO:teuthology.task.internal:Duration was 4436.147518 seconds 2023-10-09T04:46:33.250 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T04:46:33.259 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T04:46:33.260 DEBUG:teuthology.orchestra.run.smithi163:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T04:46:33.262 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T04:46:33.329 INFO:teuthology.orchestra.run.smithi163.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T04:46:33.341 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T04:46:33.612 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T04:46:33.613 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi163.front.sepia.ceph.com 2023-10-09T04:46:33.613 DEBUG:teuthology.orchestra.run.smithi163:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T04:46:33.634 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi181.front.sepia.ceph.com 2023-10-09T04:46:33.635 DEBUG:teuthology.orchestra.run.smithi181:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T04:46:33.685 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T04:46:33.685 DEBUG:teuthology.orchestra.run.smithi163:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T04:46:33.688 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T04:46:33.846 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T04:46:33.846 DEBUG:teuthology.orchestra.run.smithi163:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T04:46:33.849 DEBUG:teuthology.orchestra.run.smithi181:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T04:46:34.188 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T04:46:34.199 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T04:46:34.200 DEBUG:teuthology.orchestra.run.smithi163:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T04:46:34.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T04:46:34.278 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T04:46:34.290 DEBUG:teuthology.orchestra.run.smithi163:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T04:46:34.293 DEBUG:teuthology.orchestra.run.smithi181:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T04:46:34.318 INFO:teuthology.orchestra.run.smithi163.stdout:kernel.core_pattern = core 2023-10-09T04:46:34.355 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = core 2023-10-09T04:46:34.396 DEBUG:teuthology.orchestra.run.smithi163:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T04:46:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.413 DEBUG:teuthology.orchestra.run.smithi181:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T04:46:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.452 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T04:46:34.464 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T04:46:34.465 DEBUG:teuthology.misc:Transferring archived files from smithi163:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/remote/smithi163 2023-10-09T04:46:34.465 DEBUG:teuthology.orchestra.run.smithi163:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T04:46:34.528 DEBUG:teuthology.misc:Transferring archived files from smithi181:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444/remote/smithi181 2023-10-09T04:46:34.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T04:46:34.574 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T04:46:34.575 DEBUG:teuthology.orchestra.run.smithi163:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T04:46:34.577 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T04:46:34.622 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T04:46:34.635 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T04:46:34.635 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T04:46:34.647 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T04:46:34.647 DEBUG:teuthology.orchestra.run.smithi163:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T04:46:34.650 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T04:46:34.664 INFO:teuthology.orchestra.run.smithi163.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 04:46 /home/ubuntu/cephtest 2023-10-09T04:46:34.664 INFO:teuthology.orchestra.run.smithi163.stdout: 3932164 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 04:32 /home/ubuntu/cephtest/mnt.0 2023-10-09T04:46:34.664 INFO:teuthology.orchestra.run.smithi163.stdout: 3932170 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 04:32 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T04:46:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T04:46:34.674 INFO:teuthology.orchestra.run.smithi163.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T04:46:34.675 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi163 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T04:46:34.682 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T04:46:34.693 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T04:46:34.704 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi163 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T04:46:34.705 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T04:46:34.712 INFO:teuthology.orchestra.run.smithi181.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 04:46 /home/ubuntu/cephtest 2023-10-09T04:46:34.743 DEBUG:teuthology.nuke:shortname: smithi163 2023-10-09T04:46:34.744 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T04:46:34.766 DEBUG:teuthology.nuke:shortname: smithi181 2023-10-09T04:46:34.766 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T04:46:34.769 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi163.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:26:37.266921', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T04:46:34.796 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417444', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:26:37.268663', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T04:46:34.826 INFO:teuthology.orchestra.console.smithi163:Power off 2023-10-09T04:46:34.826 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T04:46:34.840 INFO:teuthology.orchestra.console.smithi181:Power off 2023-10-09T04:46:34.841 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T04:46:34.849 DEBUG:teuthology.orchestra.console.smithi163:power off output: Chassis Power Control: Down/Off 2023-10-09T04:46:34.849 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:34.862 DEBUG:teuthology.orchestra.console.smithi181:power off output: Chassis Power Control: Down/Off 2023-10-09T04:46:34.863 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:34.872 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is on 2023-10-09T04:46:34.886 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2023-10-09T04:46:38.874 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:38.887 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:38.984 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is on 2023-10-09T04:46:38.998 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2023-10-09T04:46:42.990 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:42.999 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:43.100 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is on 2023-10-09T04:46:43.109 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is off 2023-10-09T04:46:43.210 INFO:teuthology.orchestra.console.smithi181:Power off completed 2023-10-09T04:46:47.105 DEBUG:teuthology.orchestra.console.smithi163:pexpect command: ipmitool -H smithi163.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T04:46:47.215 DEBUG:teuthology.orchestra.console.smithi163:check power output: Chassis Power is off 2023-10-09T04:46:47.315 INFO:teuthology.orchestra.console.smithi163:Power off completed 2023-10-09T04:46:47.422 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 4436.147517681122 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=8b296e7722894848ab98c7300e7c3af4 status: fail success: false 2023-10-09T04:46:47.422 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T04:46:47.508 INFO:teuthology.run:FAIL